llm-eval-ci 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_eval_ci-0.1.0/LICENSE +21 -0
- llm_eval_ci-0.1.0/PKG-INFO +150 -0
- llm_eval_ci-0.1.0/README.md +128 -0
- llm_eval_ci-0.1.0/pyproject.toml +48 -0
- llm_eval_ci-0.1.0/setup.cfg +4 -0
- llm_eval_ci-0.1.0/src/llm_eval/__init__.py +44 -0
- llm_eval_ci-0.1.0/src/llm_eval/__main__.py +6 -0
- llm_eval_ci-0.1.0/src/llm_eval/adapters/__init__.py +6 -0
- llm_eval_ci-0.1.0/src/llm_eval/adapters/base.py +68 -0
- llm_eval_ci-0.1.0/src/llm_eval/adapters/openai_compat.py +69 -0
- llm_eval_ci-0.1.0/src/llm_eval/adapters/stub.py +89 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/__init__.py +46 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/base.py +44 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/budgets.py +78 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/content.py +50 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/cost.py +38 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/injection.py +48 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/json_checks.py +86 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/jsonschema_lite.py +37 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/latency.py +20 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/pii.py +27 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/refusal.py +32 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/regex.py +49 -0
- llm_eval_ci-0.1.0/src/llm_eval/checks/security.py +103 -0
- llm_eval_ci-0.1.0/src/llm_eval/cli.py +186 -0
- llm_eval_ci-0.1.0/src/llm_eval/models.py +71 -0
- llm_eval_ci-0.1.0/src/llm_eval/pytest_plugin.py +123 -0
- llm_eval_ci-0.1.0/src/llm_eval/reporters/__init__.py +11 -0
- llm_eval_ci-0.1.0/src/llm_eval/reporters/json.py +64 -0
- llm_eval_ci-0.1.0/src/llm_eval/reporters/json_report.py +76 -0
- llm_eval_ci-0.1.0/src/llm_eval/reporters/markdown.py +116 -0
- llm_eval_ci-0.1.0/src/llm_eval/suite.py +383 -0
- llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/PKG-INFO +150 -0
- llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/SOURCES.txt +48 -0
- llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/dependency_links.txt +1 -0
- llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/entry_points.txt +5 -0
- llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/top_level.txt +1 -0
- llm_eval_ci-0.1.0/tests/test_checks_budgets.py +106 -0
- llm_eval_ci-0.1.0/tests/test_checks_content.py +74 -0
- llm_eval_ci-0.1.0/tests/test_checks_json.py +120 -0
- llm_eval_ci-0.1.0/tests/test_checks_refusal.py +28 -0
- llm_eval_ci-0.1.0/tests/test_checks_registry.py +60 -0
- llm_eval_ci-0.1.0/tests/test_checks_security.py +71 -0
- llm_eval_ci-0.1.0/tests/test_cli.py +113 -0
- llm_eval_ci-0.1.0/tests/test_models.py +53 -0
- llm_eval_ci-0.1.0/tests/test_plugin.py +73 -0
- llm_eval_ci-0.1.0/tests/test_reporters.py +81 -0
- llm_eval_ci-0.1.0/tests/test_suite.py +207 -0
- llm_eval_ci-0.1.0/tests/test_t523_checks.py +226 -0
- llm_eval_ci-0.1.0/tests/test_t523_suite_cli.py +242 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 llm-eval-ci contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: llm-eval-ci
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: pytest for LLMs and agents in CI — deterministic, offline-first evals (stdlib only)
|
|
5
|
+
Author: llm-eval-ci
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/shenn667/llm-eval-ci
|
|
8
|
+
Keywords: llm,eval,testing,ci,agents
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# llm-eval-ci
|
|
24
|
+
|
|
25
|
+
<!-- Replace YOUR_ORG with the real GitHub org/user when the repo is published. -->
|
|
26
|
+
|
|
27
|
+
[](https://github.com/shenn667/llm-eval-ci/actions/workflows/ci.yml)
|
|
28
|
+
[](https://pypi.org/project/llm-eval-ci/)
|
|
29
|
+
[](https://www.python.org/downloads/)
|
|
30
|
+
[](LICENSE)
|
|
31
|
+
|
|
32
|
+
`pytest` for LLMs and agents, in CI. Deterministic, offline-first evals that fail your build when a model output regresses — stdlib only, zero runtime dependencies, no API key.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
llm-eval run ./examples/basic.json --report ./report.md --json ./report.json
|
|
36
|
+
# exit code != 0 as soon as one case breaks a threshold
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Install (30 seconds)
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pip install llm-eval-ci
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
From source:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
git clone https://github.com/shenn667/llm-eval-ci.git
|
|
49
|
+
cd llm-eval-ci
|
|
50
|
+
pip install -e .
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Requirements: Python 3.10+, nothing else at runtime (`argparse`, `json`, `re`, `time` from the stdlib). No network calls, no API keys.
|
|
54
|
+
|
|
55
|
+
## 10-line example
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from llm_eval import Suite
|
|
59
|
+
|
|
60
|
+
suite = Suite.from_json("examples/basic.json") # load cases
|
|
61
|
+
result = suite.run(lambda p: {"output": "Hello, world!", # your ModelFn
|
|
62
|
+
"usage": {"tokens": 12, "cost_usd": 0.0}})
|
|
63
|
+
print(result.summary()) # 3/4 passed, 1 failed: [missing_keyword]
|
|
64
|
+
result.write_markdown("report.md")
|
|
65
|
+
result.write_json("report.json")
|
|
66
|
+
assert result.ok # False -> CI step fails
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Bring your own model with one function: `ModelFn(prompt) -> {"output": str, "usage": {"tokens": int, "cost_usd": float}}`. For CI and demos, `StubModel` replays canned prompt-to-response fixtures from JSON — fully offline and reproducible.
|
|
70
|
+
|
|
71
|
+
## CLI demo
|
|
72
|
+
|
|
73
|
+
Three pedagogical suites ship in `examples/`. Green one first:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
python -m llm_eval run examples/basic.json --report report.md --json report.json
|
|
77
|
+
echo $? # 1 — 3 pass, 1 clean failure on purpose (see below)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Then the red ones — each fails the command with exit code 1, which is the point:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
python -m llm_eval run examples/injection_pii.json --report report.md --json report.json
|
|
84
|
+
echo $? # 1 — injection obeyed + PII leak detected
|
|
85
|
+
python -m llm_eval run examples/latency_cost.json --report report.md --json report.json
|
|
86
|
+
echo $? # 1 — token / latency / cost budgets exceeded
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Same suites as pytest, one test per case with native JUnit output for CI:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pytest --llm-eval=examples/basic.json
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Sample `report.md` produced by the CLI:
|
|
96
|
+
|
|
97
|
+
```markdown
|
|
98
|
+
# llm-eval report — 3/5 passed (60%)
|
|
99
|
+
|
|
100
|
+
| case | status | latency | cost |
|
|
101
|
+
|------|--------|---------|------|
|
|
102
|
+
| within_budgets | PASS | 5 ms | $0.0001 |
|
|
103
|
+
| too_many_tokens | FAIL (budget_tokens) | 5 ms | $0.0001 |
|
|
104
|
+
| too_slow | FAIL (budget_latency) | 300 ms | $0.0001 |
|
|
105
|
+
| too_expensive | FAIL (budget_cost) | 5 ms | $0.0500 |
|
|
106
|
+
| usage_not_declared | FAIL (budget_tokens) | 5 ms | n/a |
|
|
107
|
+
|
|
108
|
+
## Detections
|
|
109
|
+
- `pii_leaked_email`: PII pattern matched (email) — value redacted: ***
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`report.json` mirrors it in machine-readable form: `{version, cases: [{id, status, failed_checks[], latency_ms, usage}]}`. Secret values are always masked as `***` in both reports.
|
|
113
|
+
|
|
114
|
+
## Checks
|
|
115
|
+
|
|
116
|
+
| check | what it does |
|
|
117
|
+
|-------|--------------|
|
|
118
|
+
| `contains` / `not_contains` | substring present / absent in output |
|
|
119
|
+
| `regex` | `re.search` match on output |
|
|
120
|
+
| `json_valid` | output parses as JSON |
|
|
121
|
+
| `json_schema_lite` | in-house type/required subset (no `jsonschema` dependency) |
|
|
122
|
+
| `refusal` | output matches a refusal template (EN + FR) |
|
|
123
|
+
| `budget_tokens` / `budget_latency` / `budget_cost` | `usage` vs `max_tokens` / wall-clock vs `max_latency_ms` / `cost_usd` vs `max_cost_usd`; missing `usage` fails closed |
|
|
124
|
+
| `no_pii_leak` | FR + EN regexes (email, phone, IBAN, simplified ssn); hits are masked in reports |
|
|
125
|
+
| `no_injection_obey` | prompt matches known injection patterns and output is not a refusal |
|
|
126
|
+
|
|
127
|
+
Case-file format is JSON (array of cases) or JSONL (one case per line) — never YAML, so there is nothing to install. Each case carries a `note` field stating its expected verdict; see `SPEC_CASES.md` for the exact schema.
|
|
128
|
+
|
|
129
|
+
## Limitations & Roadmap
|
|
130
|
+
|
|
131
|
+
Honest limits of v0.1.0:
|
|
132
|
+
|
|
133
|
+
- No LLM-as-judge, on purpose. A model grading a model costs money per run, is non-deterministic (flaky CI), and needs network + secrets — the opposite of a gate that must be fast, free, and reproducible. Deterministic string/schema/budget checks catch regressions; qualitative review stays human.
|
|
134
|
+
- `no_injection_obey` and `no_pii_leak` are pattern heuristics (regex + refusal templates), not guarantees. A novel phrasing can bypass them; the security fixtures document known bypass classes. They are tripwires, not proofs.
|
|
135
|
+
- No semantic similarity or embeddings: paraphrased-but-correct answers fail `contains`/`regex`. Write checks on invariants (keys, formats, refusals), not on wording.
|
|
136
|
+
- Stub latency (`delay_ms`) only simulates slowness; with real models, `budget_latency` is wall-clock and can flake on loaded runners — set generous thresholds in CI.
|
|
137
|
+
- No streaming, no server storage, no web dashboard, no provider adapters (OpenAI & co. are a v0.2.0 opt-in, outside the stdlib-only core).
|
|
138
|
+
|
|
139
|
+
## Development
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
pip install -e . # dev install
|
|
143
|
+
pytest # >=30 tests, 100% on checks/
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
CI (GitHub Actions, <60 s, Python 3.10–3.14, no secrets): lint + pytest + CLI demo. `LICENSE` is MIT. Releases are tagged `vX.Y.Z`; PyPI publication is manual and requires founder approval.
|
|
147
|
+
|
|
148
|
+
## License
|
|
149
|
+
|
|
150
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# llm-eval-ci
|
|
2
|
+
|
|
3
|
+
<!-- Replace YOUR_ORG with the real GitHub org/user when the repo is published. -->
|
|
4
|
+
|
|
5
|
+
[](https://github.com/shenn667/llm-eval-ci/actions/workflows/ci.yml)
|
|
6
|
+
[](https://pypi.org/project/llm-eval-ci/)
|
|
7
|
+
[](https://www.python.org/downloads/)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
10
|
+
`pytest` for LLMs and agents, in CI. Deterministic, offline-first evals that fail your build when a model output regresses — stdlib only, zero runtime dependencies, no API key.
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
llm-eval run ./examples/basic.json --report ./report.md --json ./report.json
|
|
14
|
+
# exit code != 0 as soon as one case breaks a threshold
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Install (30 seconds)
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install llm-eval-ci
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
From source:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
git clone https://github.com/shenn667/llm-eval-ci.git
|
|
27
|
+
cd llm-eval-ci
|
|
28
|
+
pip install -e .
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Requirements: Python 3.10+, nothing else at runtime (`argparse`, `json`, `re`, `time` from the stdlib). No network calls, no API keys.
|
|
32
|
+
|
|
33
|
+
## 10-line example
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from llm_eval import Suite
|
|
37
|
+
|
|
38
|
+
suite = Suite.from_json("examples/basic.json") # load cases
|
|
39
|
+
result = suite.run(lambda p: {"output": "Hello, world!", # your ModelFn
|
|
40
|
+
"usage": {"tokens": 12, "cost_usd": 0.0}})
|
|
41
|
+
print(result.summary()) # 3/4 passed, 1 failed: [missing_keyword]
|
|
42
|
+
result.write_markdown("report.md")
|
|
43
|
+
result.write_json("report.json")
|
|
44
|
+
assert result.ok # False -> CI step fails
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Bring your own model with one function: `ModelFn(prompt) -> {"output": str, "usage": {"tokens": int, "cost_usd": float}}`. For CI and demos, `StubModel` replays canned prompt-to-response fixtures from JSON — fully offline and reproducible.
|
|
48
|
+
|
|
49
|
+
## CLI demo
|
|
50
|
+
|
|
51
|
+
Three pedagogical suites ship in `examples/`. Green one first:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
python -m llm_eval run examples/basic.json --report report.md --json report.json
|
|
55
|
+
echo $? # 1 — 3 pass, 1 clean failure on purpose (see below)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Then the red ones — each fails the command with exit code 1, which is the point:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
python -m llm_eval run examples/injection_pii.json --report report.md --json report.json
|
|
62
|
+
echo $? # 1 — injection obeyed + PII leak detected
|
|
63
|
+
python -m llm_eval run examples/latency_cost.json --report report.md --json report.json
|
|
64
|
+
echo $? # 1 — token / latency / cost budgets exceeded
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Same suites as pytest, one test per case with native JUnit output for CI:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pytest --llm-eval=examples/basic.json
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Sample `report.md` produced by the CLI:
|
|
74
|
+
|
|
75
|
+
```markdown
|
|
76
|
+
# llm-eval report — 3/5 passed (60%)
|
|
77
|
+
|
|
78
|
+
| case | status | latency | cost |
|
|
79
|
+
|------|--------|---------|------|
|
|
80
|
+
| within_budgets | PASS | 5 ms | $0.0001 |
|
|
81
|
+
| too_many_tokens | FAIL (budget_tokens) | 5 ms | $0.0001 |
|
|
82
|
+
| too_slow | FAIL (budget_latency) | 300 ms | $0.0001 |
|
|
83
|
+
| too_expensive | FAIL (budget_cost) | 5 ms | $0.0500 |
|
|
84
|
+
| usage_not_declared | FAIL (budget_tokens) | 5 ms | n/a |
|
|
85
|
+
|
|
86
|
+
## Detections
|
|
87
|
+
- `pii_leaked_email`: PII pattern matched (email) — value redacted: ***
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`report.json` mirrors it in machine-readable form: `{version, cases: [{id, status, failed_checks[], latency_ms, usage}]}`. Secret values are always masked as `***` in both reports.
|
|
91
|
+
|
|
92
|
+
## Checks
|
|
93
|
+
|
|
94
|
+
| check | what it does |
|
|
95
|
+
|-------|--------------|
|
|
96
|
+
| `contains` / `not_contains` | substring present / absent in output |
|
|
97
|
+
| `regex` | `re.search` match on output |
|
|
98
|
+
| `json_valid` | output parses as JSON |
|
|
99
|
+
| `json_schema_lite` | in-house type/required subset (no `jsonschema` dependency) |
|
|
100
|
+
| `refusal` | output matches a refusal template (EN + FR) |
|
|
101
|
+
| `budget_tokens` / `budget_latency` / `budget_cost` | `usage` vs `max_tokens` / wall-clock vs `max_latency_ms` / `cost_usd` vs `max_cost_usd`; missing `usage` fails closed |
|
|
102
|
+
| `no_pii_leak` | FR + EN regexes (email, phone, IBAN, simplified ssn); hits are masked in reports |
|
|
103
|
+
| `no_injection_obey` | prompt matches known injection patterns and output is not a refusal |
|
|
104
|
+
|
|
105
|
+
Case-file format is JSON (array of cases) or JSONL (one case per line) — never YAML, so there is nothing to install. Each case carries a `note` field stating its expected verdict; see `SPEC_CASES.md` for the exact schema.
|
|
106
|
+
|
|
107
|
+
## Limitations & Roadmap
|
|
108
|
+
|
|
109
|
+
Honest limits of v0.1.0:
|
|
110
|
+
|
|
111
|
+
- No LLM-as-judge, on purpose. A model grading a model costs money per run, is non-deterministic (flaky CI), and needs network + secrets — the opposite of a gate that must be fast, free, and reproducible. Deterministic string/schema/budget checks catch regressions; qualitative review stays human.
|
|
112
|
+
- `no_injection_obey` and `no_pii_leak` are pattern heuristics (regex + refusal templates), not guarantees. A novel phrasing can bypass them; the security fixtures document known bypass classes. They are tripwires, not proofs.
|
|
113
|
+
- No semantic similarity or embeddings: paraphrased-but-correct answers fail `contains`/`regex`. Write checks on invariants (keys, formats, refusals), not on wording.
|
|
114
|
+
- Stub latency (`delay_ms`) only simulates slowness; with real models, `budget_latency` is wall-clock and can flake on loaded runners — set generous thresholds in CI.
|
|
115
|
+
- No streaming, no server storage, no web dashboard, no provider adapters (OpenAI & co. are a v0.2.0 opt-in, outside the stdlib-only core).
|
|
116
|
+
|
|
117
|
+
## Development
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
pip install -e . # dev install
|
|
121
|
+
pytest # >=30 tests, 100% on checks/
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
CI (GitHub Actions, <60 s, Python 3.10–3.14, no secrets): lint + pytest + CLI demo. `LICENSE` is MIT. Releases are tagged `vX.Y.Z`; PyPI publication is manual and requires founder approval.
|
|
125
|
+
|
|
126
|
+
## License
|
|
127
|
+
|
|
128
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "llm-eval-ci"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "pytest for LLMs and agents in CI — deterministic, offline-first evals (stdlib only)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "llm-eval-ci" }]
|
|
13
|
+
keywords = ["llm", "eval", "testing", "ci", "agents"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3.10",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Programming Language :: Python :: 3.14",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Operating System :: OS Independent",
|
|
23
|
+
"Topic :: Software Development :: Testing",
|
|
24
|
+
]
|
|
25
|
+
dependencies = []
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
llm-eval = "llm_eval.cli:main"
|
|
29
|
+
|
|
30
|
+
[project.entry-points.pytest11]
|
|
31
|
+
llm-eval-ci = "llm_eval.pytest_plugin"
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/shenn667/llm-eval-ci"
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.packages.find]
|
|
37
|
+
where = ["src"]
|
|
38
|
+
|
|
39
|
+
[tool.pytest.ini_options]
|
|
40
|
+
testpaths = ["tests"]
|
|
41
|
+
|
|
42
|
+
[tool.coverage.run]
|
|
43
|
+
source = ["llm_eval"]
|
|
44
|
+
branch = false
|
|
45
|
+
|
|
46
|
+
[tool.coverage.report]
|
|
47
|
+
show_missing = true
|
|
48
|
+
exclude_also = ["raise AssertionError", "pragma: no cover"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""llm-eval-ci — pytest for LLMs/agents in CI (stdlib-only, offline-first)."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
from .models import ModelFn, StubMissingError, StubModel
|
|
6
|
+
from .suite import CaseResult, RunResult, Suite, SuiteLoadError, load_suite_file
|
|
7
|
+
from .checks import BUDGET_CHECK_FOR_KIND, CHECKS, CHECK_NAMES, run_check
|
|
8
|
+
from .checks.regex import ContainsCheck, NotContainsCheck, RegexCheck
|
|
9
|
+
from .checks.jsonschema_lite import JsonSchemaCheck, JsonValidCheck
|
|
10
|
+
from .checks.latency import BudgetLatencyCheck
|
|
11
|
+
from .checks.cost import BudgetCostCheck, BudgetTokensCheck
|
|
12
|
+
from .checks.injection import NoInjectionObeyCheck, RefusalCheck
|
|
13
|
+
from .checks.pii import NoPIILeakCheck
|
|
14
|
+
|
|
15
|
+
# Backwards-compatible alias: the JSON report contract calls these SuiteResult.
|
|
16
|
+
SuiteResult = RunResult
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ModelFn",
|
|
20
|
+
"StubMissingError",
|
|
21
|
+
"StubModel",
|
|
22
|
+
"Suite",
|
|
23
|
+
"SuiteLoadError",
|
|
24
|
+
"SuiteResult",
|
|
25
|
+
"CaseResult",
|
|
26
|
+
"RunResult",
|
|
27
|
+
"load_suite_file",
|
|
28
|
+
"CHECKS",
|
|
29
|
+
"CHECK_NAMES",
|
|
30
|
+
"BUDGET_CHECK_FOR_KIND",
|
|
31
|
+
"run_check",
|
|
32
|
+
"ContainsCheck",
|
|
33
|
+
"NotContainsCheck",
|
|
34
|
+
"RegexCheck",
|
|
35
|
+
"JsonValidCheck",
|
|
36
|
+
"JsonSchemaCheck",
|
|
37
|
+
"RefusalCheck",
|
|
38
|
+
"BudgetTokensCheck",
|
|
39
|
+
"BudgetLatencyCheck",
|
|
40
|
+
"BudgetCostCheck",
|
|
41
|
+
"NoPIILeakCheck",
|
|
42
|
+
"NoInjectionObeyCheck",
|
|
43
|
+
"__version__",
|
|
44
|
+
]
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Base model interface: ModelFn(prompt) -> dict (stdlib only)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Callable, Dict, Optional, Protocol, Tuple, runtime_checkable
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ModelResult(Dict[str, Any]):
|
|
9
|
+
"""Convention: {"output": str, "usage": {"tokens": int, "cost_usd": float}}."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
ModelFn = Callable[[str], Dict[str, Any]]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@runtime_checkable
|
|
16
|
+
class _ModelFnProtocol(Protocol):
|
|
17
|
+
def __call__(self, prompt: str) -> Dict[str, Any]: ...
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class BaseModel:
|
|
21
|
+
"""Minimal base class for model adapters (optional convenience)."""
|
|
22
|
+
|
|
23
|
+
name = "base"
|
|
24
|
+
|
|
25
|
+
def __call__(self, prompt: str) -> Dict[str, Any]: # pragma: no cover - override
|
|
26
|
+
raise NotImplementedError("subclass must implement __call__(prompt)")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _as_int(value: Any) -> Optional[int]:
|
|
30
|
+
if isinstance(value, bool):
|
|
31
|
+
return None
|
|
32
|
+
if isinstance(value, int) and value >= 0:
|
|
33
|
+
return value
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _as_cost(value: Any) -> Optional[float]:
|
|
38
|
+
if isinstance(value, bool):
|
|
39
|
+
return None
|
|
40
|
+
if isinstance(value, (int, float)) and value >= 0:
|
|
41
|
+
return float(value)
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def normalize_result(raw: Any) -> Tuple[Optional[str], Optional[Dict[str, Any]]]:
|
|
46
|
+
"""Normalize a raw ModelFn return into (output, usage).
|
|
47
|
+
|
|
48
|
+
Never raises: malformed payloads degrade to (None, None) and the
|
|
49
|
+
runner marks every check FAIL (fail-closed, SPEC section 3/5).
|
|
50
|
+
"""
|
|
51
|
+
if not isinstance(raw, dict):
|
|
52
|
+
return None, None
|
|
53
|
+
output = raw.get("output")
|
|
54
|
+
if not isinstance(output, str):
|
|
55
|
+
output = None
|
|
56
|
+
usage_raw = raw.get("usage")
|
|
57
|
+
usage: Optional[Dict[str, Any]] = None
|
|
58
|
+
if isinstance(usage_raw, dict):
|
|
59
|
+
tokens = _as_int(usage_raw.get("tokens"))
|
|
60
|
+
cost_raw = usage_raw.get("cost_usd", usage_raw.get("cost"))
|
|
61
|
+
cost = _as_cost(cost_raw)
|
|
62
|
+
if tokens is not None or cost is not None:
|
|
63
|
+
usage = {}
|
|
64
|
+
if tokens is not None:
|
|
65
|
+
usage["tokens"] = tokens
|
|
66
|
+
if cost is not None:
|
|
67
|
+
usage["cost_usd"] = cost
|
|
68
|
+
return output, usage
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Optional OpenAI-compatible adapter (stdlib only, opt-in, v0.2.0 preview).
|
|
2
|
+
|
|
3
|
+
Not imported by the core runtime: no network happens unless you
|
|
4
|
+
instantiate and call this class explicitly. Uses ``urllib`` only —
|
|
5
|
+
no third-party dependency. Disabled by default because v0.1.0 is
|
|
6
|
+
offline-first (see README "Limitations & Roadmap").
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import urllib.request
|
|
14
|
+
from typing import Any, Dict, Optional
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class OpenAICompatModel:
|
|
18
|
+
"""Minimal ``/chat/completions`` client for OpenAI-compatible APIs."""
|
|
19
|
+
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
model: str = "gpt-4o-mini",
|
|
23
|
+
base_url: str = "https://api.openai.com/v1",
|
|
24
|
+
api_key: Optional[str] = None,
|
|
25
|
+
timeout_s: float = 30.0,
|
|
26
|
+
) -> None:
|
|
27
|
+
self.model = model
|
|
28
|
+
self.base_url = base_url.rstrip("/")
|
|
29
|
+
self.api_key = api_key if api_key is not None else os.environ.get("OPENAI_API_KEY")
|
|
30
|
+
self.timeout_s = timeout_s
|
|
31
|
+
|
|
32
|
+
def __call__(self, prompt: str) -> Dict[str, Any]:
|
|
33
|
+
if not self.api_key:
|
|
34
|
+
raise RuntimeError(
|
|
35
|
+
"OpenAICompatModel needs an API key "
|
|
36
|
+
"(pass api_key= or set OPENAI_API_KEY). "
|
|
37
|
+
"The v0.1.0 core stays offline; this adapter is opt-in."
|
|
38
|
+
)
|
|
39
|
+
payload = {
|
|
40
|
+
"model": self.model,
|
|
41
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
42
|
+
}
|
|
43
|
+
req = urllib.request.Request(
|
|
44
|
+
self.base_url + "/chat/completions",
|
|
45
|
+
data=json.dumps(payload).encode("utf-8"),
|
|
46
|
+
headers={
|
|
47
|
+
"Content-Type": "application/json",
|
|
48
|
+
"Authorization": "Bearer " + self.api_key,
|
|
49
|
+
},
|
|
50
|
+
method="POST",
|
|
51
|
+
)
|
|
52
|
+
with urllib.request.urlopen(req, timeout=self.timeout_s) as resp:
|
|
53
|
+
body = json.loads(resp.read().decode("utf-8"))
|
|
54
|
+
try:
|
|
55
|
+
content = body["choices"][0]["message"]["content"]
|
|
56
|
+
except (KeyError, IndexError, TypeError) as exc:
|
|
57
|
+
raise RuntimeError(f"unexpected chat-completions payload: {exc}") from exc
|
|
58
|
+
tokens: Optional[int] = None
|
|
59
|
+
try:
|
|
60
|
+
u = body.get("usage", {})
|
|
61
|
+
tokens = int(u.get("prompt_tokens", 0)) + int(u.get("completion_tokens", 0))
|
|
62
|
+
except (TypeError, ValueError):
|
|
63
|
+
tokens = None
|
|
64
|
+
usage: Dict[str, Any] = {}
|
|
65
|
+
if tokens is not None:
|
|
66
|
+
usage["tokens"] = tokens
|
|
67
|
+
# Cost is unknown without a pricing table -> omit (budget_cost fails
|
|
68
|
+
# closed when checked, per SPEC section 5).
|
|
69
|
+
return {"output": content if isinstance(content, str) else str(content), "usage": usage or None}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""StubModel — offline canned prompt->response fixtures (stdlib only)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, Mapping, Optional, Sequence, Union
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class UnknownPromptError(KeyError):
|
|
12
|
+
"""Raised when a prompt has no stubbed response (strict mode)."""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class StubModel:
|
|
16
|
+
"""Replay canned responses — fully offline, reproducible, CI-friendly.
|
|
17
|
+
|
|
18
|
+
Mapping values accept either a plain string (used as ``output`` with
|
|
19
|
+
no usage) or a dict ``{"output": str, "usage": {...}, "delay_ms": int}``.
|
|
20
|
+
``delay_ms`` simulates latency via ``time.sleep`` (StubModel-only knob;
|
|
21
|
+
real-model latency is measured wall-clock by the runner).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
mapping: Optional[Mapping[str, Any]] = None,
|
|
27
|
+
*,
|
|
28
|
+
default_output: str = "",
|
|
29
|
+
strict: bool = False,
|
|
30
|
+
) -> None:
|
|
31
|
+
self._mapping: Dict[str, Any] = dict(mapping or {})
|
|
32
|
+
self.default_output = default_output
|
|
33
|
+
self.strict = strict
|
|
34
|
+
|
|
35
|
+
# -- constructors -----------------------------------------------------
|
|
36
|
+
@classmethod
|
|
37
|
+
def from_dict(cls, mapping: Mapping[str, Any], **kwargs: Any) -> "StubModel":
|
|
38
|
+
return cls(mapping, **kwargs)
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def from_cases(cls, cases: Sequence[Mapping[str, Any]], **kwargs: Any) -> "StubModel":
|
|
42
|
+
"""Build from loaded suite cases using each case's prompt.
|
|
43
|
+
|
|
44
|
+
Cases may carry an embedded ``stub`` block (examples/*.json style)
|
|
45
|
+
or a top-level ``output``/``stub_output`` fixture.
|
|
46
|
+
"""
|
|
47
|
+
mapping: Dict[str, Any] = {}
|
|
48
|
+
for case in cases:
|
|
49
|
+
prompt = case.get("prompt")
|
|
50
|
+
if not isinstance(prompt, str):
|
|
51
|
+
continue
|
|
52
|
+
if isinstance(case.get("stub"), dict):
|
|
53
|
+
mapping[prompt] = case["stub"]
|
|
54
|
+
elif "output" in case and isinstance(case.get("output"), str):
|
|
55
|
+
mapping[prompt] = {
|
|
56
|
+
"output": case["output"],
|
|
57
|
+
"usage": case.get("usage"),
|
|
58
|
+
}
|
|
59
|
+
return cls(mapping, **kwargs)
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def from_json(cls, path: Union[str, Path], **kwargs: Any) -> "StubModel":
|
|
63
|
+
raw = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
64
|
+
if isinstance(raw, dict) and "mapping" in raw and isinstance(raw["mapping"], dict):
|
|
65
|
+
raw = raw["mapping"]
|
|
66
|
+
if not isinstance(raw, dict):
|
|
67
|
+
raise ValueError(f"stub fixture must be a JSON object: {path}")
|
|
68
|
+
return cls(raw, **kwargs)
|
|
69
|
+
|
|
70
|
+
# -- call -------------------------------------------------------------
|
|
71
|
+
def __call__(self, prompt: str) -> Dict[str, Any]:
|
|
72
|
+
if prompt in self._mapping:
|
|
73
|
+
entry = self._mapping[prompt]
|
|
74
|
+
elif self.strict:
|
|
75
|
+
raise UnknownPromptError(f"no stubbed response for prompt: {prompt!r}")
|
|
76
|
+
else:
|
|
77
|
+
return {"output": self.default_output, "usage": None}
|
|
78
|
+
if isinstance(entry, str):
|
|
79
|
+
return {"output": entry, "usage": None}
|
|
80
|
+
if not isinstance(entry, dict):
|
|
81
|
+
return {"output": self.default_output, "usage": None}
|
|
82
|
+
delay = entry.get("delay_ms", 0)
|
|
83
|
+
try:
|
|
84
|
+
delay_ms = int(delay or 0)
|
|
85
|
+
except (TypeError, ValueError):
|
|
86
|
+
delay_ms = 0
|
|
87
|
+
if delay_ms > 0:
|
|
88
|
+
time.sleep(min(delay_ms, 5000) / 1000.0)
|
|
89
|
+
return {"output": entry.get("output", ""), "usage": entry.get("usage")}
|