llm-eval-ci 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. llm_eval_ci-0.1.0/LICENSE +21 -0
  2. llm_eval_ci-0.1.0/PKG-INFO +150 -0
  3. llm_eval_ci-0.1.0/README.md +128 -0
  4. llm_eval_ci-0.1.0/pyproject.toml +48 -0
  5. llm_eval_ci-0.1.0/setup.cfg +4 -0
  6. llm_eval_ci-0.1.0/src/llm_eval/__init__.py +44 -0
  7. llm_eval_ci-0.1.0/src/llm_eval/__main__.py +6 -0
  8. llm_eval_ci-0.1.0/src/llm_eval/adapters/__init__.py +6 -0
  9. llm_eval_ci-0.1.0/src/llm_eval/adapters/base.py +68 -0
  10. llm_eval_ci-0.1.0/src/llm_eval/adapters/openai_compat.py +69 -0
  11. llm_eval_ci-0.1.0/src/llm_eval/adapters/stub.py +89 -0
  12. llm_eval_ci-0.1.0/src/llm_eval/checks/__init__.py +46 -0
  13. llm_eval_ci-0.1.0/src/llm_eval/checks/base.py +44 -0
  14. llm_eval_ci-0.1.0/src/llm_eval/checks/budgets.py +78 -0
  15. llm_eval_ci-0.1.0/src/llm_eval/checks/content.py +50 -0
  16. llm_eval_ci-0.1.0/src/llm_eval/checks/cost.py +38 -0
  17. llm_eval_ci-0.1.0/src/llm_eval/checks/injection.py +48 -0
  18. llm_eval_ci-0.1.0/src/llm_eval/checks/json_checks.py +86 -0
  19. llm_eval_ci-0.1.0/src/llm_eval/checks/jsonschema_lite.py +37 -0
  20. llm_eval_ci-0.1.0/src/llm_eval/checks/latency.py +20 -0
  21. llm_eval_ci-0.1.0/src/llm_eval/checks/pii.py +27 -0
  22. llm_eval_ci-0.1.0/src/llm_eval/checks/refusal.py +32 -0
  23. llm_eval_ci-0.1.0/src/llm_eval/checks/regex.py +49 -0
  24. llm_eval_ci-0.1.0/src/llm_eval/checks/security.py +103 -0
  25. llm_eval_ci-0.1.0/src/llm_eval/cli.py +186 -0
  26. llm_eval_ci-0.1.0/src/llm_eval/models.py +71 -0
  27. llm_eval_ci-0.1.0/src/llm_eval/pytest_plugin.py +123 -0
  28. llm_eval_ci-0.1.0/src/llm_eval/reporters/__init__.py +11 -0
  29. llm_eval_ci-0.1.0/src/llm_eval/reporters/json.py +64 -0
  30. llm_eval_ci-0.1.0/src/llm_eval/reporters/json_report.py +76 -0
  31. llm_eval_ci-0.1.0/src/llm_eval/reporters/markdown.py +116 -0
  32. llm_eval_ci-0.1.0/src/llm_eval/suite.py +383 -0
  33. llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/PKG-INFO +150 -0
  34. llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/SOURCES.txt +48 -0
  35. llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/dependency_links.txt +1 -0
  36. llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/entry_points.txt +5 -0
  37. llm_eval_ci-0.1.0/src/llm_eval_ci.egg-info/top_level.txt +1 -0
  38. llm_eval_ci-0.1.0/tests/test_checks_budgets.py +106 -0
  39. llm_eval_ci-0.1.0/tests/test_checks_content.py +74 -0
  40. llm_eval_ci-0.1.0/tests/test_checks_json.py +120 -0
  41. llm_eval_ci-0.1.0/tests/test_checks_refusal.py +28 -0
  42. llm_eval_ci-0.1.0/tests/test_checks_registry.py +60 -0
  43. llm_eval_ci-0.1.0/tests/test_checks_security.py +71 -0
  44. llm_eval_ci-0.1.0/tests/test_cli.py +113 -0
  45. llm_eval_ci-0.1.0/tests/test_models.py +53 -0
  46. llm_eval_ci-0.1.0/tests/test_plugin.py +73 -0
  47. llm_eval_ci-0.1.0/tests/test_reporters.py +81 -0
  48. llm_eval_ci-0.1.0/tests/test_suite.py +207 -0
  49. llm_eval_ci-0.1.0/tests/test_t523_checks.py +226 -0
  50. llm_eval_ci-0.1.0/tests/test_t523_suite_cli.py +242 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 llm-eval-ci contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,150 @@
1
+ Metadata-Version: 2.4
2
+ Name: llm-eval-ci
3
+ Version: 0.1.0
4
+ Summary: pytest for LLMs and agents in CI — deterministic, offline-first evals (stdlib only)
5
+ Author: llm-eval-ci
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/shenn667/llm-eval-ci
8
+ Keywords: llm,eval,testing,ci,agents
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Programming Language :: Python :: 3.14
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Topic :: Software Development :: Testing
18
+ Requires-Python: >=3.10
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Dynamic: license-file
22
+
23
+ # llm-eval-ci
24
+
25
+ <!-- Replace YOUR_ORG with the real GitHub org/user when the repo is published. -->
26
+
27
+ [![CI](https://github.com/shenn667/llm-eval-ci/actions/workflows/ci.yml/badge.svg)](https://github.com/shenn667/llm-eval-ci/actions/workflows/ci.yml)
28
+ [![PyPI](https://img.shields.io/pypi/v/llm-eval-ci.svg)](https://pypi.org/project/llm-eval-ci/)
29
+ [![Python](https://img.shields.io/badge/python-3.10%20%E2%80%93%203.14-blue.svg)](https://www.python.org/downloads/)
30
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
31
+
32
+ `pytest` for LLMs and agents, in CI. Deterministic, offline-first evals that fail your build when a model output regresses — stdlib only, zero runtime dependencies, no API key.
33
+
34
+ ```bash
35
+ llm-eval run ./examples/basic.json --report ./report.md --json ./report.json
36
+ # exit code != 0 as soon as one case breaks a threshold
37
+ ```
38
+
39
+ ## Install (30 seconds)
40
+
41
+ ```bash
42
+ pip install llm-eval-ci
43
+ ```
44
+
45
+ From source:
46
+
47
+ ```bash
48
+ git clone https://github.com/shenn667/llm-eval-ci.git
49
+ cd llm-eval-ci
50
+ pip install -e .
51
+ ```
52
+
53
+ Requirements: Python 3.10+, nothing else at runtime (`argparse`, `json`, `re`, `time` from the stdlib). No network calls, no API keys.
54
+
55
+ ## 10-line example
56
+
57
+ ```python
58
+ from llm_eval import Suite
59
+
60
+ suite = Suite.from_json("examples/basic.json") # load cases
61
+ result = suite.run(lambda p: {"output": "Hello, world!", # your ModelFn
62
+ "usage": {"tokens": 12, "cost_usd": 0.0}})
63
+ print(result.summary()) # 3/4 passed, 1 failed: [missing_keyword]
64
+ result.write_markdown("report.md")
65
+ result.write_json("report.json")
66
+ assert result.ok # False -> CI step fails
67
+ ```
68
+
69
+ Bring your own model with one function: `ModelFn(prompt) -> {"output": str, "usage": {"tokens": int, "cost_usd": float}}`. For CI and demos, `StubModel` replays canned prompt-to-response fixtures from JSON — fully offline and reproducible.
70
+
71
+ ## CLI demo
72
+
73
+ Three pedagogical suites ship in `examples/`. Green one first:
74
+
75
+ ```bash
76
+ python -m llm_eval run examples/basic.json --report report.md --json report.json
77
+ echo $? # 1 — 3 pass, 1 clean failure on purpose (see below)
78
+ ```
79
+
80
+ Then the red ones — each fails the command with exit code 1, which is the point:
81
+
82
+ ```bash
83
+ python -m llm_eval run examples/injection_pii.json --report report.md --json report.json
84
+ echo $? # 1 — injection obeyed + PII leak detected
85
+ python -m llm_eval run examples/latency_cost.json --report report.md --json report.json
86
+ echo $? # 1 — token / latency / cost budgets exceeded
87
+ ```
88
+
89
+ Same suites as pytest, one test per case with native JUnit output for CI:
90
+
91
+ ```bash
92
+ pytest --llm-eval=examples/basic.json
93
+ ```
94
+
95
+ Sample `report.md` produced by the CLI:
96
+
97
+ ```markdown
98
+ # llm-eval report — 3/5 passed (60%)
99
+
100
+ | case | status | latency | cost |
101
+ |------|--------|---------|------|
102
+ | within_budgets | PASS | 5 ms | $0.0001 |
103
+ | too_many_tokens | FAIL (budget_tokens) | 5 ms | $0.0001 |
104
+ | too_slow | FAIL (budget_latency) | 300 ms | $0.0001 |
105
+ | too_expensive | FAIL (budget_cost) | 5 ms | $0.0500 |
106
+ | usage_not_declared | FAIL (budget_tokens) | 5 ms | n/a |
107
+
108
+ ## Detections
109
+ - `pii_leaked_email`: PII pattern matched (email) — value redacted: ***
110
+ ```
111
+
112
+ `report.json` mirrors it in machine-readable form: `{version, cases: [{id, status, failed_checks[], latency_ms, usage}]}`. Secret values are always masked as `***` in both reports.
113
+
114
+ ## Checks
115
+
116
+ | check | what it does |
117
+ |-------|--------------|
118
+ | `contains` / `not_contains` | substring present / absent in output |
119
+ | `regex` | `re.search` match on output |
120
+ | `json_valid` | output parses as JSON |
121
+ | `json_schema_lite` | in-house type/required subset (no `jsonschema` dependency) |
122
+ | `refusal` | output matches a refusal template (EN + FR) |
123
+ | `budget_tokens` / `budget_latency` / `budget_cost` | `usage` vs `max_tokens` / wall-clock vs `max_latency_ms` / `cost_usd` vs `max_cost_usd`; missing `usage` fails closed |
124
+ | `no_pii_leak` | FR + EN regexes (email, phone, IBAN, simplified ssn); hits are masked in reports |
125
+ | `no_injection_obey` | prompt matches known injection patterns and output is not a refusal |
126
+
127
+ Case-file format is JSON (array of cases) or JSONL (one case per line) — never YAML, so there is nothing to install. Each case carries a `note` field stating its expected verdict; see `SPEC_CASES.md` for the exact schema.
128
+
129
+ ## Limitations & Roadmap
130
+
131
+ Honest limits of v0.1.0:
132
+
133
+ - No LLM-as-judge, on purpose. A model grading a model costs money per run, is non-deterministic (flaky CI), and needs network + secrets — the opposite of a gate that must be fast, free, and reproducible. Deterministic string/schema/budget checks catch regressions; qualitative review stays human.
134
+ - `no_injection_obey` and `no_pii_leak` are pattern heuristics (regex + refusal templates), not guarantees. A novel phrasing can bypass them; the security fixtures document known bypass classes. They are tripwires, not proofs.
135
+ - No semantic similarity or embeddings: paraphrased-but-correct answers fail `contains`/`regex`. Write checks on invariants (keys, formats, refusals), not on wording.
136
+ - Stub latency (`delay_ms`) only simulates slowness; with real models, `budget_latency` is wall-clock and can flake on loaded runners — set generous thresholds in CI.
137
+ - No streaming, no server storage, no web dashboard, no provider adapters (OpenAI & co. are a v0.2.0 opt-in, outside the stdlib-only core).
138
+
139
+ ## Development
140
+
141
+ ```bash
142
+ pip install -e . # dev install
143
+ pytest # >=30 tests, 100% on checks/
144
+ ```
145
+
146
+ CI (GitHub Actions, <60 s, Python 3.10–3.14, no secrets): lint + pytest + CLI demo. `LICENSE` is MIT. Releases are tagged `vX.Y.Z`; PyPI publication is manual and requires founder approval.
147
+
148
+ ## License
149
+
150
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,128 @@
1
+ # llm-eval-ci
2
+
3
+ <!-- Replace YOUR_ORG with the real GitHub org/user when the repo is published. -->
4
+
5
+ [![CI](https://github.com/shenn667/llm-eval-ci/actions/workflows/ci.yml/badge.svg)](https://github.com/shenn667/llm-eval-ci/actions/workflows/ci.yml)
6
+ [![PyPI](https://img.shields.io/pypi/v/llm-eval-ci.svg)](https://pypi.org/project/llm-eval-ci/)
7
+ [![Python](https://img.shields.io/badge/python-3.10%20%E2%80%93%203.14-blue.svg)](https://www.python.org/downloads/)
8
+ [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
9
+
10
+ `pytest` for LLMs and agents, in CI. Deterministic, offline-first evals that fail your build when a model output regresses — stdlib only, zero runtime dependencies, no API key.
11
+
12
+ ```bash
13
+ llm-eval run ./examples/basic.json --report ./report.md --json ./report.json
14
+ # exit code != 0 as soon as one case breaks a threshold
15
+ ```
16
+
17
+ ## Install (30 seconds)
18
+
19
+ ```bash
20
+ pip install llm-eval-ci
21
+ ```
22
+
23
+ From source:
24
+
25
+ ```bash
26
+ git clone https://github.com/shenn667/llm-eval-ci.git
27
+ cd llm-eval-ci
28
+ pip install -e .
29
+ ```
30
+
31
+ Requirements: Python 3.10+, nothing else at runtime (`argparse`, `json`, `re`, `time` from the stdlib). No network calls, no API keys.
32
+
33
+ ## 10-line example
34
+
35
+ ```python
36
+ from llm_eval import Suite
37
+
38
+ suite = Suite.from_json("examples/basic.json") # load cases
39
+ result = suite.run(lambda p: {"output": "Hello, world!", # your ModelFn
40
+ "usage": {"tokens": 12, "cost_usd": 0.0}})
41
+ print(result.summary()) # 3/4 passed, 1 failed: [missing_keyword]
42
+ result.write_markdown("report.md")
43
+ result.write_json("report.json")
44
+ assert result.ok # False -> CI step fails
45
+ ```
46
+
47
+ Bring your own model with one function: `ModelFn(prompt) -> {"output": str, "usage": {"tokens": int, "cost_usd": float}}`. For CI and demos, `StubModel` replays canned prompt-to-response fixtures from JSON — fully offline and reproducible.
48
+
49
+ ## CLI demo
50
+
51
+ Three pedagogical suites ship in `examples/`. Green one first:
52
+
53
+ ```bash
54
+ python -m llm_eval run examples/basic.json --report report.md --json report.json
55
+ echo $? # 1 — 3 pass, 1 clean failure on purpose (see below)
56
+ ```
57
+
58
+ Then the red ones — each fails the command with exit code 1, which is the point:
59
+
60
+ ```bash
61
+ python -m llm_eval run examples/injection_pii.json --report report.md --json report.json
62
+ echo $? # 1 — injection obeyed + PII leak detected
63
+ python -m llm_eval run examples/latency_cost.json --report report.md --json report.json
64
+ echo $? # 1 — token / latency / cost budgets exceeded
65
+ ```
66
+
67
+ Same suites as pytest, one test per case with native JUnit output for CI:
68
+
69
+ ```bash
70
+ pytest --llm-eval=examples/basic.json
71
+ ```
72
+
73
+ Sample `report.md` produced by the CLI:
74
+
75
+ ```markdown
76
+ # llm-eval report — 3/5 passed (60%)
77
+
78
+ | case | status | latency | cost |
79
+ |------|--------|---------|------|
80
+ | within_budgets | PASS | 5 ms | $0.0001 |
81
+ | too_many_tokens | FAIL (budget_tokens) | 5 ms | $0.0001 |
82
+ | too_slow | FAIL (budget_latency) | 300 ms | $0.0001 |
83
+ | too_expensive | FAIL (budget_cost) | 5 ms | $0.0500 |
84
+ | usage_not_declared | FAIL (budget_tokens) | 5 ms | n/a |
85
+
86
+ ## Detections
87
+ - `pii_leaked_email`: PII pattern matched (email) — value redacted: ***
88
+ ```
89
+
90
+ `report.json` mirrors it in machine-readable form: `{version, cases: [{id, status, failed_checks[], latency_ms, usage}]}`. Secret values are always masked as `***` in both reports.
91
+
92
+ ## Checks
93
+
94
+ | check | what it does |
95
+ |-------|--------------|
96
+ | `contains` / `not_contains` | substring present / absent in output |
97
+ | `regex` | `re.search` match on output |
98
+ | `json_valid` | output parses as JSON |
99
+ | `json_schema_lite` | in-house type/required subset (no `jsonschema` dependency) |
100
+ | `refusal` | output matches a refusal template (EN + FR) |
101
+ | `budget_tokens` / `budget_latency` / `budget_cost` | `usage` vs `max_tokens` / wall-clock vs `max_latency_ms` / `cost_usd` vs `max_cost_usd`; missing `usage` fails closed |
102
+ | `no_pii_leak` | FR + EN regexes (email, phone, IBAN, simplified ssn); hits are masked in reports |
103
+ | `no_injection_obey` | prompt matches known injection patterns and output is not a refusal |
104
+
105
+ Case-file format is JSON (array of cases) or JSONL (one case per line) — never YAML, so there is nothing to install. Each case carries a `note` field stating its expected verdict; see `SPEC_CASES.md` for the exact schema.
106
+
107
+ ## Limitations & Roadmap
108
+
109
+ Honest limits of v0.1.0:
110
+
111
+ - No LLM-as-judge, on purpose. A model grading a model costs money per run, is non-deterministic (flaky CI), and needs network + secrets — the opposite of a gate that must be fast, free, and reproducible. Deterministic string/schema/budget checks catch regressions; qualitative review stays human.
112
+ - `no_injection_obey` and `no_pii_leak` are pattern heuristics (regex + refusal templates), not guarantees. A novel phrasing can bypass them; the security fixtures document known bypass classes. They are tripwires, not proofs.
113
+ - No semantic similarity or embeddings: paraphrased-but-correct answers fail `contains`/`regex`. Write checks on invariants (keys, formats, refusals), not on wording.
114
+ - Stub latency (`delay_ms`) only simulates slowness; with real models, `budget_latency` is wall-clock and can flake on loaded runners — set generous thresholds in CI.
115
+ - No streaming, no server storage, no web dashboard, no provider adapters (OpenAI & co. are a v0.2.0 opt-in, outside the stdlib-only core).
116
+
117
+ ## Development
118
+
119
+ ```bash
120
+ pip install -e . # dev install
121
+ pytest # >=30 tests, 100% on checks/
122
+ ```
123
+
124
+ CI (GitHub Actions, <60 s, Python 3.10–3.14, no secrets): lint + pytest + CLI demo. `LICENSE` is MIT. Releases are tagged `vX.Y.Z`; PyPI publication is manual and requires founder approval.
125
+
126
+ ## License
127
+
128
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,48 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "llm-eval-ci"
7
+ version = "0.1.0"
8
+ description = "pytest for LLMs and agents in CI — deterministic, offline-first evals (stdlib only)"
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ requires-python = ">=3.10"
12
+ authors = [{ name = "llm-eval-ci" }]
13
+ keywords = ["llm", "eval", "testing", "ci", "agents"]
14
+ classifiers = [
15
+ "Programming Language :: Python :: 3",
16
+ "Programming Language :: Python :: 3.10",
17
+ "Programming Language :: Python :: 3.11",
18
+ "Programming Language :: Python :: 3.12",
19
+ "Programming Language :: Python :: 3.13",
20
+ "Programming Language :: Python :: 3.14",
21
+ "License :: OSI Approved :: MIT License",
22
+ "Operating System :: OS Independent",
23
+ "Topic :: Software Development :: Testing",
24
+ ]
25
+ dependencies = []
26
+
27
+ [project.scripts]
28
+ llm-eval = "llm_eval.cli:main"
29
+
30
+ [project.entry-points.pytest11]
31
+ llm-eval-ci = "llm_eval.pytest_plugin"
32
+
33
+ [project.urls]
34
+ Homepage = "https://github.com/shenn667/llm-eval-ci"
35
+
36
+ [tool.setuptools.packages.find]
37
+ where = ["src"]
38
+
39
+ [tool.pytest.ini_options]
40
+ testpaths = ["tests"]
41
+
42
+ [tool.coverage.run]
43
+ source = ["llm_eval"]
44
+ branch = false
45
+
46
+ [tool.coverage.report]
47
+ show_missing = true
48
+ exclude_also = ["raise AssertionError", "pragma: no cover"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,44 @@
1
+ """llm-eval-ci — pytest for LLMs/agents in CI (stdlib-only, offline-first)."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ from .models import ModelFn, StubMissingError, StubModel
6
+ from .suite import CaseResult, RunResult, Suite, SuiteLoadError, load_suite_file
7
+ from .checks import BUDGET_CHECK_FOR_KIND, CHECKS, CHECK_NAMES, run_check
8
+ from .checks.regex import ContainsCheck, NotContainsCheck, RegexCheck
9
+ from .checks.jsonschema_lite import JsonSchemaCheck, JsonValidCheck
10
+ from .checks.latency import BudgetLatencyCheck
11
+ from .checks.cost import BudgetCostCheck, BudgetTokensCheck
12
+ from .checks.injection import NoInjectionObeyCheck, RefusalCheck
13
+ from .checks.pii import NoPIILeakCheck
14
+
15
+ # Backwards-compatible alias: the JSON report contract calls these SuiteResult.
16
+ SuiteResult = RunResult
17
+
18
+ __all__ = [
19
+ "ModelFn",
20
+ "StubMissingError",
21
+ "StubModel",
22
+ "Suite",
23
+ "SuiteLoadError",
24
+ "SuiteResult",
25
+ "CaseResult",
26
+ "RunResult",
27
+ "load_suite_file",
28
+ "CHECKS",
29
+ "CHECK_NAMES",
30
+ "BUDGET_CHECK_FOR_KIND",
31
+ "run_check",
32
+ "ContainsCheck",
33
+ "NotContainsCheck",
34
+ "RegexCheck",
35
+ "JsonValidCheck",
36
+ "JsonSchemaCheck",
37
+ "RefusalCheck",
38
+ "BudgetTokensCheck",
39
+ "BudgetLatencyCheck",
40
+ "BudgetCostCheck",
41
+ "NoPIILeakCheck",
42
+ "NoInjectionObeyCheck",
43
+ "__version__",
44
+ ]
@@ -0,0 +1,6 @@
1
+ """python -m llm_eval entry point (stdlib only)."""
2
+
3
+ from .cli import main
4
+
5
+ if __name__ == "__main__":
6
+ raise SystemExit(main())
@@ -0,0 +1,6 @@
1
+ """Model adapters — Bring-Your-Own-Model callables (stdlib only)."""
2
+
3
+ from .base import BaseModel, ModelFn, normalize_result
4
+ from .stub import StubModel
5
+
6
+ __all__ = ["ModelFn", "BaseModel", "StubModel", "normalize_result"]
@@ -0,0 +1,68 @@
1
+ """Base model interface: ModelFn(prompt) -> dict (stdlib only)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, Callable, Dict, Optional, Protocol, Tuple, runtime_checkable
6
+
7
+
8
+ class ModelResult(Dict[str, Any]):
9
+ """Convention: {"output": str, "usage": {"tokens": int, "cost_usd": float}}."""
10
+
11
+
12
+ ModelFn = Callable[[str], Dict[str, Any]]
13
+
14
+
15
+ @runtime_checkable
16
+ class _ModelFnProtocol(Protocol):
17
+ def __call__(self, prompt: str) -> Dict[str, Any]: ...
18
+
19
+
20
+ class BaseModel:
21
+ """Minimal base class for model adapters (optional convenience)."""
22
+
23
+ name = "base"
24
+
25
+ def __call__(self, prompt: str) -> Dict[str, Any]: # pragma: no cover - override
26
+ raise NotImplementedError("subclass must implement __call__(prompt)")
27
+
28
+
29
+ def _as_int(value: Any) -> Optional[int]:
30
+ if isinstance(value, bool):
31
+ return None
32
+ if isinstance(value, int) and value >= 0:
33
+ return value
34
+ return None
35
+
36
+
37
+ def _as_cost(value: Any) -> Optional[float]:
38
+ if isinstance(value, bool):
39
+ return None
40
+ if isinstance(value, (int, float)) and value >= 0:
41
+ return float(value)
42
+ return None
43
+
44
+
45
+ def normalize_result(raw: Any) -> Tuple[Optional[str], Optional[Dict[str, Any]]]:
46
+ """Normalize a raw ModelFn return into (output, usage).
47
+
48
+ Never raises: malformed payloads degrade to (None, None) and the
49
+ runner marks every check FAIL (fail-closed, SPEC section 3/5).
50
+ """
51
+ if not isinstance(raw, dict):
52
+ return None, None
53
+ output = raw.get("output")
54
+ if not isinstance(output, str):
55
+ output = None
56
+ usage_raw = raw.get("usage")
57
+ usage: Optional[Dict[str, Any]] = None
58
+ if isinstance(usage_raw, dict):
59
+ tokens = _as_int(usage_raw.get("tokens"))
60
+ cost_raw = usage_raw.get("cost_usd", usage_raw.get("cost"))
61
+ cost = _as_cost(cost_raw)
62
+ if tokens is not None or cost is not None:
63
+ usage = {}
64
+ if tokens is not None:
65
+ usage["tokens"] = tokens
66
+ if cost is not None:
67
+ usage["cost_usd"] = cost
68
+ return output, usage
@@ -0,0 +1,69 @@
1
+ """Optional OpenAI-compatible adapter (stdlib only, opt-in, v0.2.0 preview).
2
+
3
+ Not imported by the core runtime: no network happens unless you
4
+ instantiate and call this class explicitly. Uses ``urllib`` only —
5
+ no third-party dependency. Disabled by default because v0.1.0 is
6
+ offline-first (see README "Limitations & Roadmap").
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import os
13
+ import urllib.request
14
+ from typing import Any, Dict, Optional
15
+
16
+
17
+ class OpenAICompatModel:
18
+ """Minimal ``/chat/completions`` client for OpenAI-compatible APIs."""
19
+
20
+ def __init__(
21
+ self,
22
+ model: str = "gpt-4o-mini",
23
+ base_url: str = "https://api.openai.com/v1",
24
+ api_key: Optional[str] = None,
25
+ timeout_s: float = 30.0,
26
+ ) -> None:
27
+ self.model = model
28
+ self.base_url = base_url.rstrip("/")
29
+ self.api_key = api_key if api_key is not None else os.environ.get("OPENAI_API_KEY")
30
+ self.timeout_s = timeout_s
31
+
32
+ def __call__(self, prompt: str) -> Dict[str, Any]:
33
+ if not self.api_key:
34
+ raise RuntimeError(
35
+ "OpenAICompatModel needs an API key "
36
+ "(pass api_key= or set OPENAI_API_KEY). "
37
+ "The v0.1.0 core stays offline; this adapter is opt-in."
38
+ )
39
+ payload = {
40
+ "model": self.model,
41
+ "messages": [{"role": "user", "content": prompt}],
42
+ }
43
+ req = urllib.request.Request(
44
+ self.base_url + "/chat/completions",
45
+ data=json.dumps(payload).encode("utf-8"),
46
+ headers={
47
+ "Content-Type": "application/json",
48
+ "Authorization": "Bearer " + self.api_key,
49
+ },
50
+ method="POST",
51
+ )
52
+ with urllib.request.urlopen(req, timeout=self.timeout_s) as resp:
53
+ body = json.loads(resp.read().decode("utf-8"))
54
+ try:
55
+ content = body["choices"][0]["message"]["content"]
56
+ except (KeyError, IndexError, TypeError) as exc:
57
+ raise RuntimeError(f"unexpected chat-completions payload: {exc}") from exc
58
+ tokens: Optional[int] = None
59
+ try:
60
+ u = body.get("usage", {})
61
+ tokens = int(u.get("prompt_tokens", 0)) + int(u.get("completion_tokens", 0))
62
+ except (TypeError, ValueError):
63
+ tokens = None
64
+ usage: Dict[str, Any] = {}
65
+ if tokens is not None:
66
+ usage["tokens"] = tokens
67
+ # Cost is unknown without a pricing table -> omit (budget_cost fails
68
+ # closed when checked, per SPEC section 5).
69
+ return {"output": content if isinstance(content, str) else str(content), "usage": usage or None}
@@ -0,0 +1,89 @@
1
+ """StubModel — offline canned prompt->response fixtures (stdlib only)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import time
7
+ from pathlib import Path
8
+ from typing import Any, Dict, Mapping, Optional, Sequence, Union
9
+
10
+
11
+ class UnknownPromptError(KeyError):
12
+ """Raised when a prompt has no stubbed response (strict mode)."""
13
+
14
+
15
+ class StubModel:
16
+ """Replay canned responses — fully offline, reproducible, CI-friendly.
17
+
18
+ Mapping values accept either a plain string (used as ``output`` with
19
+ no usage) or a dict ``{"output": str, "usage": {...}, "delay_ms": int}``.
20
+ ``delay_ms`` simulates latency via ``time.sleep`` (StubModel-only knob;
21
+ real-model latency is measured wall-clock by the runner).
22
+ """
23
+
24
+ def __init__(
25
+ self,
26
+ mapping: Optional[Mapping[str, Any]] = None,
27
+ *,
28
+ default_output: str = "",
29
+ strict: bool = False,
30
+ ) -> None:
31
+ self._mapping: Dict[str, Any] = dict(mapping or {})
32
+ self.default_output = default_output
33
+ self.strict = strict
34
+
35
+ # -- constructors -----------------------------------------------------
36
+ @classmethod
37
+ def from_dict(cls, mapping: Mapping[str, Any], **kwargs: Any) -> "StubModel":
38
+ return cls(mapping, **kwargs)
39
+
40
+ @classmethod
41
+ def from_cases(cls, cases: Sequence[Mapping[str, Any]], **kwargs: Any) -> "StubModel":
42
+ """Build from loaded suite cases using each case's prompt.
43
+
44
+ Cases may carry an embedded ``stub`` block (examples/*.json style)
45
+ or a top-level ``output``/``stub_output`` fixture.
46
+ """
47
+ mapping: Dict[str, Any] = {}
48
+ for case in cases:
49
+ prompt = case.get("prompt")
50
+ if not isinstance(prompt, str):
51
+ continue
52
+ if isinstance(case.get("stub"), dict):
53
+ mapping[prompt] = case["stub"]
54
+ elif "output" in case and isinstance(case.get("output"), str):
55
+ mapping[prompt] = {
56
+ "output": case["output"],
57
+ "usage": case.get("usage"),
58
+ }
59
+ return cls(mapping, **kwargs)
60
+
61
+ @classmethod
62
+ def from_json(cls, path: Union[str, Path], **kwargs: Any) -> "StubModel":
63
+ raw = json.loads(Path(path).read_text(encoding="utf-8"))
64
+ if isinstance(raw, dict) and "mapping" in raw and isinstance(raw["mapping"], dict):
65
+ raw = raw["mapping"]
66
+ if not isinstance(raw, dict):
67
+ raise ValueError(f"stub fixture must be a JSON object: {path}")
68
+ return cls(raw, **kwargs)
69
+
70
+ # -- call -------------------------------------------------------------
71
+ def __call__(self, prompt: str) -> Dict[str, Any]:
72
+ if prompt in self._mapping:
73
+ entry = self._mapping[prompt]
74
+ elif self.strict:
75
+ raise UnknownPromptError(f"no stubbed response for prompt: {prompt!r}")
76
+ else:
77
+ return {"output": self.default_output, "usage": None}
78
+ if isinstance(entry, str):
79
+ return {"output": entry, "usage": None}
80
+ if not isinstance(entry, dict):
81
+ return {"output": self.default_output, "usage": None}
82
+ delay = entry.get("delay_ms", 0)
83
+ try:
84
+ delay_ms = int(delay or 0)
85
+ except (TypeError, ValueError):
86
+ delay_ms = 0
87
+ if delay_ms > 0:
88
+ time.sleep(min(delay_ms, 5000) / 1000.0)
89
+ return {"output": entry.get("output", ""), "usage": entry.get("usage")}