slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,201 @@
1
+ """What a run was, so a later comparison is evidence and not guesswork.
2
+
3
+ Two scorecards side by side are only meaningful if the things around the models
4
+ were held still: same dataset, same config, same harness. Nobody remembers
5
+ whether they edited a case last Tuesday. So every run records what it was built
6
+ from, and ``slmeval compare`` can refuse a comparison whose premises differ.
7
+
8
+ That refusal is the feature. A tool that cheerfully prints a 12-point
9
+ regression caused by a dataset edit teaches people to distrust its numbers,
10
+ once, permanently.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import os
16
+ import platform
17
+ import secrets
18
+ import subprocess
19
+ import sys
20
+ from dataclasses import dataclass
21
+ from datetime import UTC, datetime
22
+ from pathlib import Path
23
+ from typing import Any, Final
24
+
25
+ from slmeval import __version__
26
+ from slmeval.storage import codec
27
+
28
+ GIT_TIMEOUT_S: Final = 2.0
29
+ """Provenance is never worth stalling a run for. A slow or wedged git - a
30
+ network-mounted repository, a stale index lock - degrades to "no SHA recorded",
31
+ which is a gap in a report, not a failed benchmark.
32
+ """
33
+
34
+
35
+ def new_run_id(*, now: datetime | None = None) -> str:
36
+ """``YYYYMMDD-HHMMSS-<6 hex>``.
37
+
38
+ Lexically sortable, so ``ORDER BY run_id`` is chronological and a user
39
+ skimming ``.slmeval/`` can read the dates. The random suffix because two
40
+ runs started in the same second on one machine is not hypothetical - it is
41
+ what a CI matrix does.
42
+ """
43
+ stamp = (now or datetime.now()).strftime("%Y%m%d-%H%M%S")
44
+ return f"{stamp}-{secrets.token_hex(3)}"
45
+
46
+
47
+ def git_sha(root: Path | None = None) -> str:
48
+ """Short commit SHA, with ``-dirty`` when the tree has changes.
49
+
50
+ The dirty marker earns its place: "which commit scored 83%" is useless if
51
+ the answer is a commit plus uncommitted edits nobody kept. Marked as dirty,
52
+ a user at least knows not to trust the SHA.
53
+
54
+ Every failure returns the empty string. Git may be absent, the directory
55
+ may not be a repository, and neither is a reason to fail a benchmark.
56
+ """
57
+ cwd = str(root or Path.cwd())
58
+ try:
59
+ sha = subprocess.run(
60
+ ["git", "rev-parse", "--short", "HEAD"],
61
+ cwd=cwd,
62
+ capture_output=True,
63
+ text=True,
64
+ timeout=GIT_TIMEOUT_S,
65
+ check=False,
66
+ )
67
+ if sha.returncode != 0:
68
+ return ""
69
+ value = sha.stdout.strip()
70
+ if not value:
71
+ return ""
72
+ dirty = subprocess.run(
73
+ ["git", "status", "--porcelain", "--untracked-files=no"],
74
+ cwd=cwd,
75
+ capture_output=True,
76
+ text=True,
77
+ timeout=GIT_TIMEOUT_S,
78
+ check=False,
79
+ )
80
+ if dirty.returncode == 0 and dirty.stdout.strip():
81
+ return f"{value}-dirty"
82
+ return value
83
+ except (OSError, subprocess.SubprocessError):
84
+ return ""
85
+
86
+
87
+ def host_info() -> dict[str, Any]:
88
+ """Enough about the machine to explain a latency difference.
89
+
90
+ Latency and p95 are headline columns, and they are properties of the
91
+ machine as much as the model. A run from a laptop on battery and one from a
92
+ CI runner are not the same measurement; recorded here, that is visible
93
+ instead of mysterious.
94
+
95
+ Deliberately no hostname, no username, no local paths beyond the project
96
+ the user chose: this file gets attached to bug reports and pasted into
97
+ issues.
98
+ """
99
+ return {
100
+ "os": platform.system(),
101
+ "release": platform.release(),
102
+ "machine": platform.machine(),
103
+ "python": platform.python_version(),
104
+ "implementation": sys.implementation.name,
105
+ "cpus": os.cpu_count() or 0,
106
+ }
107
+
108
+
109
+ @dataclass(frozen=True, slots=True)
110
+ class Provenance:
111
+ """The inputs to one run, hashed and stamped."""
112
+
113
+ config_hash: str
114
+ dataset_hash: str
115
+ dataset_path: str = ""
116
+ git_sha: str = ""
117
+ slmeval_version: str = __version__
118
+ host: str = "{}"
119
+ started_at: str = ""
120
+
121
+ @classmethod
122
+ def collect(
123
+ cls,
124
+ *,
125
+ config_hash: str,
126
+ dataset_hash: str,
127
+ dataset_path: Path | str = "",
128
+ root: Path | None = None,
129
+ ) -> Provenance:
130
+ return cls(
131
+ config_hash=config_hash,
132
+ dataset_hash=dataset_hash,
133
+ dataset_path=str(dataset_path),
134
+ git_sha=git_sha(root),
135
+ slmeval_version=__version__,
136
+ host=codec.dumps(host_info()),
137
+ started_at=datetime.now(UTC).isoformat(),
138
+ )
139
+
140
+
141
+ # Fields that must match for two runs to be comparable, and what to say when
142
+ # they do not. Ordered by how badly each one invalidates a comparison, because
143
+ # the CLI shows the first line most prominently.
144
+ _COMPARABLE: Final = (
145
+ ("dataset_hash", "the dataset changed between these runs"),
146
+ ("config_hash", "the benchmark config changed between these runs"),
147
+ ("mode", "one run replayed cassettes and the other did not"),
148
+ ("seed", "the runs used different seeds, so they met different faults"),
149
+ ("slmeval_version", "the runs were produced by different slmeval versions"),
150
+ )
151
+
152
+
153
+ @dataclass(frozen=True, slots=True)
154
+ class Divergence:
155
+ """One reason two runs are not comparable."""
156
+
157
+ field: str
158
+ reason: str
159
+ left: str
160
+ right: str
161
+
162
+ @property
163
+ def blocking(self) -> bool:
164
+ """Whether this difference should stop a comparison outright.
165
+
166
+ A different ``slmeval_version`` is a caveat - the numbers were computed
167
+ by different code, worth printing, rarely worth refusing over. A
168
+ different dataset, config, mode or seed means the two tables answer
169
+ different questions, and putting them in one view manufactures a
170
+ regression out of thin air.
171
+ """
172
+ return self.field != "slmeval_version"
173
+
174
+
175
+ def divergences(left: object, right: object) -> list[Divergence]:
176
+ """Why two :class:`~slmeval.storage.models.RunRecord` values differ.
177
+
178
+ Takes anything with the provenance attributes rather than importing the
179
+ record type, so the config layer can check a pending run against a stored
180
+ one before spending twenty minutes on a comparison nobody can use.
181
+ """
182
+ out: list[Divergence] = []
183
+ for field_name, reason in _COMPARABLE:
184
+ a = getattr(left, field_name, None)
185
+ b = getattr(right, field_name, None)
186
+ if a != b:
187
+ out.append(
188
+ Divergence(field=field_name, reason=reason, left=str(a), right=str(b))
189
+ )
190
+ return out
191
+
192
+
193
+ __all__ = [
194
+ "GIT_TIMEOUT_S",
195
+ "Divergence",
196
+ "Provenance",
197
+ "divergences",
198
+ "git_sha",
199
+ "host_info",
200
+ "new_run_id",
201
+ ]
@@ -0,0 +1,484 @@
1
+ Metadata-Version: 2.5
2
+ Name: slmeval-cli
3
+ Version: 0.1.0
4
+ Summary: Benchmark agent workflows across small language models — locally, deterministically, in your terminal.
5
+ Project-URL: Homepage, https://github.com/mohitraj3697/slmeval
6
+ Project-URL: Documentation, https://github.com/mohitraj3697/slmeval/tree/main/docs
7
+ Project-URL: Changelog, https://github.com/mohitraj3697/slmeval/blob/main/CHANGELOG.md
8
+ Project-URL: Issues, https://github.com/mohitraj3697/slmeval/issues
9
+ Author: Mohit Raj
10
+ License: Apache License
11
+ Version 2.0, January 2004
12
+ http://www.apache.org/licenses/
13
+
14
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
15
+
16
+ 1. Definitions.
17
+
18
+ "License" shall mean the terms and conditions for use, reproduction,
19
+ and distribution as defined by Sections 1 through 9 of this document.
20
+
21
+ "Licensor" shall mean the copyright owner or entity authorized by
22
+ the copyright owner that is granting the License.
23
+
24
+ "Legal Entity" shall mean the union of the acting entity and all
25
+ other entities that control, are controlled by, or are under common
26
+ control with that entity. For the purposes of this definition,
27
+ "control" means (i) the power, direct or indirect, to cause the
28
+ direction or management of such entity, whether by contract or
29
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
30
+ outstanding shares, or (iii) beneficial ownership of such entity.
31
+
32
+ "You" (or "Your") shall mean an individual or Legal Entity
33
+ exercising permissions granted by this License.
34
+
35
+ "Source" form shall mean the preferred form for making modifications,
36
+ including but not limited to software source code, documentation
37
+ source, and configuration files.
38
+
39
+ "Object" form shall mean any form resulting from mechanical
40
+ transformation or translation of a Source form, including but
41
+ not limited to compiled object code, generated documentation,
42
+ and conversions to other media types.
43
+
44
+ "Work" shall mean the work of authorship, whether in Source or
45
+ Object form, made available under the License, as indicated by a
46
+ copyright notice that is included in or attached to the work
47
+ (an example is provided in the Appendix below).
48
+
49
+ "Derivative Works" shall mean any work, whether in Source or Object
50
+ form, that is based on (or derived from) the Work and for which the
51
+ editorial revisions, annotations, elaborations, or other modifications
52
+ represent, as a whole, an original work of authorship. For the purposes
53
+ of this License, Derivative Works shall not include works that remain
54
+ separable from, or merely link (or bind by name) to the interfaces of,
55
+ the Work and Derivative Works thereof.
56
+
57
+ "Contribution" shall mean any work of authorship, including
58
+ the original version of the Work and any modifications or additions
59
+ to that Work or Derivative Works thereof, that is intentionally
60
+ submitted to Licensor for inclusion in the Work by the copyright owner
61
+ or by an individual or Legal Entity authorized to submit on behalf of
62
+ the copyright owner. For the purposes of this definition, "submitted"
63
+ means any form of electronic, verbal, or written communication sent
64
+ to the Licensor or its representatives, including but not limited to
65
+ communication on electronic mailing lists, source code control systems,
66
+ and issue tracking systems that are managed by, or on behalf of, the
67
+ Licensor for the purpose of discussing and improving the Work, but
68
+ excluding communication that is conspicuously marked or otherwise
69
+ designated in writing by the copyright owner as "Not a Contribution."
70
+
71
+ "Contributor" shall mean Licensor and any individual or Legal Entity
72
+ on behalf of whom a Contribution has been received by Licensor and
73
+ subsequently incorporated within the Work.
74
+
75
+ 2. Grant of Copyright License. Subject to the terms and conditions of
76
+ this License, each Contributor hereby grants to You a perpetual,
77
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
78
+ copyright license to reproduce, prepare Derivative Works of,
79
+ publicly display, publicly perform, sublicense, and distribute the
80
+ Work and such Derivative Works in Source or Object form.
81
+
82
+ 3. Grant of Patent License. Subject to the terms and conditions of
83
+ this License, each Contributor hereby grants to You a perpetual,
84
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
85
+ (except as stated in this section) patent license to make, have made,
86
+ use, offer to sell, sell, import, and otherwise transfer the Work,
87
+ where such license applies only to those patent claims licensable
88
+ by such Contributor that are necessarily infringed by their
89
+ Contribution(s) alone or by combination of their Contribution(s)
90
+ with the Work to which such Contribution(s) was submitted. If You
91
+ institute patent litigation against any entity (including a
92
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
93
+ or a Contribution incorporated within the Work constitutes direct
94
+ or contributory patent infringement, then any patent licenses
95
+ granted to You under this License for that Work shall terminate
96
+ as of the date such litigation is filed.
97
+
98
+ 4. Redistribution. You may reproduce and distribute copies of the
99
+ Work or Derivative Works thereof in any medium, with or without
100
+ modifications, and in Source or Object form, provided that You
101
+ meet the following conditions:
102
+
103
+ (a) You must give any other recipients of the Work or
104
+ Derivative Works a copy of this License; and
105
+
106
+ (b) You must cause any modified files to carry prominent notices
107
+ stating that You changed the files; and
108
+
109
+ (c) You must retain, in the Source form of any Derivative Works
110
+ that You distribute, all copyright, patent, trademark, and
111
+ attribution notices from the Source form of the Work,
112
+ excluding those notices that do not pertain to any part of
113
+ the Derivative Works; and
114
+
115
+ (d) If the Work includes a "NOTICE" text file as part of its
116
+ distribution, then any Derivative Works that You distribute must
117
+ include a readable copy of the attribution notices contained
118
+ within such NOTICE file, excluding those notices that do not
119
+ pertain to any part of the Derivative Works, in at least one
120
+ of the following places: within a NOTICE text file distributed
121
+ as part of the Derivative Works; within the Source form or
122
+ documentation, if provided along with the Derivative Works; or,
123
+ within a display generated by the Derivative Works, if and
124
+ wherever such third-party notices normally appear. The contents
125
+ of the NOTICE file are for informational purposes only and
126
+ do not modify the License. You may add Your own attribution
127
+ notices within Derivative Works that You distribute, alongside
128
+ or as an addendum to the NOTICE text from the Work, provided
129
+ that such additional attribution notices cannot be construed
130
+ as modifying the License.
131
+
132
+ You may add Your own copyright statement to Your modifications and
133
+ may provide additional or different license terms and conditions
134
+ for use, reproduction, or distribution of Your modifications, or
135
+ for any such Derivative Works as a whole, provided Your use,
136
+ reproduction, and distribution of the Work otherwise complies with
137
+ the conditions stated in this License.
138
+
139
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
140
+ any Contribution intentionally submitted for inclusion in the Work
141
+ by You to the Licensor shall be under the terms and conditions of
142
+ this License, without any additional terms or conditions.
143
+ Notwithstanding the above, nothing herein shall supersede or modify
144
+ the terms of any separate license agreement you may have executed
145
+ with Licensor regarding such Contributions.
146
+
147
+ 6. Trademarks. This License does not grant permission to use the trade
148
+ names, trademarks, service marks, or product names of the Licensor,
149
+ except as required for reasonable and customary use in describing the
150
+ origin of the Work and reproducing the content of the NOTICE file.
151
+
152
+ 7. Disclaimer of Warranty. Unless required by applicable law or
153
+ agreed to in writing, Licensor provides the Work (and each
154
+ Contributor provides its Contributions) on an "AS IS" BASIS,
155
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
156
+ implied, including, without limitation, any warranties or conditions
157
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
158
+ PARTICULAR PURPOSE. You are solely responsible for determining the
159
+ appropriateness of using or redistributing the Work and assume any
160
+ risks associated with Your exercise of permissions under this License.
161
+
162
+ 8. Limitation of Liability. In no event and under no legal theory,
163
+ whether in tort (including negligence), contract, or otherwise,
164
+ unless required by applicable law (such as deliberate and grossly
165
+ negligent acts) or agreed to in writing, shall any Contributor be
166
+ liable to You for damages, including any direct, indirect, special,
167
+ incidental, or consequential damages of any character arising as a
168
+ result of this License or out of the use or inability to use the
169
+ Work (including but not limited to damages for loss of goodwill,
170
+ work stoppage, computer failure or malfunction, or any and all
171
+ other commercial damages or losses), even if such Contributor
172
+ has been advised of the possibility of such damages.
173
+
174
+ 9. Accepting Warranty or Additional Liability. While redistributing
175
+ the Work or Derivative Works thereof, You may choose to offer,
176
+ and charge a fee for, acceptance of support, warranty, indemnity,
177
+ or other liability obligations and/or rights consistent with this
178
+ License. However, in accepting such obligations, You may act only
179
+ on Your own behalf and on Your sole responsibility, not on behalf
180
+ of any other Contributor, and only if You agree to indemnify,
181
+ defend, and hold each Contributor harmless for any liability
182
+ incurred by, or claims asserted against, such Contributor by reason
183
+ of your accepting any such warranty or additional liability.
184
+
185
+ END OF TERMS AND CONDITIONS
186
+
187
+ APPENDIX: How to apply the Apache License to your work.
188
+
189
+ To apply the Apache License to your work, attach the following
190
+ boilerplate notice, with the fields enclosed by brackets "[]"
191
+ replaced with your own identifying information. (Don't include
192
+ the brackets!) The text should be enclosed in the appropriate
193
+ comment syntax for the file format. We also recommend that a
194
+ file or class name and description of purpose be included on the
195
+ same "printed page" as the copyright notice for easier
196
+ identification within third-party archives.
197
+
198
+ Copyright 2026 slmeval contributors
199
+
200
+ Licensed under the Apache License, Version 2.0 (the "License");
201
+ you may not use this file except in compliance with the License.
202
+ You may obtain a copy of the License at
203
+
204
+ http://www.apache.org/licenses/LICENSE-2.0
205
+
206
+ Unless required by applicable law or agreed to in writing, software
207
+ distributed under the License is distributed on an "AS IS" BASIS,
208
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
209
+ See the License for the specific language governing permissions and
210
+ limitations under the License.
211
+ License-File: LICENSE
212
+ Keywords: agents,benchmark,ci,evaluation,langgraph,llm,ollama,slm
213
+ Classifier: Development Status :: 3 - Alpha
214
+ Classifier: Environment :: Console
215
+ Classifier: Intended Audience :: Developers
216
+ Classifier: License :: OSI Approved :: Apache Software License
217
+ Classifier: Programming Language :: Python :: 3.11
218
+ Classifier: Programming Language :: Python :: 3.12
219
+ Classifier: Programming Language :: Python :: 3.13
220
+ Classifier: Topic :: Software Development :: Testing
221
+ Classifier: Typing :: Typed
222
+ Requires-Python: >=3.11
223
+ Requires-Dist: httpx>=0.27
224
+ Requires-Dist: pydantic>=2.7
225
+ Requires-Dist: rich>=13.7
226
+ Requires-Dist: typer>=0.12
227
+ Provides-Extra: anthropic
228
+ Requires-Dist: anthropic>=0.34; extra == 'anthropic'
229
+ Provides-Extra: dev
230
+ Requires-Dist: mypy>=1.11; extra == 'dev'
231
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
232
+ Requires-Dist: pytest>=8.0; extra == 'dev'
233
+ Requires-Dist: pyyaml>=6.0; extra == 'dev'
234
+ Requires-Dist: ruff>=0.6; extra == 'dev'
235
+ Provides-Extra: langgraph
236
+ Requires-Dist: langchain-openai>=0.2; extra == 'langgraph'
237
+ Requires-Dist: langgraph>=0.2; extra == 'langgraph'
238
+ Description-Content-Type: text/markdown
239
+
240
+ # slmeval
241
+
242
+ [![CI](https://github.com/mohitraj3697/slmeval/actions/workflows/ci.yml/badge.svg)](https://github.com/mohitraj3697/slmeval/actions/workflows/ci.yml)
243
+ [![PyPI](https://img.shields.io/pypi/v/slmeval-cli)](https://pypi.org/project/slmeval-cli/)
244
+ [![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://github.com/mohitraj3697/slmeval/blob/main/pyproject.toml)
245
+ [![License](https://img.shields.io/badge/license-Apache%202.0-blue)](https://github.com/mohitraj3697/slmeval/blob/main/LICENSE)
246
+
247
+ Test whether a small model can take over steps of your agent, without calling
248
+ your real tools again and without a GPU in CI.
249
+
250
+ slmeval records your agent's tool calls once, then replays those responses while
251
+ each candidate model runs your test cases. You get a scorecard per model and per
252
+ agent step, and a command that fails the build when a model falls below your
253
+ thresholds.
254
+
255
+ ![Terminal output of slmeval report: a warning about a forbidden tool call, a table comparing four models, and a table showing which agent step each model failed at](https://github.com/mohitraj3697/slmeval/raw/main/docs/assets/scorecard.svg)
256
+
257
+ Output from the bundled demo, shortened. **The four models are scripted stand-ins,
258
+ not real SLMs**: each one is written to fail in a known way, to show what the tool
259
+ reports. These numbers say nothing about any real model.
260
+
261
+ ## Why
262
+
263
+ Parts of most agents, such as routing, extraction and formatting, may not need a
264
+ frontier model. Finding out is awkward: every test run calls your real tools, which
265
+ costs money and can have side effects, and the tool results change between runs.
266
+
267
+ With slmeval, the tool responses stay fixed, so the model is the only thing that
268
+ changes. Runs repeat exactly on the tool side and need no network for tools. You can
269
+ also swap the model on a single step and leave the rest of the agent alone.
270
+
271
+ ## Try the demo
272
+
273
+ ```bash
274
+ git clone https://github.com/mohitraj3697/slmeval
275
+ cd slmeval/examples/support_agent
276
+ pip install slmeval-cli
277
+ slmeval record
278
+ slmeval run
279
+ ```
280
+
281
+ This takes under a minute and needs no API key or GPU. `record` runs the agent's
282
+ tools against an in-memory stand-in for an order system. `run` replays them against
283
+ four models.
284
+
285
+ The package is `slmeval-cli` on PyPI. The command and the Python import are both
286
+ `slmeval`. `slm-eval` on PyPI is a different project.
287
+
288
+ ## What the demo shows
289
+
290
+ The agent is a small customer support bot with four steps and five tools. The
291
+ results below come from [Teardown #1](https://github.com/mohitraj3697/slmeval/tree/main/teardown),
292
+ which publishes every output along with the config that produced it.
293
+
294
+ | Model | What it is scripted to do | What slmeval reports |
295
+ |---|---|---|
296
+ | `baseline` | Answers correctly | The reference for the others |
297
+ | `fenced-json` | Answers correctly, but wraps its JSON in prose and a code fence | Passes every case, with 0% schema-valid output on the first try |
298
+ | `sloppy-router` | Drops the `ORD-` prefix from order ids | A failed tool lookup, with a diff of the arguments |
299
+ | `refund-happy` | Claims the customer confirmed a refund they only asked for | A forbidden tool call, which fails the gate at any pass rate |
300
+
301
+ | Candidate | pass^3 | 95% CI | runs | Schema | Tool F1 | Steps |
302
+ |---|---|---|---|---|---|---|
303
+ | **baseline** | 100.0% | [51%, 100%] | 12/12 | 100.0% | 1.00 | 4.8 |
304
+ | sloppy-router | 33.3% | [6%, 79%] | 3/9 | 100.0% | 0.40 | 1.8 |
305
+ | fenced-json | 100.0% | [51%, 100%] | 12/12 | 0.0% | 1.00 | 4.8 |
306
+ | refund-happy | 75.0% | [30%, 95%] | 9/12 | 100.0% | 0.96 | 4.8 |
307
+
308
+ `pass^3` is the share of cases that passed all three repeats. The demo uses only four
309
+ cases, so the intervals are wide. A real comparison needs far more.
310
+
311
+ ### Pass rates come with intervals
312
+
313
+ ![Chart of pass rate and 95 percent interval for each model. baseline and fenced-json reach 100 percent with a lower bound of 51 percent, which clears a gate floor of 50 percent. sloppy-router and refund-happy fall below it.](https://github.com/mohitraj3697/slmeval/raw/main/docs/assets/pass-rates.svg)
314
+
315
+ Passing four cases out of four still only gives a lower bound of 51%. `slmeval gate`
316
+ checks the lower bound by default, so a small sample can't pass on its point estimate
317
+ alone.
318
+
319
+ ### Failures point to a step
320
+
321
+ ![Grid of models against agent steps. sloppy-router failed 6 times at order_lookup, refund-happy failed 3 times at policy_check, and the other steps passed or made no model call.](https://github.com/mohitraj3697/slmeval/raw/main/docs/assets/where-models-failed.svg)
322
+
323
+ When a model asks for something that was never recorded, slmeval finds the closest
324
+ recorded call and shows what differs:
325
+
326
+ ```text
327
+ ✗ status_shipped · sloppy-router · node=order_lookup · CASSETTE_MISS
328
+ tool: fetch_order
329
+ called with: {"order_id": "9921", "request_id": "req-9921"}
330
+ nearest recorded: {"order_id": "ORD-9921", "request_id": "req-ORD-9921"}
331
+ diff: order_id: "ORD-9921" → "9921" (dropped prefix)
332
+ request_id: "req-ORD-9921" → "req-9921" (different value)
333
+ verdict: hallucinated_parameter
334
+ ```
335
+
336
+ If the model took a reasonable path that the baseline just never recorded, slmeval
337
+ reports `unrecorded_valid_path` instead and leaves that run out of the pass rate.
338
+ That is why `sloppy-router` shows 3/9 runs and not 3/12.
339
+
340
+ ### Fault injection finds agent bugs too
341
+
342
+ ![Chart comparing pass rates with no faults and with 30 percent of tool calls faulted. baseline and fenced-json drop from 100 to 50 percent, refund-happy from 75 to 50 percent, and sloppy-router stays at 33 percent.](https://github.com/mohitraj3697/slmeval/raw/main/docs/assets/fault-injection.svg)
343
+
344
+ Re-running with injected tool failures found a bug in our own demo agent: it crashes
345
+ with `KeyError: 'carrier'` when a tool returns an empty response. No model could
346
+ recover from that, including the baseline. We left the bug in the demo because it
347
+ shows what fault injection is for.
348
+
349
+ ## Use it on your agent
350
+
351
+ Install it and create a starter config:
352
+
353
+ ```bash
354
+ pip install slmeval-cli
355
+ slmeval init
356
+ slmeval doctor
357
+ ```
358
+
359
+ Then mark the model calls and tools in your agent. Both helpers do nothing outside a
360
+ benchmark, so they are safe to leave in production code:
361
+
362
+ ```python
363
+ from slmeval import chat_model, node, sandbox_tool
364
+
365
+ @sandbox_tool(match_on=["order_id"], ignore_args=["request_id"])
366
+ async def fetch_order(order_id: str, request_id: str) -> dict:
367
+ ... # recorded once, then replayed
368
+
369
+ async def run_case(case_input: dict) -> dict:
370
+ with node("intent_router"): # names this step in the report
371
+ llm = chat_model(provider="openai", model="gpt-4o-mini") # used outside a benchmark
372
+ completion = await llm.complete(messages, schema=Routing)
373
+ ...
374
+ ```
375
+
376
+ Point `eval.toml` at that function and list the models to try. `target_nodes` swaps
377
+ the model on the named steps only:
378
+
379
+ ```toml
380
+ [target]
381
+ entrypoint = "my_agent:run_case"
382
+
383
+ [baseline]
384
+ provider = "openai"
385
+ model = "gpt-4o-mini"
386
+
387
+ [[candidates]]
388
+ id = "gemma3-router"
389
+ provider = "ollama"
390
+ model = "gemma3:4b"
391
+ target_nodes = ["intent_router"]
392
+ ```
393
+
394
+ ```bash
395
+ slmeval record
396
+ slmeval run
397
+ slmeval show run
398
+ ```
399
+
400
+ For LangGraph, `slmeval.instrument_langgraph(builder)` replaces the `node()` blocks.
401
+ The [getting started guide](https://github.com/mohitraj3697/slmeval/blob/main/docs/getting-started.md)
402
+ covers the rest, including how to write test cases.
403
+
404
+ ## Use it in CI
405
+
406
+ ```bash
407
+ slmeval run --tool-mode replay
408
+ slmeval gate --min-pass-k 0.90 --max-cost-ratio 0.40
409
+ ```
410
+
411
+ Tool responses come from a local SQLite file, so CI needs no tool credentials. If
412
+ your candidates are hosted or local models, the runner still needs to reach them.
413
+ Use the `fake` provider for a check with no model server at all.
414
+
415
+ | Exit code | Meaning |
416
+ |---|---|
417
+ | 0 | Every threshold was met |
418
+ | 1 | A threshold was missed, or a forbidden tool was called |
419
+ | 2 | Bad invocation, such as no threshold set |
420
+ | 3 | A model server was unreachable, so nothing was measured |
421
+ | 4 | Not enough recorded responses to judge the run |
422
+
423
+ There is also a [GitHub Action](https://github.com/mohitraj3697/slmeval/tree/main/examples/github-action)
424
+ that writes the scorecard to the job summary and keeps one comment updated on the pull
425
+ request:
426
+
427
+ ```yaml
428
+ - uses: actions/checkout@v4
429
+ - uses: mohitraj3697/slmeval/examples/github-action@main
430
+ with:
431
+ working-directory: evals
432
+ cassettes: evals/cassettes.sqlite
433
+ ```
434
+
435
+ ## How it works
436
+
437
+ ![Diagram. Once: your agent and its real tools go through slmeval record into cassettes.sqlite. Every run: candidates go through slmeval run, which reads the cassettes, then checks, a report, and slmeval gate.](https://github.com/mohitraj3697/slmeval/raw/main/docs/assets/how-it-works.svg)
438
+
439
+ - **Tools** are recorded once and served from SQLite after that. Return types such as
440
+ `datetime` and `Decimal` come back as the same types.
441
+ - **Checks** run in tiers: assertions from your dataset, then tool call and path
442
+ comparisons against the baseline, then pass rates across repeats. An LLM judge exists
443
+ but isn't connected to `slmeval run` yet.
444
+ - **Results** are stored per run, so you can re-render a report or inspect a single
445
+ execution later without loading your agent.
446
+
447
+ ## Status
448
+
449
+ Alpha. Version 0.1.0 works end to end, and the exit codes and JSON report schema are
450
+ treated as stable.
451
+
452
+ - **Providers:** OpenAI, Ollama, anything that speaks `/v1/chat/completions`, and a
453
+ scripted `fake` provider for offline runs.
454
+ - **Not built yet:** `slmeval compare`, `slmeval cassette`, a native Anthropic provider
455
+ and time-to-first-token measurement.
456
+ - **No published results for real models yet.** Teardown #1 uses scripted models. If
457
+ you run it against real ones,
458
+ [share the result](https://github.com/mohitraj3697/slmeval/issues/new?template=benchmark_result.yml).
459
+
460
+ slmeval sends no telemetry. Its only network traffic is to the model endpoints you
461
+ configure, and `eval.toml` holds the name of an API key's environment variable, never
462
+ the key itself.
463
+
464
+ ## Documentation
465
+
466
+ | | |
467
+ |---|---|
468
+ | [Getting started](https://github.com/mohitraj3697/slmeval/blob/main/docs/getting-started.md) | Connect your agent and write test cases |
469
+ | [Reading results](https://github.com/mohitraj3697/slmeval/blob/main/docs/reading-results.md) | What each number means, and how to drill into a failure |
470
+ | [Configuration](https://github.com/mohitraj3697/slmeval/blob/main/docs/configuration.md) | Providers, structured output, the `fake` provider and fault injection |
471
+ | [CI](https://github.com/mohitraj3697/slmeval/blob/main/docs/ci.md) | Gate thresholds, baselines and the GitHub Action |
472
+ | [Architecture](https://github.com/mohitraj3697/slmeval/blob/main/docs/architecture.md) | How the pieces fit, what is stored, and the LLM judge |
473
+ | [Teardown #1](https://github.com/mohitraj3697/slmeval/tree/main/teardown) | The demo's full results and their limitations |
474
+
475
+ ## Contributing
476
+
477
+ Bug reports and pull requests are welcome. See
478
+ [CONTRIBUTING.md](https://github.com/mohitraj3697/slmeval/blob/main/CONTRIBUTING.md). To
479
+ report a security issue, follow
480
+ [SECURITY.md](https://github.com/mohitraj3697/slmeval/blob/main/SECURITY.md).
481
+
482
+ ## License
483
+
484
+ [Apache 2.0](https://github.com/mohitraj3697/slmeval/blob/main/LICENSE)