wiretruth 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. wiretruth-0.2.0/LICENSE +21 -0
  2. wiretruth-0.2.0/PKG-INFO +287 -0
  3. wiretruth-0.2.0/README.md +258 -0
  4. wiretruth-0.2.0/pyproject.toml +57 -0
  5. wiretruth-0.2.0/setup.cfg +4 -0
  6. wiretruth-0.2.0/src/wiretruth/__init__.py +3 -0
  7. wiretruth-0.2.0/src/wiretruth/__main__.py +4 -0
  8. wiretruth-0.2.0/src/wiretruth/catalog.py +78 -0
  9. wiretruth-0.2.0/src/wiretruth/checks.py +558 -0
  10. wiretruth-0.2.0/src/wiretruth/cli.py +539 -0
  11. wiretruth-0.2.0/src/wiretruth/envs.py +120 -0
  12. wiretruth-0.2.0/src/wiretruth/formats.py +1136 -0
  13. wiretruth-0.2.0/src/wiretruth/grading.py +164 -0
  14. wiretruth-0.2.0/src/wiretruth/lint.py +283 -0
  15. wiretruth-0.2.0/src/wiretruth/providers/__init__.py +235 -0
  16. wiretruth-0.2.0/src/wiretruth/providers/anthropic.py +340 -0
  17. wiretruth-0.2.0/src/wiretruth/providers/gemini.py +322 -0
  18. wiretruth-0.2.0/src/wiretruth/providers/ollama.py +271 -0
  19. wiretruth-0.2.0/src/wiretruth/providers/ollama_openai.py +28 -0
  20. wiretruth-0.2.0/src/wiretruth/providers/openai_chat.py +405 -0
  21. wiretruth-0.2.0/src/wiretruth/providers/openai_responses.py +368 -0
  22. wiretruth-0.2.0/src/wiretruth/recorder.py +519 -0
  23. wiretruth-0.2.0/src/wiretruth/replay.py +391 -0
  24. wiretruth-0.2.0/src/wiretruth/results.py +216 -0
  25. wiretruth-0.2.0/src/wiretruth/runner.py +353 -0
  26. wiretruth-0.2.0/src/wiretruth/schemas/bug-classes.schema.json +29 -0
  27. wiretruth-0.2.0/src/wiretruth/schemas/call.schema.json +113 -0
  28. wiretruth-0.2.0/src/wiretruth/schemas/common.schema.json +98 -0
  29. wiretruth-0.2.0/src/wiretruth/schemas/fixture.schema.json +243 -0
  30. wiretruth-0.2.0/src/wiretruth/schemas/observation.schema.json +191 -0
  31. wiretruth-0.2.0/src/wiretruth/schemas/results.schema.json +91 -0
  32. wiretruth-0.2.0/src/wiretruth/schemas/scenario.schema.json +91 -0
  33. wiretruth-0.2.0/src/wiretruth/schemas/shim-manifest.schema.json +125 -0
  34. wiretruth-0.2.0/src/wiretruth/schemas/upstream.schema.json +60 -0
  35. wiretruth-0.2.0/src/wiretruth/secret_scan.py +191 -0
  36. wiretruth-0.2.0/src/wiretruth/transforms.py +239 -0
  37. wiretruth-0.2.0/src/wiretruth.egg-info/PKG-INFO +287 -0
  38. wiretruth-0.2.0/src/wiretruth.egg-info/SOURCES.txt +41 -0
  39. wiretruth-0.2.0/src/wiretruth.egg-info/dependency_links.txt +1 -0
  40. wiretruth-0.2.0/src/wiretruth.egg-info/entry_points.txt +2 -0
  41. wiretruth-0.2.0/src/wiretruth.egg-info/requires.txt +8 -0
  42. wiretruth-0.2.0/src/wiretruth.egg-info/top_level.txt +1 -0
  43. wiretruth-0.2.0/tests/test_cli.py +349 -0
wiretruth-0.2.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 kartsan03
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
wiretruth-0.2.0/PKG-INFO ADDED
@@ -0,0 +1,287 @@
1
+ Metadata-Version: 2.4
2
+ Name: wiretruth
3
+ Version: 0.2.0
4
+ Summary: Replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually happened.
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/kartsan03/wiretruth
7
+ Project-URL: Repository, https://github.com/kartsan03/wiretruth
8
+ Project-URL: Issues, https://github.com/kartsan03/wiretruth/issues
9
+ Project-URL: Changelog, https://github.com/kartsan03/wiretruth/blob/main/CHANGELOG.md
10
+ Keywords: llm,sdk,conformance,test-suite,openai,anthropic,gemini,ollama,testing
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Topic :: Software Development :: Testing
14
+ Classifier: Topic :: Software Development :: Quality Assurance
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Requires-Python: >=3.11
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: jsonschema>=4.21
22
+ Provides-Extra: dev
23
+ Requires-Dist: pytest>=8; extra == "dev"
24
+ Requires-Dist: pytest-cov>=5; extra == "dev"
25
+ Requires-Dist: ruff>=0.6; extra == "dev"
26
+ Requires-Dist: mypy>=1.11; extra == "dev"
27
+ Requires-Dist: types-jsonschema; extra == "dev"
28
+ Dynamic: license-file
29
+
30
+ # wiretruth
31
+
32
+ wiretruth replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually
33
+ happened.
34
+
35
+ **Does your LLM SDK tell you the truth?**
36
+
37
+ [![CI](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml/badge.svg)](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml)
38
+ [![PyPI](https://img.shields.io/pypi/v/wiretruth?label=PyPI)](https://pypi.org/project/wiretruth/)
39
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/kartsan03/wiretruth/blob/main/LICENSE)
40
+
41
+ So far wiretruth tests one library, any-llm, on six APIs: OpenAI Chat Completions and Responses, Ollama's native
42
+ and OpenAI-compatible endpoints, Gemini GenerateContent and Anthropic Messages. That is 24 scenarios and 95 cells.
43
+ The Anthropic responses are transcribed from Anthropic's documentation rather than recorded: each of those fixtures
44
+ is marked `documented` and links the page it comes from. More providers and libraries follow, and a public
45
+ results matrix arrives in v0.4. Until then, the grades are in
46
+ [`results/`](https://github.com/kartsan03/wiretruth/tree/main/results).
47
+
48
+ ## Why
49
+
50
+ A model hits its output token limit halfway through a JSON object. The provider says so plainly:
51
+ `"finish_reason": "length"`. What your code sees depends on the SDK in between. If the SDK raises a generic JSON
52
+ parse error, nothing tells you that a higher limit would fix it. If it returns the partial object as a success,
53
+ your pipeline stores an incomplete record.
54
+
55
+ wiretruth has this case recorded as `truncation.structured-output` on `openai-chat`. Replayed through any-llm:
56
+
57
+ | Library | What the caller gets | Grade |
58
+ |---|---|---|
59
+ | any-llm 1.33.0 | Raises `openai.LengthFinishReasonError`. Its `completion` has `finish_reason: "length"` and the token usage. | ✓ PASS |
60
+
61
+ wiretruth runs the same recorded edge cases through every library it supports and grades each one against what the
62
+ provider sent.
63
+
64
+ ## What it checks
65
+
66
+ | Bug class | What goes wrong for users | Scenarios |
67
+ |---|---|---|
68
+ | Silent truncation | Agents accept partial output or retry without raising the limit | 6 |
69
+ | Finish reason mapping | Control flow branches on the wrong reason | 2 |
70
+ | Usage accounting | Cost and quota tracking is off (cached and reasoning tokens) | 3 |
71
+ | Stream integrity | Corrupted or unfinished text is treated as final | 4 |
72
+ | Tool call fidelity | Tools run with broken or merged arguments | 4 |
73
+ | Error classification | Retries and backoff misbehave | 1 |
74
+ | Request fidelity | The library sends fields the caller didn't set, or ones the model rejects | 4 |
75
+
76
+ The scenarios are in
77
+ [`suite/scenarios/`](https://github.com/kartsan03/wiretruth/tree/main/suite/scenarios), and the bug classes in
78
+ [`suite/bug-classes.json`](https://github.com/kartsan03/wiretruth/blob/main/suite/bug-classes.json).
79
+
80
+ ## Grades
81
+
82
+ | | Grade | Meaning |
83
+ |---|---|---|
84
+ | ✓ | PASS | Reports what happened: meets every MUST requirement of the scenario. |
85
+ | ~ | LOSSY | Doesn't contradict what happened, but the fact is only in raw metadata, or missing. |
86
+ | ✗ | WRONG | Reports something that contradicts what happened. |
87
+ | ! | CRASH | Fails with an internal error, such as a `KeyError` in its own parser. |
88
+ | ⧗ | HANG | Doesn't return within 20 s on a response that ended within 1 s. |
89
+ | – | UNSUPPORTED | The library doesn't support this provider or feature (documented). |
90
+ | ? | HARNESS | A problem on our side, not the library's. |
91
+
92
+ Each scenario lists its requirements. MUST requirements set the grade. SHOULD requirements, such as the exact
93
+ partial text in the truncation scenarios, add a note when they aren't met. How grades are computed:
94
+ [docs/grading.md](https://github.com/kartsan03/wiretruth/blob/main/docs/grading.md).
95
+ Scenario catalog: [docs/scenarios.md](https://github.com/kartsan03/wiretruth/blob/main/docs/scenarios.md).
96
+
97
+ ## Quickstart
98
+
99
+ You need Python 3.11 or newer and [uv](https://docs.astral.sh/uv/), which builds each library's own environment.
100
+
101
+ ```bash
102
+ git clone https://github.com/kartsan03/wiretruth && cd wiretruth
103
+ uv run wiretruth run --lib any-llm
104
+ ```
105
+
106
+ ```text
107
+ scenario anthropic gemini ollama ollama-openai openai-chat openai-responses
108
+ finish.stop-sequence ✓ PASS ✓ PASS ✓ PASS ✓ PASS
109
+ finish.unknown-value ✗ WRONG ✗ WRONG ✗ WRONG ✗ WRONG ✗ WRONG ✓ PASS
110
+ request.max-tokens-field ✓ PASS
111
+ request.max-tokens-ollama-openai ✗ WRONG
112
+ request.no-injected-options ✓ PASS
113
+ request.tools-with-reasoning-chat ✓ PASS
114
+ stream.error-midstream ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
115
+ stream.missing-finish ~ LOSSY ~ LOSSY ~ LOSSY ~ LOSSY ~ LOSSY ✓ PASS
116
+ stream.missing-finish-abort ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
117
+ stream.utf8 ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
118
+ stream.utf8-split ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
119
+ tools.parallel-calls ✓ PASS ✗ WRONG ✗ WRONG ✓ PASS ✓ PASS ✓ PASS
120
+ tools.parallel-calls-stream ✓ PASS ✗ WRONG ✗ WRONG ✓ PASS ✓ PASS ✓ PASS
121
+ truncation.reasoning-only ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
122
+ truncation.structured-output ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ~ LOSSY
123
+ truncation.structured-output-forced-tool – UNSUPPORTED
124
+ truncation.text ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
125
+ truncation.text-stream ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
126
+ truncation.text-stream-usage ✓ PASS ✓ PASS
127
+ truncation.tool-call-args ✓ PASS ✓ PASS ✓ PASS
128
+ truncation.tool-call-args-stream ✓ PASS ✓ PASS ✓ PASS
129
+ usage.cached-tokens ✓ PASS
130
+ usage.reasoning-tokens ✓ PASS
131
+ usage.stream-final-chunk ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
132
+
133
+ finish.stop-sequence gemini
134
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
135
+
136
+ finish.unknown-value anthropic
137
+ MUST finish-is-other: CONTRADICTED: finish.value="stop" (primary: choices[0].finish_reason); truth.finish="other"; required finish="other"
138
+ SHOULD raw-finish-reported: CONTRADICTED: finish.raw="stop"; truth.raw_finish="WIRETRUTH_UNKNOWN_REASON"
139
+
140
+ finish.unknown-value gemini
141
+ MUST finish-is-other: CONTRADICTED: finish.value="stop" (primary: choices[0].finish_reason); truth.finish="other"; required finish="other"
142
+ SHOULD raw-finish-reported: CONTRADICTED: finish.raw="stop"; truth.raw_finish="WIRETRUTH_UNKNOWN_REASON"
143
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
144
+
145
+ finish.unknown-value ollama
146
+ MUST finish-is-other: CONTRADICTED: finish.value="stop" (primary: choices[0].finish_reason); truth.finish="other"; required finish="other"
147
+ SHOULD raw-finish-reported: CONTRADICTED: finish.raw="stop"; truth.raw_finish="wiretruth_unknown_reason"
148
+
149
+ finish.unknown-value ollama-openai
150
+ MUST response-returned: CONTRADICTED: outcome=raised; error.category="validation"; truth.error=null
151
+ MUST finish-is-other: NOT_OBSERVABLE: finish.value=null (absent); truth.finish="other"; required finish="other"
152
+ MUST text-reported: NOT_OBSERVABLE: text.value=null (absent); truth.text="Octopuses, like many marine invertebrates, employ sophisticated methods to camouflage"
153
+ SHOULD raw-finish-reported: NOT_OBSERVABLE: finish.raw=null; truth.raw_finish="wiretruth_unknown_reason"
154
+
155
+ finish.unknown-value openai-chat
156
+ MUST response-returned: CONTRADICTED: outcome=raised; error.category="validation"; truth.error=null
157
+ MUST finish-is-other: NOT_OBSERVABLE: finish.value=null (absent); truth.finish="other"; required finish="other"
158
+ MUST text-reported: NOT_OBSERVABLE: text.value=null (absent); truth.text="Io, Europa, and Ganymede."
159
+ SHOULD raw-finish-reported: NOT_OBSERVABLE: finish.raw=null; truth.raw_finish="wiretruth_unknown_reason"
160
+
161
+ request.max-tokens-ollama-openai ollama-openai
162
+ MUST limit-field: CONTRADICTED: request[0].json_body.max_tokens=absent; required=16
163
+
164
+ stream.error-midstream gemini
165
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
166
+
167
+ stream.missing-finish anthropic
168
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
169
+
170
+ stream.missing-finish gemini
171
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
172
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
173
+
174
+ stream.missing-finish ollama
175
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
176
+
177
+ stream.missing-finish ollama-openai
178
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
179
+
180
+ stream.missing-finish openai-chat
181
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
182
+
183
+ stream.missing-finish-abort gemini
184
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
185
+
186
+ stream.utf8 gemini
187
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
188
+
189
+ stream.utf8-split gemini
190
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
191
+
192
+ tools.parallel-calls gemini
193
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="call_-5201471136765337120_0", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_120847", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="call_-5201471136765337120_1", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_120848", name="get_weather", args={"city": "Zürich"}, parsed=true
194
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
195
+
196
+ tools.parallel-calls ollama
197
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="3972c760-858b-4728-a05a-279fc6e6a649", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_9epwsqu6", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="4d3747a6-5e75-4637-b82e-f269adbba01e", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_bvzy1ra9", name="get_weather", args={"city": "Zürich"}, parsed=true
198
+
199
+ tools.parallel-calls-stream gemini
200
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="call_-1615951408272134819_0", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_89893", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="call_-1615951408272134819_1", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_89896", name="get_weather", args={"city": "Zürich"}, parsed=true
201
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
202
+
203
+ tools.parallel-calls-stream ollama
204
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="8b75c529-6eb6-4a63-a375-4551bfe56f39", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_8uaqob2r", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="8e864d64-6145-4ca4-81ba-cbdbad0523e1", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_nyjtbrz4", name="get_weather", args={"city": "Zürich"}, parsed=true
205
+
206
+ truncation.structured-output gemini
207
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
208
+
209
+ truncation.structured-output openai-responses
210
+ MUST cause-is-truncation: NOT_OBSERVABLE: finish.value=null (absent); truth.finish="length"; error.category="validation"; structured.returned_as_success=false
211
+ SHOULD text-reported: NOT_OBSERVABLE: text.value=null (absent); truth.text="{\"islands\":[{\"name\":\"Κρήτη\",\"sea\":\"Κρητικό Πέ"
212
+ SHOULD usage-reported: NOT_OBSERVABLE: usage.input=null (absent); truth.usage.input=68; usage.output=null (absent); truth.usage.output=24
213
+
214
+ truncation.structured-output-forced-tool anthropic
215
+ any-llm completion maps typed response_format to native output_config.format and has no structured parser for an explicitly forced tool. (https://github.com/mozilla-ai/any-llm/blob/1.33.0/src/any_llm/providers/anthropic/utils.py#L628-L655)
216
+
217
+ truncation.text gemini
218
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
219
+
220
+ truncation.text-stream gemini
221
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
222
+
223
+ truncation.text-stream-usage gemini
224
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
225
+
226
+ usage.stream-final-chunk gemini
227
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
228
+
229
+ any-llm 1.33.0: 95 cells, 78 PASS, 6 LOSSY, 10 WRONG, 1 UNSUPPORTED
230
+ stderr: .wiretruth/runs/20261013T134918.993664Z
231
+ ```
232
+
233
+ No API keys are needed. Everything runs against stored responses on `127.0.0.1`. The first run downloads any-llm
234
+ into `.wiretruth/envs/`; later runs work offline. `uv run wiretruth run --shim shims/reference` runs the
235
+ built-in reference client, which must pass every cell. The scenarios, fixtures and shims live in this repository,
236
+ so wiretruth runs from a clone.
237
+
238
+ Releases are also on [PyPI](https://pypi.org/project/wiretruth/) (`pip install wiretruth`). The package is the
239
+ command only, so run it inside a checkout of the matching release tag (for example `v0.2.0`); on `main`, use
240
+ `uv run` as above.
241
+
242
+ ## Add your library
243
+
244
+ A shim is a small program. It reads a call spec as JSON on stdin, calls your library the way its docs show, and
245
+ prints what the library reported as JSON on stdout. The
246
+ [any-llm shim](https://github.com/kartsan03/wiretruth/blob/main/shims/python/any-llm/shim.py) is 150 lines. The
247
+ protocol is in [shims/README.md](https://github.com/kartsan03/wiretruth/blob/main/shims/README.md); to propose a
248
+ library, [open a new-library issue](https://github.com/kartsan03/wiretruth/issues/new?template=new-library.yml).
249
+
250
+ ## Related work
251
+
252
+ - [Open Responses](https://github.com/openresponses/openresponses): a spec and compliance tests for servers that
253
+ implement the Responses API. wiretruth tests clients across several native APIs.
254
+ - [LLMConform](https://github.com/aitk-org/LLMConform): conformance tests for LLM gateways and APIs, run live.
255
+ - [llmprobe](https://github.com/ivanfioravanti/llmprobe) and [conform](https://github.com/NiLabs-Org/conform):
256
+ conformance for inference engines (vLLM, llama.cpp, Ollama, …).
257
+ - [aimock](https://github.com/CopilotKit/aimock): a mock server for testing AI apps, with record and replay. Use it
258
+ for your app's tests; wiretruth audits the SDK layer itself.
259
+ - [spantruth](https://github.com/Plexito-de/spantruth): a separate project, called wiretruth until 2026-10-12.
260
+ It replays provider responses through LLM instrumentation libraries (OpenLLMetry, OpenInference, the OpenTelemetry
261
+ GenAI instrumentation) and checks what they write on spans. Use it for your traces; wiretruth checks what the SDK
262
+ returns to the caller.
263
+ - The design borrows from [JSONTestSuite](https://github.com/nst/JSONTestSuite),
264
+ [Wycheproof](https://github.com/C2SP/wycheproof), [toml-test](https://github.com/toml-lang/toml-test) and
265
+ [compat-table](https://github.com/compat-table/compat-table).
266
+
267
+ ## Author note
268
+
269
+ I've contributed fixes to libraries that wiretruth tests or will test:
270
+ [mozilla-ai/any-llm#1389](https://github.com/mozilla-ai/any-llm/pull/1389) and
271
+ [TanStack/ai#1548](https://github.com/TanStack/ai/pull/1548). Grades are computed by the code in this repository.
272
+ If you think a grade is wrong, please
273
+ [open a grade dispute](https://github.com/kartsan03/wiretruth/issues/new?template=grade-dispute.yml).
274
+
275
+ ## Contributing
276
+
277
+ Scenario ideas, new libraries and grade disputes are all welcome. See
278
+ [CONTRIBUTING.md](https://github.com/kartsan03/wiretruth/blob/main/CONTRIBUTING.md).
279
+
280
+ ## Citing
281
+
282
+ See [CITATION.cff](https://github.com/kartsan03/wiretruth/blob/main/CITATION.cff).
283
+
284
+ ## License
285
+
286
+ MIT. Fixtures contain model outputs recorded from provider APIs;
287
+ [docs/recording.md](https://github.com/kartsan03/wiretruth/blob/main/docs/recording.md) explains how they're recorded.
wiretruth-0.2.0/README.md ADDED
@@ -0,0 +1,258 @@
1
+ # wiretruth
2
+
3
+ wiretruth replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually
4
+ happened.
5
+
6
+ **Does your LLM SDK tell you the truth?**
7
+
8
+ [![CI](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml/badge.svg)](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml)
9
+ [![PyPI](https://img.shields.io/pypi/v/wiretruth?label=PyPI)](https://pypi.org/project/wiretruth/)
10
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/kartsan03/wiretruth/blob/main/LICENSE)
11
+
12
+ So far wiretruth tests one library, any-llm, on six APIs: OpenAI Chat Completions and Responses, Ollama's native
13
+ and OpenAI-compatible endpoints, Gemini GenerateContent and Anthropic Messages. That is 24 scenarios and 95 cells.
14
+ The Anthropic responses are transcribed from Anthropic's documentation rather than recorded: each of those fixtures
15
+ is marked `documented` and links the page it comes from. More providers and libraries follow, and a public
16
+ results matrix arrives in v0.4. Until then, the grades are in
17
+ [`results/`](https://github.com/kartsan03/wiretruth/tree/main/results).
18
+
19
+ ## Why
20
+
21
+ A model hits its output token limit halfway through a JSON object. The provider says so plainly:
22
+ `"finish_reason": "length"`. What your code sees depends on the SDK in between. If the SDK raises a generic JSON
23
+ parse error, nothing tells you that a higher limit would fix it. If it returns the partial object as a success,
24
+ your pipeline stores an incomplete record.
25
+
26
+ wiretruth has this case recorded as `truncation.structured-output` on `openai-chat`. Replayed through any-llm:
27
+
28
+ | Library | What the caller gets | Grade |
29
+ |---|---|---|
30
+ | any-llm 1.33.0 | Raises `openai.LengthFinishReasonError`. Its `completion` has `finish_reason: "length"` and the token usage. | ✓ PASS |
31
+
32
+ wiretruth runs the same recorded edge cases through every library it supports and grades each one against what the
33
+ provider sent.
34
+
35
+ ## What it checks
36
+
37
+ | Bug class | What goes wrong for users | Scenarios |
38
+ |---|---|---|
39
+ | Silent truncation | Agents accept partial output or retry without raising the limit | 6 |
40
+ | Finish reason mapping | Control flow branches on the wrong reason | 2 |
41
+ | Usage accounting | Cost and quota tracking is off (cached and reasoning tokens) | 3 |
42
+ | Stream integrity | Corrupted or unfinished text is treated as final | 4 |
43
+ | Tool call fidelity | Tools run with broken or merged arguments | 4 |
44
+ | Error classification | Retries and backoff misbehave | 1 |
45
+ | Request fidelity | The library sends fields the caller didn't set, or ones the model rejects | 4 |
46
+
47
+ The scenarios are in
48
+ [`suite/scenarios/`](https://github.com/kartsan03/wiretruth/tree/main/suite/scenarios), and the bug classes in
49
+ [`suite/bug-classes.json`](https://github.com/kartsan03/wiretruth/blob/main/suite/bug-classes.json).
50
+
51
+ ## Grades
52
+
53
+ | | Grade | Meaning |
54
+ |---|---|---|
55
+ | ✓ | PASS | Reports what happened: meets every MUST requirement of the scenario. |
56
+ | ~ | LOSSY | Doesn't contradict what happened, but the fact is only in raw metadata, or missing. |
57
+ | ✗ | WRONG | Reports something that contradicts what happened. |
58
+ | ! | CRASH | Fails with an internal error, such as a `KeyError` in its own parser. |
59
+ | ⧗ | HANG | Doesn't return within 20 s on a response that ended within 1 s. |
60
+ | – | UNSUPPORTED | The library doesn't support this provider or feature (documented). |
61
+ | ? | HARNESS | A problem on our side, not the library's. |
62
+
63
+ Each scenario lists its requirements. MUST requirements set the grade. SHOULD requirements, such as the exact
64
+ partial text in the truncation scenarios, add a note when they aren't met. How grades are computed:
65
+ [docs/grading.md](https://github.com/kartsan03/wiretruth/blob/main/docs/grading.md).
66
+ Scenario catalog: [docs/scenarios.md](https://github.com/kartsan03/wiretruth/blob/main/docs/scenarios.md).
67
+
68
+ ## Quickstart
69
+
70
+ You need Python 3.11 or newer and [uv](https://docs.astral.sh/uv/), which builds each library's own environment.
71
+
72
+ ```bash
73
+ git clone https://github.com/kartsan03/wiretruth && cd wiretruth
74
+ uv run wiretruth run --lib any-llm
75
+ ```
76
+
77
+ ```text
78
+ scenario anthropic gemini ollama ollama-openai openai-chat openai-responses
79
+ finish.stop-sequence ✓ PASS ✓ PASS ✓ PASS ✓ PASS
80
+ finish.unknown-value ✗ WRONG ✗ WRONG ✗ WRONG ✗ WRONG ✗ WRONG ✓ PASS
81
+ request.max-tokens-field ✓ PASS
82
+ request.max-tokens-ollama-openai ✗ WRONG
83
+ request.no-injected-options ✓ PASS
84
+ request.tools-with-reasoning-chat ✓ PASS
85
+ stream.error-midstream ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
86
+ stream.missing-finish ~ LOSSY ~ LOSSY ~ LOSSY ~ LOSSY ~ LOSSY ✓ PASS
87
+ stream.missing-finish-abort ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
88
+ stream.utf8 ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
89
+ stream.utf8-split ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
90
+ tools.parallel-calls ✓ PASS ✗ WRONG ✗ WRONG ✓ PASS ✓ PASS ✓ PASS
91
+ tools.parallel-calls-stream ✓ PASS ✗ WRONG ✗ WRONG ✓ PASS ✓ PASS ✓ PASS
92
+ truncation.reasoning-only ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
93
+ truncation.structured-output ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ~ LOSSY
94
+ truncation.structured-output-forced-tool – UNSUPPORTED
95
+ truncation.text ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
96
+ truncation.text-stream ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
97
+ truncation.text-stream-usage ✓ PASS ✓ PASS
98
+ truncation.tool-call-args ✓ PASS ✓ PASS ✓ PASS
99
+ truncation.tool-call-args-stream ✓ PASS ✓ PASS ✓ PASS
100
+ usage.cached-tokens ✓ PASS
101
+ usage.reasoning-tokens ✓ PASS
102
+ usage.stream-final-chunk ✓ PASS ✓ PASS ✓ PASS ✓ PASS ✓ PASS
103
+
104
+ finish.stop-sequence gemini
105
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
106
+
107
+ finish.unknown-value anthropic
108
+ MUST finish-is-other: CONTRADICTED: finish.value="stop" (primary: choices[0].finish_reason); truth.finish="other"; required finish="other"
109
+ SHOULD raw-finish-reported: CONTRADICTED: finish.raw="stop"; truth.raw_finish="WIRETRUTH_UNKNOWN_REASON"
110
+
111
+ finish.unknown-value gemini
112
+ MUST finish-is-other: CONTRADICTED: finish.value="stop" (primary: choices[0].finish_reason); truth.finish="other"; required finish="other"
113
+ SHOULD raw-finish-reported: CONTRADICTED: finish.raw="stop"; truth.raw_finish="WIRETRUTH_UNKNOWN_REASON"
114
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
115
+
116
+ finish.unknown-value ollama
117
+ MUST finish-is-other: CONTRADICTED: finish.value="stop" (primary: choices[0].finish_reason); truth.finish="other"; required finish="other"
118
+ SHOULD raw-finish-reported: CONTRADICTED: finish.raw="stop"; truth.raw_finish="wiretruth_unknown_reason"
119
+
120
+ finish.unknown-value ollama-openai
121
+ MUST response-returned: CONTRADICTED: outcome=raised; error.category="validation"; truth.error=null
122
+ MUST finish-is-other: NOT_OBSERVABLE: finish.value=null (absent); truth.finish="other"; required finish="other"
123
+ MUST text-reported: NOT_OBSERVABLE: text.value=null (absent); truth.text="Octopuses, like many marine invertebrates, employ sophisticated methods to camouflage"
124
+ SHOULD raw-finish-reported: NOT_OBSERVABLE: finish.raw=null; truth.raw_finish="wiretruth_unknown_reason"
125
+
126
+ finish.unknown-value openai-chat
127
+ MUST response-returned: CONTRADICTED: outcome=raised; error.category="validation"; truth.error=null
128
+ MUST finish-is-other: NOT_OBSERVABLE: finish.value=null (absent); truth.finish="other"; required finish="other"
129
+ MUST text-reported: NOT_OBSERVABLE: text.value=null (absent); truth.text="Io, Europa, and Ganymede."
130
+ SHOULD raw-finish-reported: NOT_OBSERVABLE: finish.raw=null; truth.raw_finish="wiretruth_unknown_reason"
131
+
132
+ request.max-tokens-ollama-openai ollama-openai
133
+ MUST limit-field: CONTRADICTED: request[0].json_body.max_tokens=absent; required=16
134
+
135
+ stream.error-midstream gemini
136
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
137
+
138
+ stream.missing-finish anthropic
139
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
140
+
141
+ stream.missing-finish gemini
142
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
143
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
144
+
145
+ stream.missing-finish ollama
146
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
147
+
148
+ stream.missing-finish ollama-openai
149
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
150
+
151
+ stream.missing-finish openai-chat
152
+ MUST stream-incomplete: NOT_OBSERVABLE: stream.completed=null; outcome=returned; truth.stream_completed=false
153
+
154
+ stream.missing-finish-abort gemini
155
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
156
+
157
+ stream.utf8 gemini
158
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
159
+
160
+ stream.utf8-split gemini
161
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
162
+
163
+ tools.parallel-calls gemini
164
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="call_-5201471136765337120_0", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_120847", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="call_-5201471136765337120_1", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_120848", name="get_weather", args={"city": "Zürich"}, parsed=true
165
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
166
+
167
+ tools.parallel-calls ollama
168
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="3972c760-858b-4728-a05a-279fc6e6a649", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_9epwsqu6", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="4d3747a6-5e75-4637-b82e-f269adbba01e", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_bvzy1ra9", name="get_weather", args={"city": "Zürich"}, parsed=true
169
+
170
+ tools.parallel-calls-stream gemini
171
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="call_-1615951408272134819_0", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_89893", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="call_-1615951408272134819_1", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_89896", name="get_weather", args={"city": "Zürich"}, parsed=true
172
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
173
+
174
+ tools.parallel-calls-stream ollama
175
+ MUST calls-reported: CONTRADICTED: tool_calls[0] id="8b75c529-6eb6-4a63-a375-4551bfe56f39", name="get_weather", args={"city": "São Paulo"}, parsed=true; truth id="call_8uaqob2r", name="get_weather", args={"city": "São Paulo"}, parsed=true; tool_calls[1] id="8e864d64-6145-4ca4-81ba-cbdbad0523e1", name="get_weather", args={"city": "Zürich"}, parsed=true; truth id="call_nyjtbrz4", name="get_weather", args={"city": "Zürich"}, parsed=true
176
+
177
+ truncation.structured-output gemini
178
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
179
+
180
+ truncation.structured-output openai-responses
181
+ MUST cause-is-truncation: NOT_OBSERVABLE: finish.value=null (absent); truth.finish="length"; error.category="validation"; structured.returned_as_success=false
182
+ SHOULD text-reported: NOT_OBSERVABLE: text.value=null (absent); truth.text="{\"islands\":[{\"name\":\"Κρήτη\",\"sea\":\"Κρητικό Πέ"
183
+ SHOULD usage-reported: NOT_OBSERVABLE: usage.input=null (absent); truth.usage.input=68; usage.output=null (absent); truth.usage.output=24
184
+
185
+ truncation.structured-output-forced-tool anthropic
186
+ any-llm completion maps typed response_format to native output_config.format and has no structured parser for an explicitly forced tool. (https://github.com/mozilla-ai/any-llm/blob/1.33.0/src/any_llm/providers/anthropic/utils.py#L628-L655)
187
+
188
+ truncation.text gemini
189
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
190
+
191
+ truncation.text-stream gemini
192
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
193
+
194
+ truncation.text-stream-usage gemini
195
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
196
+
197
+ usage.stream-final-chunk gemini
198
+ shim: Gemini reasoning_effort none is unsupported by any-llm; omitted to use the model default.
199
+
200
+ any-llm 1.33.0: 95 cells, 78 PASS, 6 LOSSY, 10 WRONG, 1 UNSUPPORTED
201
+ stderr: .wiretruth/runs/20261013T134918.993664Z
202
+ ```
203
+
204
+ No API keys are needed. Everything runs against stored responses on `127.0.0.1`. The first run downloads any-llm
205
+ into `.wiretruth/envs/`; later runs work offline. `uv run wiretruth run --shim shims/reference` runs the
206
+ built-in reference client, which must pass every cell. The scenarios, fixtures and shims live in this repository,
207
+ so wiretruth runs from a clone.
208
+
209
+ Releases are also on [PyPI](https://pypi.org/project/wiretruth/) (`pip install wiretruth`). The package is the
210
+ command only, so run it inside a checkout of the matching release tag (for example `v0.2.0`); on `main`, use
211
+ `uv run` as above.
212
+
213
+ ## Add your library
214
+
215
+ A shim is a small program. It reads a call spec as JSON on stdin, calls your library the way its docs show, and
216
+ prints what the library reported as JSON on stdout. The
217
+ [any-llm shim](https://github.com/kartsan03/wiretruth/blob/main/shims/python/any-llm/shim.py) is 150 lines. The
218
+ protocol is in [shims/README.md](https://github.com/kartsan03/wiretruth/blob/main/shims/README.md); to propose a
219
+ library, [open a new-library issue](https://github.com/kartsan03/wiretruth/issues/new?template=new-library.yml).
220
+
221
+ ## Related work
222
+
223
+ - [Open Responses](https://github.com/openresponses/openresponses): a spec and compliance tests for servers that
224
+ implement the Responses API. wiretruth tests clients across several native APIs.
225
+ - [LLMConform](https://github.com/aitk-org/LLMConform): conformance tests for LLM gateways and APIs, run live.
226
+ - [llmprobe](https://github.com/ivanfioravanti/llmprobe) and [conform](https://github.com/NiLabs-Org/conform):
227
+ conformance for inference engines (vLLM, llama.cpp, Ollama, …).
228
+ - [aimock](https://github.com/CopilotKit/aimock): a mock server for testing AI apps, with record and replay. Use it
229
+ for your app's tests; wiretruth audits the SDK layer itself.
230
+ - [spantruth](https://github.com/Plexito-de/spantruth): a separate project, called wiretruth until 2026-10-12.
231
+ It replays provider responses through LLM instrumentation libraries (OpenLLMetry, OpenInference, the OpenTelemetry
232
+ GenAI instrumentation) and checks what they write on spans. Use it for your traces; wiretruth checks what the SDK
233
+ returns to the caller.
234
+ - The design borrows from [JSONTestSuite](https://github.com/nst/JSONTestSuite),
235
+ [Wycheproof](https://github.com/C2SP/wycheproof), [toml-test](https://github.com/toml-lang/toml-test) and
236
+ [compat-table](https://github.com/compat-table/compat-table).
237
+
238
+ ## Author note
239
+
240
+ I've contributed fixes to libraries that wiretruth tests or will test:
241
+ [mozilla-ai/any-llm#1389](https://github.com/mozilla-ai/any-llm/pull/1389) and
242
+ [TanStack/ai#1548](https://github.com/TanStack/ai/pull/1548). Grades are computed by the code in this repository.
243
+ If you think a grade is wrong, please
244
+ [open a grade dispute](https://github.com/kartsan03/wiretruth/issues/new?template=grade-dispute.yml).
245
+
246
+ ## Contributing
247
+
248
+ Scenario ideas, new libraries and grade disputes are all welcome. See
249
+ [CONTRIBUTING.md](https://github.com/kartsan03/wiretruth/blob/main/CONTRIBUTING.md).
250
+
251
+ ## Citing
252
+
253
+ See [CITATION.cff](https://github.com/kartsan03/wiretruth/blob/main/CITATION.cff).
254
+
255
+ ## License
256
+
257
+ MIT. Fixtures contain model outputs recorded from provider APIs;
258
+ [docs/recording.md](https://github.com/kartsan03/wiretruth/blob/main/docs/recording.md) explains how they're recorded.
wiretruth-0.2.0/pyproject.toml ADDED
@@ -0,0 +1,57 @@
1
+ [project]
2
+ name = "wiretruth"
3
+ dynamic = ["version"]
4
+ description = "Replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually happened."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ requires-python = ">=3.11"
8
+ # 4.18 added the `referencing` registry used for $ref between the bundled schemas; the tests
9
+ # pass from 4.21 on (CI runs them on 4.21.0), where some error messages changed wording.
10
+ dependencies = ["jsonschema>=4.21"]
11
+ keywords = ["llm", "sdk", "conformance", "test-suite", "openai", "anthropic", "gemini", "ollama", "testing"]
12
+ classifiers = [
13
+ "Development Status :: 3 - Alpha",
14
+ "Intended Audience :: Developers",
15
+ "Topic :: Software Development :: Testing",
16
+ "Topic :: Software Development :: Quality Assurance",
17
+ "Programming Language :: Python :: 3.11",
18
+ "Programming Language :: Python :: 3.12",
19
+ "Programming Language :: Python :: 3.13",
20
+ ]
21
+
22
+ [project.optional-dependencies]
23
+ dev = ["pytest>=8", "pytest-cov>=5", "ruff>=0.6", "mypy>=1.11", "types-jsonschema"]
24
+
25
+ [project.scripts]
26
+ wiretruth = "wiretruth.cli:main"
27
+
28
+ [project.urls]
29
+ Homepage = "https://github.com/kartsan03/wiretruth"
30
+ Repository = "https://github.com/kartsan03/wiretruth"
31
+ Issues = "https://github.com/kartsan03/wiretruth/issues"
32
+ Changelog = "https://github.com/kartsan03/wiretruth/blob/main/CHANGELOG.md"
33
+
34
+ [build-system]
35
+ requires = ["setuptools>=77"]
36
+ build-backend = "setuptools.build_meta"
37
+
38
+ [tool.setuptools.dynamic]
39
+ version = { attr = "wiretruth.__version__" }
40
+
41
+ [tool.setuptools.package-data]
42
+ wiretruth = ["schemas/*.json"]
43
+
44
+ [tool.ruff]
45
+ line-length = 100
46
+ target-version = "py311"
47
+
48
+ [tool.ruff.lint]
49
+ select = ["E", "F", "I", "B", "UP", "SIM", "RUF"]
50
+
51
+ [tool.mypy]
52
+ strict = true
53
+ python_version = "3.11"
54
+ files = ["src"]
55
+
56
+ [tool.pytest.ini_options]
57
+ testpaths = ["tests"]
wiretruth-0.2.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
wiretruth-0.2.0/src/wiretruth/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """wiretruth: replay real LLM provider responses through LLM SDKs and grade what they report."""
2
+
3
+ __version__ = "0.2.0"
wiretruth-0.2.0/src/wiretruth/__main__.py ADDED
@@ -0,0 +1,4 @@
1
+ from wiretruth.cli import main
2
+
3
+ if __name__ == "__main__":
4
+ raise SystemExit(main())