wiretruth 0.1.0__tar.gz

wiretruth-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 kartsan03
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wiretruth-0.1.0/PKG-INFO

@@ -0,0 +1,158 @@
+Metadata-Version: 2.4
+Name: wiretruth
+Version: 0.1.0
+Summary: Replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually happened.
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/kartsan03/wiretruth
+Project-URL: Repository, https://github.com/kartsan03/wiretruth
+Project-URL: Issues, https://github.com/kartsan03/wiretruth/issues
+Project-URL: Changelog, https://github.com/kartsan03/wiretruth/blob/main/CHANGELOG.md
+Keywords: llm,sdk,conformance,test-suite,openai,anthropic,gemini,ollama,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: jsonschema>=4.21
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-cov>=5; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Requires-Dist: mypy>=1.11; extra == "dev"
+Requires-Dist: types-jsonschema; extra == "dev"
+Dynamic: license-file
+
+# wiretruth
+
+wiretruth replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually
+happened.
+
+**Does your LLM SDK tell you the truth?**
+
+[![CI](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml/badge.svg)](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/wiretruth)](https://pypi.org/project/wiretruth/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/kartsan03/wiretruth/blob/main/LICENSE)
+
+Version 0.1 tests one library, any-llm, on the OpenAI Chat Completions API with three truncation scenarios. More
+providers and libraries follow, and a public results matrix arrives in v0.4. Until then, the grades are in
+[`results/`](https://github.com/kartsan03/wiretruth/tree/main/results).
+
+## Why
+
+A model hits its output token limit halfway through a JSON object. The provider says so plainly:
+`"finish_reason": "length"`. What your code sees depends on the SDK in between. If the SDK raises a generic JSON
+parse error, nothing tells you that a higher limit would fix it. If it returns the partial object as a success,
+your pipeline stores an incomplete record.
+
+wiretruth has this case recorded as `truncation.structured-output` on `openai-chat`. Replayed through any-llm:
+
+| Library | What the caller gets | Grade |
+|---|---|---|
+| any-llm 1.33.0 | Raises `openai.LengthFinishReasonError`. Its `completion` has `finish_reason: "length"` and the token usage. | ✓ PASS |
+
+wiretruth runs the same recorded edge cases through every library it supports and grades each one against what the
+provider sent.
+
+## What it checks
+
+| Bug class | What goes wrong for users | Scenarios |
+|---|---|---|
+| Silent truncation | Agents accept partial output or retry without raising the limit | 3 |
+| Finish reason mapping | Control flow branches on the wrong reason | planned |
+| Usage accounting | Cost and quota tracking is off (cached and reasoning tokens) | planned |
+| Stream integrity | Corrupted or unfinished text is treated as final | planned |
+| Tool call fidelity | Tools run with broken or merged arguments | planned |
+| Error classification | Retries and backoff misbehave | planned |
+| Request fidelity | The library sends options the caller didn't set | planned |
+
+The scenarios are in
+[`suite/scenarios/`](https://github.com/kartsan03/wiretruth/tree/main/suite/scenarios), and the bug classes in
+[`suite/bug-classes.json`](https://github.com/kartsan03/wiretruth/blob/main/suite/bug-classes.json).
+
+## Grades
+
+| | Grade | Meaning |
+|---|---|---|
+| ✓ | PASS | Reports what happened: meets every MUST requirement of the scenario. |
+| ~ | LOSSY | Doesn't contradict what happened, but the fact is only in raw metadata, or missing. |
+| ✗ | WRONG | Reports something that contradicts what happened. |
+| ! | CRASH | Fails with an internal error, such as a `KeyError` in its own parser. |
+| ⧗ | HANG | Doesn't return within 20 s on a response that ended within 1 s. |
+| – | UNSUPPORTED | The library doesn't support this provider or feature (documented). |
+| ? | HARNESS | A problem on our side, not the library's. |
+
+Each scenario lists its requirements. MUST requirements set the grade. SHOULD requirements, such as the exact
+partial text in the truncation scenarios, add a note when they aren't met. How grades are computed:
+[docs/grading.md](https://github.com/kartsan03/wiretruth/blob/main/docs/grading.md).
+
+## Quickstart
+
+You need Python 3.11 or newer and [uv](https://docs.astral.sh/uv/), which builds each library's own environment.
+
+```bash
+git clone https://github.com/kartsan03/wiretruth && cd wiretruth
+uv run wiretruth run --lib any-llm
+```
+
+```text
+scenario                      openai-chat
+truncation.structured-output  ✓ PASS
+truncation.text               ✓ PASS
+truncation.text-stream        ✓ PASS
+
+any-llm 1.33.0: 3 cells, 3 PASS
+stderr: .wiretruth/runs/20261010T193444.312067Z
+```
+
+No API keys are needed. Everything runs against recorded responses on `127.0.0.1`. The first run downloads any-llm
+into `.wiretruth/envs/`; later runs work offline. `uv run wiretruth run --shim shims/reference` runs the
+built-in reference client, which must pass every cell. The scenarios, fixtures and shims live in this repository,
+so wiretruth runs from a clone.
+
+## Add your library
+
+A shim is a small program. It reads a call spec as JSON on stdin, calls your library the way its docs show, and
+prints what the library reported as JSON on stdout. The
+[any-llm shim](https://github.com/kartsan03/wiretruth/blob/main/shims/python/any-llm/shim.py) is 150 lines. The
+protocol is in [shims/README.md](https://github.com/kartsan03/wiretruth/blob/main/shims/README.md); to propose a
+library, [open a new-library issue](https://github.com/kartsan03/wiretruth/issues/new?template=new-library.yml).
+
+## Related work
+
+- [Open Responses](https://github.com/openresponses/openresponses): a spec and compliance tests for servers that
+  implement the Responses API. wiretruth tests clients across several native APIs.
+- [LLMConform](https://github.com/aitk-org/LLMConform): conformance tests for LLM gateways and APIs, run live.
+- [llmprobe](https://github.com/ivanfioravanti/llmprobe) and [conform](https://github.com/NiLabs-Org/conform):
+  conformance for inference engines (vLLM, llama.cpp, Ollama, …).
+- [aimock](https://github.com/CopilotKit/aimock): a mock server for testing AI apps, with record and replay. Use it
+  for your app's tests; wiretruth audits the SDK layer itself.
+- The design borrows from [JSONTestSuite](https://github.com/nst/JSONTestSuite),
+  [Wycheproof](https://github.com/C2SP/wycheproof), [toml-test](https://github.com/toml-lang/toml-test) and
+  [compat-table](https://github.com/compat-table/compat-table).
+
+## Author note
+
+I've contributed fixes to libraries that wiretruth tests or will test:
+[mozilla-ai/any-llm#1389](https://github.com/mozilla-ai/any-llm/pull/1389) and
+[TanStack/ai#1548](https://github.com/TanStack/ai/pull/1548). Grades are computed by the code in this repository.
+If you think a grade is wrong, please
+[open a grade dispute](https://github.com/kartsan03/wiretruth/issues/new?template=grade-dispute.yml).
+
+## Contributing
+
+Scenario ideas, new libraries and grade disputes are all welcome. See
+[CONTRIBUTING.md](https://github.com/kartsan03/wiretruth/blob/main/CONTRIBUTING.md).
+
+## Citing
+
+See [CITATION.cff](https://github.com/kartsan03/wiretruth/blob/main/CITATION.cff).
+
+## License
+
+MIT. Fixtures contain model outputs recorded from provider APIs;
+[docs/recording.md](https://github.com/kartsan03/wiretruth/blob/main/docs/recording.md) explains how they're recorded.

wiretruth-0.1.0/README.md

@@ -0,0 +1,129 @@
+# wiretruth
+
+wiretruth replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually
+happened.
+
+**Does your LLM SDK tell you the truth?**
+
+[![CI](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml/badge.svg)](https://github.com/kartsan03/wiretruth/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/wiretruth)](https://pypi.org/project/wiretruth/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/kartsan03/wiretruth/blob/main/LICENSE)
+
+Version 0.1 tests one library, any-llm, on the OpenAI Chat Completions API with three truncation scenarios. More
+providers and libraries follow, and a public results matrix arrives in v0.4. Until then, the grades are in
+[`results/`](https://github.com/kartsan03/wiretruth/tree/main/results).
+
+## Why
+
+A model hits its output token limit halfway through a JSON object. The provider says so plainly:
+`"finish_reason": "length"`. What your code sees depends on the SDK in between. If the SDK raises a generic JSON
+parse error, nothing tells you that a higher limit would fix it. If it returns the partial object as a success,
+your pipeline stores an incomplete record.
+
+wiretruth has this case recorded as `truncation.structured-output` on `openai-chat`. Replayed through any-llm:
+
+| Library | What the caller gets | Grade |
+|---|---|---|
+| any-llm 1.33.0 | Raises `openai.LengthFinishReasonError`. Its `completion` has `finish_reason: "length"` and the token usage. | ✓ PASS |
+
+wiretruth runs the same recorded edge cases through every library it supports and grades each one against what the
+provider sent.
+
+## What it checks
+
+| Bug class | What goes wrong for users | Scenarios |
+|---|---|---|
+| Silent truncation | Agents accept partial output or retry without raising the limit | 3 |
+| Finish reason mapping | Control flow branches on the wrong reason | planned |
+| Usage accounting | Cost and quota tracking is off (cached and reasoning tokens) | planned |
+| Stream integrity | Corrupted or unfinished text is treated as final | planned |
+| Tool call fidelity | Tools run with broken or merged arguments | planned |
+| Error classification | Retries and backoff misbehave | planned |
+| Request fidelity | The library sends options the caller didn't set | planned |
+
+The scenarios are in
+[`suite/scenarios/`](https://github.com/kartsan03/wiretruth/tree/main/suite/scenarios), and the bug classes in
+[`suite/bug-classes.json`](https://github.com/kartsan03/wiretruth/blob/main/suite/bug-classes.json).
+
+## Grades
+
+| | Grade | Meaning |
+|---|---|---|
+| ✓ | PASS | Reports what happened: meets every MUST requirement of the scenario. |
+| ~ | LOSSY | Doesn't contradict what happened, but the fact is only in raw metadata, or missing. |
+| ✗ | WRONG | Reports something that contradicts what happened. |
+| ! | CRASH | Fails with an internal error, such as a `KeyError` in its own parser. |
+| ⧗ | HANG | Doesn't return within 20 s on a response that ended within 1 s. |
+| – | UNSUPPORTED | The library doesn't support this provider or feature (documented). |
+| ? | HARNESS | A problem on our side, not the library's. |
+
+Each scenario lists its requirements. MUST requirements set the grade. SHOULD requirements, such as the exact
+partial text in the truncation scenarios, add a note when they aren't met. How grades are computed:
+[docs/grading.md](https://github.com/kartsan03/wiretruth/blob/main/docs/grading.md).
+
+## Quickstart
+
+You need Python 3.11 or newer and [uv](https://docs.astral.sh/uv/), which builds each library's own environment.
+
+```bash
+git clone https://github.com/kartsan03/wiretruth && cd wiretruth
+uv run wiretruth run --lib any-llm
+```
+
+```text
+scenario                      openai-chat
+truncation.structured-output  ✓ PASS
+truncation.text               ✓ PASS
+truncation.text-stream        ✓ PASS
+
+any-llm 1.33.0: 3 cells, 3 PASS
+stderr: .wiretruth/runs/20261010T193444.312067Z
+```
+
+No API keys are needed. Everything runs against recorded responses on `127.0.0.1`. The first run downloads any-llm
+into `.wiretruth/envs/`; later runs work offline. `uv run wiretruth run --shim shims/reference` runs the
+built-in reference client, which must pass every cell. The scenarios, fixtures and shims live in this repository,
+so wiretruth runs from a clone.
+
+## Add your library
+
+A shim is a small program. It reads a call spec as JSON on stdin, calls your library the way its docs show, and
+prints what the library reported as JSON on stdout. The
+[any-llm shim](https://github.com/kartsan03/wiretruth/blob/main/shims/python/any-llm/shim.py) is 150 lines. The
+protocol is in [shims/README.md](https://github.com/kartsan03/wiretruth/blob/main/shims/README.md); to propose a
+library, [open a new-library issue](https://github.com/kartsan03/wiretruth/issues/new?template=new-library.yml).
+
+## Related work
+
+- [Open Responses](https://github.com/openresponses/openresponses): a spec and compliance tests for servers that
+  implement the Responses API. wiretruth tests clients across several native APIs.
+- [LLMConform](https://github.com/aitk-org/LLMConform): conformance tests for LLM gateways and APIs, run live.
+- [llmprobe](https://github.com/ivanfioravanti/llmprobe) and [conform](https://github.com/NiLabs-Org/conform):
+  conformance for inference engines (vLLM, llama.cpp, Ollama, …).
+- [aimock](https://github.com/CopilotKit/aimock): a mock server for testing AI apps, with record and replay. Use it
+  for your app's tests; wiretruth audits the SDK layer itself.
+- The design borrows from [JSONTestSuite](https://github.com/nst/JSONTestSuite),
+  [Wycheproof](https://github.com/C2SP/wycheproof), [toml-test](https://github.com/toml-lang/toml-test) and
+  [compat-table](https://github.com/compat-table/compat-table).
+
+## Author note
+
+I've contributed fixes to libraries that wiretruth tests or will test:
+[mozilla-ai/any-llm#1389](https://github.com/mozilla-ai/any-llm/pull/1389) and
+[TanStack/ai#1548](https://github.com/TanStack/ai/pull/1548). Grades are computed by the code in this repository.
+If you think a grade is wrong, please
+[open a grade dispute](https://github.com/kartsan03/wiretruth/issues/new?template=grade-dispute.yml).
+
+## Contributing
+
+Scenario ideas, new libraries and grade disputes are all welcome. See
+[CONTRIBUTING.md](https://github.com/kartsan03/wiretruth/blob/main/CONTRIBUTING.md).
+
+## Citing
+
+See [CITATION.cff](https://github.com/kartsan03/wiretruth/blob/main/CITATION.cff).
+
+## License
+
+MIT. Fixtures contain model outputs recorded from provider APIs;
+[docs/recording.md](https://github.com/kartsan03/wiretruth/blob/main/docs/recording.md) explains how they're recorded.

wiretruth-0.1.0/pyproject.toml

@@ -0,0 +1,57 @@
+[project]
+name = "wiretruth"
+dynamic = ["version"]
+description = "Replays real LLM provider responses through LLM SDKs and checks whether each SDK reports what actually happened."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.11"
+# 4.18 added the `referencing` registry used for $ref between the bundled schemas; the tests
+# pass from 4.21 on (CI runs them on 4.21.0), where some error messages changed wording.
+dependencies = ["jsonschema>=4.21"]
+keywords = ["llm", "sdk", "conformance", "test-suite", "openai", "anthropic", "gemini", "ollama", "testing"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "pytest-cov>=5", "ruff>=0.6", "mypy>=1.11", "types-jsonschema"]
+
+[project.scripts]
+wiretruth = "wiretruth.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/kartsan03/wiretruth"
+Repository = "https://github.com/kartsan03/wiretruth"
+Issues = "https://github.com/kartsan03/wiretruth/issues"
+Changelog = "https://github.com/kartsan03/wiretruth/blob/main/CHANGELOG.md"
+
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+version = { attr = "wiretruth.__version__" }
+
+[tool.setuptools.package-data]
+wiretruth = ["schemas/*.json"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "SIM", "RUF"]
+
+[tool.mypy]
+strict = true
+python_version = "3.11"
+files = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

wiretruth-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

wiretruth-0.1.0/src/wiretruth/__init__.py

@@ -0,0 +1,3 @@
+"""wiretruth: replay real LLM provider responses through LLM SDKs and grade what they report."""
+
+__version__ = "0.1.0"

wiretruth-0.1.0/src/wiretruth/__main__.py

@@ -0,0 +1,4 @@
+from wiretruth.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())