failloud 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- failloud-0.2.0/LICENSE +21 -0
- failloud-0.2.0/PKG-INFO +232 -0
- failloud-0.2.0/README.md +205 -0
- failloud-0.2.0/pyproject.toml +53 -0
- failloud-0.2.0/setup.cfg +4 -0
- failloud-0.2.0/src/failloud/__init__.py +51 -0
- failloud-0.2.0/src/failloud/checks/__init__.py +17 -0
- failloud-0.2.0/src/failloud/checks/completeness.py +241 -0
- failloud-0.2.0/src/failloud/checks/counts.py +169 -0
- failloud-0.2.0/src/failloud/checks/evidence.py +182 -0
- failloud-0.2.0/src/failloud/checks/placeholder.py +187 -0
- failloud-0.2.0/src/failloud/checks/stale.py +184 -0
- failloud-0.2.0/src/failloud/cli.py +190 -0
- failloud-0.2.0/src/failloud/engine.py +44 -0
- failloud-0.2.0/src/failloud/mcp_server.py +204 -0
- failloud-0.2.0/src/failloud/models.py +140 -0
- failloud-0.2.0/src/failloud/receipts.py +321 -0
- failloud-0.2.0/src/failloud/report.py +110 -0
- failloud-0.2.0/src/failloud/util.py +148 -0
- failloud-0.2.0/src/failloud.egg-info/PKG-INFO +232 -0
- failloud-0.2.0/src/failloud.egg-info/SOURCES.txt +32 -0
- failloud-0.2.0/src/failloud.egg-info/dependency_links.txt +1 -0
- failloud-0.2.0/src/failloud.egg-info/entry_points.txt +3 -0
- failloud-0.2.0/src/failloud.egg-info/requires.txt +6 -0
- failloud-0.2.0/src/failloud.egg-info/top_level.txt +1 -0
- failloud-0.2.0/tests/test_cli.py +76 -0
- failloud-0.2.0/tests/test_completeness.py +90 -0
- failloud-0.2.0/tests/test_counts.py +65 -0
- failloud-0.2.0/tests/test_engine_and_report.py +92 -0
- failloud-0.2.0/tests/test_evidence.py +74 -0
- failloud-0.2.0/tests/test_mcp.py +99 -0
- failloud-0.2.0/tests/test_placeholder.py +71 -0
- failloud-0.2.0/tests/test_receipts.py +200 -0
- failloud-0.2.0/tests/test_stale.py +85 -0
failloud-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Naveen Yalakala
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
failloud-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: failloud
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Verification harness for LLM agent outputs — catches the silent failures agents ship as 'done'.
|
|
5
|
+
Author: Naveen Yalakala
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Yalakala-Naveen/failloud
|
|
8
|
+
Project-URL: Issues, https://github.com/Yalakala-Naveen/failloud/issues
|
|
9
|
+
Keywords: llm,agents,verification,evaluation,mcp,reliability,silent-failures
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Provides-Extra: mcp
|
|
23
|
+
Requires-Dist: mcp>=1.2; extra == "mcp"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# FailLoud
|
|
29
|
+
|
|
30
|
+
**A verification harness for LLM agent outputs. It catches the silent failures agents ship as `done`.**
|
|
31
|
+
|
|
32
|
+
Agents rarely fail loudly. They return `status: "done"` with an empty payload. They claim they sent, posted, and published — and nothing in the output proves any of it happened. They plan from state that went stale two steps ago, report counts their own itemized record contradicts, and deliver files with silent holes inside them. Every dashboard stays green, because the agent graded its own homework.
|
|
33
|
+
|
|
34
|
+
FailLoud reads an agent's output the way a skeptical reviewer would — and produces a **failure report** instead of a shrug.
|
|
35
|
+
|
|
36
|
+
> Built by a Forward Deployed Engineer from production failures — every check in this tool exists because a real agent failed this way in production and the miss became a rule.
|
|
37
|
+
|
|
38
|
+
**▶ Try it in your browser: [FailLoud Playground](https://yalakala-naveen.github.io/failloud/)** — paste an agent output, get the failure report. No install, nothing uploaded.
|
|
39
|
+
|
|
40
|
+
## The five silent failures
|
|
41
|
+
|
|
42
|
+
| Check | Name | What it catches |
|
|
43
|
+
|---|---|---|
|
|
44
|
+
| FL001 | `placeholder_shipped` | Output declared done while the payload is empty or still holds draft scaffolding — lorem ipsum, unrendered `{{template}}` braces, "coming soon". *A page once went live, launch marked complete, with placeholder copy still in the body. Nothing crashed; every signal said success.* |
|
|
45
|
+
| FL002 | `unbacked_claim` | A claimed action (sent, posted, published, updated…) with no evidence — no message id, URL, receipt, or artifact. *A run once reported a batch as sent while a bounce sat unrecorded; "sent" was asserted by the same component that did the sending.* |
|
|
46
|
+
| FL003 | `stale_state` | Work based on state/context already older than allowed, state with no freshness timestamp at all, or a state-version skew between handover and run. *A pipeline planned flawlessly from a snapshot of a world that no longer existed.* |
|
|
47
|
+
| FL004 | `count_mismatch` | A reported count that disagrees with the itemized record in the same document, or a total that isn't the sum of its parts. *Reports counted actions attempted; audiences read them as actions completed. Only the list was ever true.* |
|
|
48
|
+
| FL005 | `missing_components` | Required fields present-but-empty, and dead stretches inside segmented deliverables (zero-length segments, silent gaps). *A delivered video passed every existence check while one segment inside it was silent.* |
|
|
49
|
+
|
|
50
|
+
These are not eval scores. FailLoud never asks a model whether an output "looks good" — every verdict is deterministic, reproducible, and explainable line by line. No LLM sits in the verdict path.
|
|
51
|
+
|
|
52
|
+
## Install
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install failloud # CLI + library (stdlib-only core, zero dependencies)
|
|
56
|
+
pip install "failloud[mcp]" # + the MCP server
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Or from a clone: `pip install .` (add `.[mcp]` for the server).
|
|
60
|
+
|
|
61
|
+
## As an agent skill
|
|
62
|
+
|
|
63
|
+
The same five checks also ship as a [`SKILL.md`](skill/SKILL.md) — a judgment playbook an agent runs on its own output before claiming done, for Claude Code, Cursor, Copilot and other SKILL.md-compatible agents. Listing copy and worked examples live in [`skill/`](skill/).
|
|
64
|
+
|
|
65
|
+
## 60-second demo
|
|
66
|
+
|
|
67
|
+
The repo ships a realistic agent output that commits all five failures (`examples/failing_output.json`) and its corrected twin (`examples/passing_output.json`):
|
|
68
|
+
|
|
69
|
+
```console
|
|
70
|
+
$ failloud check examples/failing_output.json
|
|
71
|
+
FAILLOUD — agent output verification
|
|
72
|
+
Source: examples/failing_output.json
|
|
73
|
+
Verdict: FAIL (8 error(s), 1 warning(s))
|
|
74
|
+
|
|
75
|
+
[FL001] placeholder_shipped — FAIL
|
|
76
|
+
✗ $.output.body — Placeholder content shipped as final output: matched 'coming soon', 'lorem ipsum', '{{', '}}'.
|
|
77
|
+
[FL002] unbacked_claim — FAIL
|
|
78
|
+
✗ $.claims[1] — Claimed action 'card_published' has no evidence — no id, URL, receipt, or artifact in the claim backs that it happened.
|
|
79
|
+
[FL003] stale_state — FAIL
|
|
80
|
+
✗ $.state.context_updated_at — State timestamp is 77.1h old at run time (limit 24h) — the run worked from context older than allowed.
|
|
81
|
+
✗ $ — Run based itself on state version 9 but the state it carried is version 7 — the handover and the work disagree about the world.
|
|
82
|
+
[FL004] count_mismatch — FAIL
|
|
83
|
+
✗ $.counts.delivered — Count for 'delivered' is reported as 12 but the itemized record at $.items.delivered contains 9 item(s) — the report overstates what the record supports.
|
|
84
|
+
✗ $.counts.sent — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
|
|
85
|
+
[FL005] missing_components — FAIL
|
|
86
|
+
! $.output.archive_url — Field 'archive_url' is present but empty while its siblings carry content — verify it was meant to be blank.
|
|
87
|
+
✗ $.segments[1] — Segment 1 has zero length (4s–4s) — a component that occupies a slot and contains nothing.
|
|
88
|
+
✗ $.segments[2] — Dead stretch of 5.5s before segment 2 (gap limit 2s) — the deliverable goes silent where content should be.
|
|
89
|
+
|
|
90
|
+
$ echo $?
|
|
91
|
+
1
|
|
92
|
+
|
|
93
|
+
$ failloud check examples/passing_output.json
|
|
94
|
+
FAILLOUD — agent output verification
|
|
95
|
+
Source: examples/passing_output.json
|
|
96
|
+
Verdict: PASS (0 error(s), 0 warning(s))
|
|
97
|
+
|
|
98
|
+
[FL001] placeholder_shipped — PASS
|
|
99
|
+
[FL002] unbacked_claim — PASS
|
|
100
|
+
[FL003] stale_state — PASS
|
|
101
|
+
[FL004] count_mismatch — PASS
|
|
102
|
+
[FL005] missing_components — PASS
|
|
103
|
+
No silent failures found. The output backs its claims.
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Exit codes are the CI contract: **0** pass, **1** verification failed (the agent output failed, not the tool), **2** the tool itself could not run.
|
|
107
|
+
|
|
108
|
+
### The failure report
|
|
109
|
+
|
|
110
|
+
`--format md` produces the artifact you attach to a run, an issue, or a PR:
|
|
111
|
+
|
|
112
|
+
```console
|
|
113
|
+
$ failloud check examples/failing_output.json --format md --out report.md
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
```markdown
|
|
117
|
+
# FAILURE REPORT
|
|
118
|
+
|
|
119
|
+
**Verdict:** **FAIL** — 8 error(s), 1 warning(s)
|
|
120
|
+
|
|
121
|
+
## Checks
|
|
122
|
+
|
|
123
|
+
| Check | Name | Result | Findings |
|
|
124
|
+
|---|---|---|---|
|
|
125
|
+
| FL001 | placeholder_shipped | FAIL | 1 |
|
|
126
|
+
| FL002 | unbacked_claim | FAIL | 1 |
|
|
127
|
+
| FL003 | stale_state | FAIL | 2 |
|
|
128
|
+
| FL004 | count_mismatch | FAIL | 2 |
|
|
129
|
+
| FL005 | missing_components | FAIL | 3 |
|
|
130
|
+
|
|
131
|
+
## Findings
|
|
132
|
+
|
|
133
|
+
### FL004 — count_mismatch
|
|
134
|
+
|
|
135
|
+
*Why it matters: The claimed number is what travels into dashboards and decisions; the itemized record is the only part that was ever true.*
|
|
136
|
+
|
|
137
|
+
- **ERROR** `$.counts.sent` — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Verification receipts (new in 0.2.0)
|
|
141
|
+
|
|
142
|
+
A report says what FailLoud found; a **receipt proves it**. Seal any run into a hash-chained receipt file:
|
|
143
|
+
|
|
144
|
+
```console
|
|
145
|
+
$ failloud check examples/failing_output.json --receipt-out receipts.jsonl
|
|
146
|
+
$ failloud check examples/passing_output.json --receipt-out receipts.jsonl
|
|
147
|
+
$ failloud verify-receipts receipts.jsonl
|
|
148
|
+
Receipt chain OK — 2 receipt(s) verified in receipts.jsonl.
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Each receipt embeds the full result and two SHA-256 hashes: `result_hash` seals the result and its source, and `receipt_hash` seals every receipt field including `prev_hash` — the previous receipt's hash. Edit a verdict, erase a finding, drop or reorder a receipt, and `verify-receipts` (exit 1) names the break. Anyone holding the chain can re-verify it offline, stdlib only — the audit trail an orchestrator needs before one agent's "done" becomes the next agent's input. In Python: `create_receipt`, `verify_receipt`, `verify_chain`; over MCP: `create_verification_receipt`, `verify_receipt_chain`.
|
|
152
|
+
|
|
153
|
+
## What to feed it
|
|
154
|
+
|
|
155
|
+
FailLoud works on **any JSON an agent produces** — the checks are shape-tolerant heuristics, not a schema straitjacket. They verify the most when the output follows this envelope (every field optional; each one present is one more thing that gets checked instead of trusted):
|
|
156
|
+
|
|
157
|
+
```json
|
|
158
|
+
{
|
|
159
|
+
"task": "Send the weekly digest and publish the summary card",
|
|
160
|
+
"status": "done",
|
|
161
|
+
"generated_at": "2026-11-11T14:05:00Z",
|
|
162
|
+
"summary": "Sent the digest to 9 subscribers and published the summary card.",
|
|
163
|
+
"claims": [
|
|
164
|
+
{"action": "email_sent", "target": "subscriber-list",
|
|
165
|
+
"evidence": {"message_id": "msg_8f42"}},
|
|
166
|
+
{"action": "card_published", "target": "community-feed",
|
|
167
|
+
"evidence": {"post_id": "post_2481", "url": "https://…/posts/2481"}}
|
|
168
|
+
],
|
|
169
|
+
"counts": {"sent": 9, "delivered": 9},
|
|
170
|
+
"items": {"sent": [ … ], "delivered": [ … ]},
|
|
171
|
+
"state": {"context_updated_at": "2026-11-11T13:40:00Z", "version": 9},
|
|
172
|
+
"based_on": {"expected_state_version": 9},
|
|
173
|
+
"output": {"subject": "…", "body": "…", "archive_url": "https://…"},
|
|
174
|
+
"segments": [{"start": 0, "end": 4}, {"start": 4, "end": 9}]
|
|
175
|
+
}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Enforce your own required fields with `--require output.archive_url,output.body` or a JSON Schema (`--schema schema.json`; the `required`/`properties`/`items` subset is supported). Thresholds are flags: `--max-state-age-hours`, `--max-segment-gap-seconds`. Run a subset with `--only FL002,FL004`. `--format json` gives machines the same verdict.
|
|
179
|
+
|
|
180
|
+
## Python API
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
import json
|
|
184
|
+
from failloud import check_output, CheckConfig
|
|
185
|
+
|
|
186
|
+
result = check_output(json.load(open("agent_output.json")))
|
|
187
|
+
if not result.passed:
|
|
188
|
+
for finding in result.errors:
|
|
189
|
+
print(finding.check_id, finding.path, finding.message)
|
|
190
|
+
|
|
191
|
+
# Tuned run:
|
|
192
|
+
result = check_output(doc, CheckConfig(max_state_age_hours=4,
|
|
193
|
+
required_paths=("output.url",)))
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
## MCP server
|
|
197
|
+
|
|
198
|
+
FailLoud also ships as an [MCP](https://modelcontextprotocol.io) server, so an agent or orchestrator can verify a sub-agent's output *inside the loop*, one tool call before acting on it:
|
|
199
|
+
|
|
200
|
+
```json
|
|
201
|
+
{
|
|
202
|
+
"mcpServers": {
|
|
203
|
+
"failloud": { "command": "failloud-mcp" }
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Tools: `check_output` (all checks, JSON result), `check_placeholder`, `check_evidence`, `check_stale_state`, `check_counts`, `check_completeness`, `failure_report` (the Markdown report), `create_verification_receipt`, and `verify_receipt_chain`. Works with both the 1.x and 2.x Python MCP SDKs.
|
|
209
|
+
|
|
210
|
+
## Design principles
|
|
211
|
+
|
|
212
|
+
- **Deterministic.** Same input, same verdict, every time. No model in the verdict path.
|
|
213
|
+
- **Zero-dependency core.** A verification tool should not expand your supply chain.
|
|
214
|
+
- **Fail loud, specifically.** Every finding carries a JSON path, a plain-language explanation, and the numbers behind it. "Something seems off" is not a finding.
|
|
215
|
+
- **Honest scope.** Heuristics are labelled as heuristics. A claim with no itemized record is *unverifiable*, and FailLoud says so instead of inventing a verdict.
|
|
216
|
+
|
|
217
|
+
## Development
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
pip install -e ".[mcp,dev]"
|
|
221
|
+
python -m pytest # 90 tests, including a live MCP stdio end-to-end test
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
## Roadmap
|
|
225
|
+
|
|
226
|
+
- PyPI release and a public GitHub home (this repo is release-ready; publishing is a maintainer decision, not a code gap).
|
|
227
|
+
- Framework adapters that capture outputs directly from popular agent frameworks instead of reading files.
|
|
228
|
+
- A `failloud watch` mode for streaming/runner integration.
|
|
229
|
+
|
|
230
|
+
## License
|
|
231
|
+
|
|
232
|
+
MIT — see [LICENSE](LICENSE).
|
failloud-0.2.0/README.md
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# FailLoud
|
|
2
|
+
|
|
3
|
+
**A verification harness for LLM agent outputs. It catches the silent failures agents ship as `done`.**
|
|
4
|
+
|
|
5
|
+
Agents rarely fail loudly. They return `status: "done"` with an empty payload. They claim they sent, posted, and published — and nothing in the output proves any of it happened. They plan from state that went stale two steps ago, report counts their own itemized record contradicts, and deliver files with silent holes inside them. Every dashboard stays green, because the agent graded its own homework.
|
|
6
|
+
|
|
7
|
+
FailLoud reads an agent's output the way a skeptical reviewer would — and produces a **failure report** instead of a shrug.
|
|
8
|
+
|
|
9
|
+
> Built by a Forward Deployed Engineer from production failures — every check in this tool exists because a real agent failed this way in production and the miss became a rule.
|
|
10
|
+
|
|
11
|
+
**▶ Try it in your browser: [FailLoud Playground](https://yalakala-naveen.github.io/failloud/)** — paste an agent output, get the failure report. No install, nothing uploaded.
|
|
12
|
+
|
|
13
|
+
## The five silent failures
|
|
14
|
+
|
|
15
|
+
| Check | Name | What it catches |
|
|
16
|
+
|---|---|---|
|
|
17
|
+
| FL001 | `placeholder_shipped` | Output declared done while the payload is empty or still holds draft scaffolding — lorem ipsum, unrendered `{{template}}` braces, "coming soon". *A page once went live, launch marked complete, with placeholder copy still in the body. Nothing crashed; every signal said success.* |
|
|
18
|
+
| FL002 | `unbacked_claim` | A claimed action (sent, posted, published, updated…) with no evidence — no message id, URL, receipt, or artifact. *A run once reported a batch as sent while a bounce sat unrecorded; "sent" was asserted by the same component that did the sending.* |
|
|
19
|
+
| FL003 | `stale_state` | Work based on state/context already older than allowed, state with no freshness timestamp at all, or a state-version skew between handover and run. *A pipeline planned flawlessly from a snapshot of a world that no longer existed.* |
|
|
20
|
+
| FL004 | `count_mismatch` | A reported count that disagrees with the itemized record in the same document, or a total that isn't the sum of its parts. *Reports counted actions attempted; audiences read them as actions completed. Only the list was ever true.* |
|
|
21
|
+
| FL005 | `missing_components` | Required fields present-but-empty, and dead stretches inside segmented deliverables (zero-length segments, silent gaps). *A delivered video passed every existence check while one segment inside it was silent.* |
|
|
22
|
+
|
|
23
|
+
These are not eval scores. FailLoud never asks a model whether an output "looks good" — every verdict is deterministic, reproducible, and explainable line by line. No LLM sits in the verdict path.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install failloud # CLI + library (stdlib-only core, zero dependencies)
|
|
29
|
+
pip install "failloud[mcp]" # + the MCP server
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Or from a clone: `pip install .` (add `.[mcp]` for the server).
|
|
33
|
+
|
|
34
|
+
## As an agent skill
|
|
35
|
+
|
|
36
|
+
The same five checks also ship as a [`SKILL.md`](skill/SKILL.md) — a judgment playbook an agent runs on its own output before claiming done, for Claude Code, Cursor, Copilot and other SKILL.md-compatible agents. Listing copy and worked examples live in [`skill/`](skill/).
|
|
37
|
+
|
|
38
|
+
## 60-second demo
|
|
39
|
+
|
|
40
|
+
The repo ships a realistic agent output that commits all five failures (`examples/failing_output.json`) and its corrected twin (`examples/passing_output.json`):
|
|
41
|
+
|
|
42
|
+
```console
|
|
43
|
+
$ failloud check examples/failing_output.json
|
|
44
|
+
FAILLOUD — agent output verification
|
|
45
|
+
Source: examples/failing_output.json
|
|
46
|
+
Verdict: FAIL (8 error(s), 1 warning(s))
|
|
47
|
+
|
|
48
|
+
[FL001] placeholder_shipped — FAIL
|
|
49
|
+
✗ $.output.body — Placeholder content shipped as final output: matched 'coming soon', 'lorem ipsum', '{{', '}}'.
|
|
50
|
+
[FL002] unbacked_claim — FAIL
|
|
51
|
+
✗ $.claims[1] — Claimed action 'card_published' has no evidence — no id, URL, receipt, or artifact in the claim backs that it happened.
|
|
52
|
+
[FL003] stale_state — FAIL
|
|
53
|
+
✗ $.state.context_updated_at — State timestamp is 77.1h old at run time (limit 24h) — the run worked from context older than allowed.
|
|
54
|
+
✗ $ — Run based itself on state version 9 but the state it carried is version 7 — the handover and the work disagree about the world.
|
|
55
|
+
[FL004] count_mismatch — FAIL
|
|
56
|
+
✗ $.counts.delivered — Count for 'delivered' is reported as 12 but the itemized record at $.items.delivered contains 9 item(s) — the report overstates what the record supports.
|
|
57
|
+
✗ $.counts.sent — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
|
|
58
|
+
[FL005] missing_components — FAIL
|
|
59
|
+
! $.output.archive_url — Field 'archive_url' is present but empty while its siblings carry content — verify it was meant to be blank.
|
|
60
|
+
✗ $.segments[1] — Segment 1 has zero length (4s–4s) — a component that occupies a slot and contains nothing.
|
|
61
|
+
✗ $.segments[2] — Dead stretch of 5.5s before segment 2 (gap limit 2s) — the deliverable goes silent where content should be.
|
|
62
|
+
|
|
63
|
+
$ echo $?
|
|
64
|
+
1
|
|
65
|
+
|
|
66
|
+
$ failloud check examples/passing_output.json
|
|
67
|
+
FAILLOUD — agent output verification
|
|
68
|
+
Source: examples/passing_output.json
|
|
69
|
+
Verdict: PASS (0 error(s), 0 warning(s))
|
|
70
|
+
|
|
71
|
+
[FL001] placeholder_shipped — PASS
|
|
72
|
+
[FL002] unbacked_claim — PASS
|
|
73
|
+
[FL003] stale_state — PASS
|
|
74
|
+
[FL004] count_mismatch — PASS
|
|
75
|
+
[FL005] missing_components — PASS
|
|
76
|
+
No silent failures found. The output backs its claims.
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Exit codes are the CI contract: **0** pass, **1** verification failed (the agent output failed, not the tool), **2** the tool itself could not run.
|
|
80
|
+
|
|
81
|
+
### The failure report
|
|
82
|
+
|
|
83
|
+
`--format md` produces the artifact you attach to a run, an issue, or a PR:
|
|
84
|
+
|
|
85
|
+
```console
|
|
86
|
+
$ failloud check examples/failing_output.json --format md --out report.md
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
```markdown
|
|
90
|
+
# FAILURE REPORT
|
|
91
|
+
|
|
92
|
+
**Verdict:** **FAIL** — 8 error(s), 1 warning(s)
|
|
93
|
+
|
|
94
|
+
## Checks
|
|
95
|
+
|
|
96
|
+
| Check | Name | Result | Findings |
|
|
97
|
+
|---|---|---|---|
|
|
98
|
+
| FL001 | placeholder_shipped | FAIL | 1 |
|
|
99
|
+
| FL002 | unbacked_claim | FAIL | 1 |
|
|
100
|
+
| FL003 | stale_state | FAIL | 2 |
|
|
101
|
+
| FL004 | count_mismatch | FAIL | 2 |
|
|
102
|
+
| FL005 | missing_components | FAIL | 3 |
|
|
103
|
+
|
|
104
|
+
## Findings
|
|
105
|
+
|
|
106
|
+
### FL004 — count_mismatch
|
|
107
|
+
|
|
108
|
+
*Why it matters: The claimed number is what travels into dashboards and decisions; the itemized record is the only part that was ever true.*
|
|
109
|
+
|
|
110
|
+
- **ERROR** `$.counts.sent` — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Verification receipts (new in 0.2.0)
|
|
114
|
+
|
|
115
|
+
A report says what FailLoud found; a **receipt proves it**. Seal any run into a hash-chained receipt file:
|
|
116
|
+
|
|
117
|
+
```console
|
|
118
|
+
$ failloud check examples/failing_output.json --receipt-out receipts.jsonl
|
|
119
|
+
$ failloud check examples/passing_output.json --receipt-out receipts.jsonl
|
|
120
|
+
$ failloud verify-receipts receipts.jsonl
|
|
121
|
+
Receipt chain OK — 2 receipt(s) verified in receipts.jsonl.
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Each receipt embeds the full result and two SHA-256 hashes: `result_hash` seals the result and its source, and `receipt_hash` seals every receipt field including `prev_hash` — the previous receipt's hash. Edit a verdict, erase a finding, drop or reorder a receipt, and `verify-receipts` (exit 1) names the break. Anyone holding the chain can re-verify it offline, stdlib only — the audit trail an orchestrator needs before one agent's "done" becomes the next agent's input. In Python: `create_receipt`, `verify_receipt`, `verify_chain`; over MCP: `create_verification_receipt`, `verify_receipt_chain`.
|
|
125
|
+
|
|
126
|
+
## What to feed it
|
|
127
|
+
|
|
128
|
+
FailLoud works on **any JSON an agent produces** — the checks are shape-tolerant heuristics, not a schema straitjacket. They verify the most when the output follows this envelope (every field optional; each one present is one more thing that gets checked instead of trusted):
|
|
129
|
+
|
|
130
|
+
```json
|
|
131
|
+
{
|
|
132
|
+
"task": "Send the weekly digest and publish the summary card",
|
|
133
|
+
"status": "done",
|
|
134
|
+
"generated_at": "2026-11-11T14:05:00Z",
|
|
135
|
+
"summary": "Sent the digest to 9 subscribers and published the summary card.",
|
|
136
|
+
"claims": [
|
|
137
|
+
{"action": "email_sent", "target": "subscriber-list",
|
|
138
|
+
"evidence": {"message_id": "msg_8f42"}},
|
|
139
|
+
{"action": "card_published", "target": "community-feed",
|
|
140
|
+
"evidence": {"post_id": "post_2481", "url": "https://…/posts/2481"}}
|
|
141
|
+
],
|
|
142
|
+
"counts": {"sent": 9, "delivered": 9},
|
|
143
|
+
"items": {"sent": [ … ], "delivered": [ … ]},
|
|
144
|
+
"state": {"context_updated_at": "2026-11-11T13:40:00Z", "version": 9},
|
|
145
|
+
"based_on": {"expected_state_version": 9},
|
|
146
|
+
"output": {"subject": "…", "body": "…", "archive_url": "https://…"},
|
|
147
|
+
"segments": [{"start": 0, "end": 4}, {"start": 4, "end": 9}]
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Enforce your own required fields with `--require output.archive_url,output.body` or a JSON Schema (`--schema schema.json`; the `required`/`properties`/`items` subset is supported). Thresholds are flags: `--max-state-age-hours`, `--max-segment-gap-seconds`. Run a subset with `--only FL002,FL004`. `--format json` gives machines the same verdict.
|
|
152
|
+
|
|
153
|
+
## Python API
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
import json
|
|
157
|
+
from failloud import check_output, CheckConfig
|
|
158
|
+
|
|
159
|
+
result = check_output(json.load(open("agent_output.json")))
|
|
160
|
+
if not result.passed:
|
|
161
|
+
for finding in result.errors:
|
|
162
|
+
print(finding.check_id, finding.path, finding.message)
|
|
163
|
+
|
|
164
|
+
# Tuned run:
|
|
165
|
+
result = check_output(doc, CheckConfig(max_state_age_hours=4,
|
|
166
|
+
required_paths=("output.url",)))
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## MCP server
|
|
170
|
+
|
|
171
|
+
FailLoud also ships as an [MCP](https://modelcontextprotocol.io) server, so an agent or orchestrator can verify a sub-agent's output *inside the loop*, one tool call before acting on it:
|
|
172
|
+
|
|
173
|
+
```json
|
|
174
|
+
{
|
|
175
|
+
"mcpServers": {
|
|
176
|
+
"failloud": { "command": "failloud-mcp" }
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Tools: `check_output` (all checks, JSON result), `check_placeholder`, `check_evidence`, `check_stale_state`, `check_counts`, `check_completeness`, `failure_report` (the Markdown report), `create_verification_receipt`, and `verify_receipt_chain`. Works with both the 1.x and 2.x Python MCP SDKs.
|
|
182
|
+
|
|
183
|
+
## Design principles
|
|
184
|
+
|
|
185
|
+
- **Deterministic.** Same input, same verdict, every time. No model in the verdict path.
|
|
186
|
+
- **Zero-dependency core.** A verification tool should not expand your supply chain.
|
|
187
|
+
- **Fail loud, specifically.** Every finding carries a JSON path, a plain-language explanation, and the numbers behind it. "Something seems off" is not a finding.
|
|
188
|
+
- **Honest scope.** Heuristics are labelled as heuristics. A claim with no itemized record is *unverifiable*, and FailLoud says so instead of inventing a verdict.
|
|
189
|
+
|
|
190
|
+
## Development
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
pip install -e ".[mcp,dev]"
|
|
194
|
+
python -m pytest # 90 tests, including a live MCP stdio end-to-end test
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## Roadmap
|
|
198
|
+
|
|
199
|
+
- PyPI release and a public GitHub home (this repo is release-ready; publishing is a maintainer decision, not a code gap).
|
|
200
|
+
- Framework adapters that capture outputs directly from popular agent frameworks instead of reading files.
|
|
201
|
+
- A `failloud watch` mode for streaming/runner integration.
|
|
202
|
+
|
|
203
|
+
## License
|
|
204
|
+
|
|
205
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "failloud"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Verification harness for LLM agent outputs — catches the silent failures agents ship as 'done'."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Naveen Yalakala" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"llm",
|
|
15
|
+
"agents",
|
|
16
|
+
"verification",
|
|
17
|
+
"evaluation",
|
|
18
|
+
"mcp",
|
|
19
|
+
"reliability",
|
|
20
|
+
"silent-failures",
|
|
21
|
+
]
|
|
22
|
+
classifiers = [
|
|
23
|
+
"Development Status :: 4 - Beta",
|
|
24
|
+
"Intended Audience :: Developers",
|
|
25
|
+
"License :: OSI Approved :: MIT License",
|
|
26
|
+
"Programming Language :: Python :: 3",
|
|
27
|
+
"Programming Language :: Python :: 3.10",
|
|
28
|
+
"Programming Language :: Python :: 3.11",
|
|
29
|
+
"Programming Language :: Python :: 3.12",
|
|
30
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
32
|
+
]
|
|
33
|
+
# The core is stdlib-only on purpose: a verification tool should add no
|
|
34
|
+
# supply-chain surface of its own.
|
|
35
|
+
dependencies = []
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
mcp = ["mcp>=1.2"]
|
|
39
|
+
dev = ["pytest>=7"]
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
failloud = "failloud.cli:main"
|
|
43
|
+
failloud-mcp = "failloud.mcp_server:main"
|
|
44
|
+
|
|
45
|
+
[project.urls]
|
|
46
|
+
Homepage = "https://github.com/Yalakala-Naveen/failloud"
|
|
47
|
+
Issues = "https://github.com/Yalakala-Naveen/failloud/issues"
|
|
48
|
+
|
|
49
|
+
[tool.setuptools.packages.find]
|
|
50
|
+
where = ["src"]
|
|
51
|
+
|
|
52
|
+
[tool.pytest.ini_options]
|
|
53
|
+
testpaths = ["tests"]
|
failloud-0.2.0/setup.cfg
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""FailLoud — a verification harness for LLM agent outputs.
|
|
2
|
+
|
|
3
|
+
Agents rarely fail loudly. They return ``status: done`` with an empty
|
|
4
|
+
payload, claim an action no artifact backs, plan from state that went stale
|
|
5
|
+
two steps ago, report counts their own itemized record contradicts, and
|
|
6
|
+
ship deliverables with silent holes in them. FailLoud reads the output the
|
|
7
|
+
way a skeptical reviewer would and produces a failure report instead of a
|
|
8
|
+
shrug.
|
|
9
|
+
|
|
10
|
+
Quick start::
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from failloud import check_output
|
|
14
|
+
|
|
15
|
+
result = check_output(json.load(open("agent_output.json")))
|
|
16
|
+
if not result.passed:
|
|
17
|
+
print(result.verdict) # "FAIL"
|
|
18
|
+
for finding in result.errors:
|
|
19
|
+
print(finding.check_id, finding.path, finding.message)
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from .engine import run_checks, run_single
|
|
27
|
+
from .models import CheckConfig, CheckOutcome, CheckResult, Finding, Severity
|
|
28
|
+
from .receipts import Receipt, create_receipt, verify_chain, verify_receipt
|
|
29
|
+
|
|
30
|
+
__version__ = "0.2.0"
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"check_output",
|
|
34
|
+
"run_checks",
|
|
35
|
+
"run_single",
|
|
36
|
+
"CheckConfig",
|
|
37
|
+
"CheckOutcome",
|
|
38
|
+
"CheckResult",
|
|
39
|
+
"Finding",
|
|
40
|
+
"Severity",
|
|
41
|
+
"Receipt",
|
|
42
|
+
"create_receipt",
|
|
43
|
+
"verify_receipt",
|
|
44
|
+
"verify_chain",
|
|
45
|
+
"__version__",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def check_output(doc: Any, config: CheckConfig | None = None) -> CheckResult:
|
|
50
|
+
"""Verify one agent output document. The main entry point."""
|
|
51
|
+
return run_checks(doc, config)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""The check registry.
|
|
2
|
+
|
|
3
|
+
Each module in this package detects one class of silent failure and exposes
|
|
4
|
+
``CHECK_ID``, ``NAME``, ``DESCRIPTION`` and ``check(doc, config)``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from types import ModuleType
|
|
10
|
+
|
|
11
|
+
from . import completeness, counts, evidence, placeholder, stale
|
|
12
|
+
|
|
13
|
+
MODULES: list[ModuleType] = [placeholder, evidence, stale, counts, completeness]
|
|
14
|
+
|
|
15
|
+
BY_ID: dict[str, ModuleType] = {m.CHECK_ID: m for m in MODULES}
|
|
16
|
+
|
|
17
|
+
__all__ = ["MODULES", "BY_ID"]
|