failloud 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. failloud-0.2.0/LICENSE +21 -0
  2. failloud-0.2.0/PKG-INFO +232 -0
  3. failloud-0.2.0/README.md +205 -0
  4. failloud-0.2.0/pyproject.toml +53 -0
  5. failloud-0.2.0/setup.cfg +4 -0
  6. failloud-0.2.0/src/failloud/__init__.py +51 -0
  7. failloud-0.2.0/src/failloud/checks/__init__.py +17 -0
  8. failloud-0.2.0/src/failloud/checks/completeness.py +241 -0
  9. failloud-0.2.0/src/failloud/checks/counts.py +169 -0
  10. failloud-0.2.0/src/failloud/checks/evidence.py +182 -0
  11. failloud-0.2.0/src/failloud/checks/placeholder.py +187 -0
  12. failloud-0.2.0/src/failloud/checks/stale.py +184 -0
  13. failloud-0.2.0/src/failloud/cli.py +190 -0
  14. failloud-0.2.0/src/failloud/engine.py +44 -0
  15. failloud-0.2.0/src/failloud/mcp_server.py +204 -0
  16. failloud-0.2.0/src/failloud/models.py +140 -0
  17. failloud-0.2.0/src/failloud/receipts.py +321 -0
  18. failloud-0.2.0/src/failloud/report.py +110 -0
  19. failloud-0.2.0/src/failloud/util.py +148 -0
  20. failloud-0.2.0/src/failloud.egg-info/PKG-INFO +232 -0
  21. failloud-0.2.0/src/failloud.egg-info/SOURCES.txt +32 -0
  22. failloud-0.2.0/src/failloud.egg-info/dependency_links.txt +1 -0
  23. failloud-0.2.0/src/failloud.egg-info/entry_points.txt +3 -0
  24. failloud-0.2.0/src/failloud.egg-info/requires.txt +6 -0
  25. failloud-0.2.0/src/failloud.egg-info/top_level.txt +1 -0
  26. failloud-0.2.0/tests/test_cli.py +76 -0
  27. failloud-0.2.0/tests/test_completeness.py +90 -0
  28. failloud-0.2.0/tests/test_counts.py +65 -0
  29. failloud-0.2.0/tests/test_engine_and_report.py +92 -0
  30. failloud-0.2.0/tests/test_evidence.py +74 -0
  31. failloud-0.2.0/tests/test_mcp.py +99 -0
  32. failloud-0.2.0/tests/test_placeholder.py +71 -0
  33. failloud-0.2.0/tests/test_receipts.py +200 -0
  34. failloud-0.2.0/tests/test_stale.py +85 -0
failloud-0.2.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Naveen Yalakala
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,232 @@
1
+ Metadata-Version: 2.4
2
+ Name: failloud
3
+ Version: 0.2.0
4
+ Summary: Verification harness for LLM agent outputs — catches the silent failures agents ship as 'done'.
5
+ Author: Naveen Yalakala
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/Yalakala-Naveen/failloud
8
+ Project-URL: Issues, https://github.com/Yalakala-Naveen/failloud/issues
9
+ Keywords: llm,agents,verification,evaluation,mcp,reliability,silent-failures
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Software Development :: Quality Assurance
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Requires-Python: >=3.10
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Provides-Extra: mcp
23
+ Requires-Dist: mcp>=1.2; extra == "mcp"
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest>=7; extra == "dev"
26
+ Dynamic: license-file
27
+
28
+ # FailLoud
29
+
30
+ **A verification harness for LLM agent outputs. It catches the silent failures agents ship as `done`.**
31
+
32
+ Agents rarely fail loudly. They return `status: "done"` with an empty payload. They claim they sent, posted, and published — and nothing in the output proves any of it happened. They plan from state that went stale two steps ago, report counts their own itemized record contradicts, and deliver files with silent holes inside them. Every dashboard stays green, because the agent graded its own homework.
33
+
34
+ FailLoud reads an agent's output the way a skeptical reviewer would — and produces a **failure report** instead of a shrug.
35
+
36
+ > Built by a Forward Deployed Engineer from production failures — every check in this tool exists because a real agent failed this way in production and the miss became a rule.
37
+
38
+ **▶ Try it in your browser: [FailLoud Playground](https://yalakala-naveen.github.io/failloud/)** — paste an agent output, get the failure report. No install, nothing uploaded.
39
+
40
+ ## The five silent failures
41
+
42
+ | Check | Name | What it catches |
43
+ |---|---|---|
44
+ | FL001 | `placeholder_shipped` | Output declared done while the payload is empty or still holds draft scaffolding — lorem ipsum, unrendered `{{template}}` braces, "coming soon". *A page once went live, launch marked complete, with placeholder copy still in the body. Nothing crashed; every signal said success.* |
45
+ | FL002 | `unbacked_claim` | A claimed action (sent, posted, published, updated…) with no evidence — no message id, URL, receipt, or artifact. *A run once reported a batch as sent while a bounce sat unrecorded; "sent" was asserted by the same component that did the sending.* |
46
+ | FL003 | `stale_state` | Work based on state/context already older than allowed, state with no freshness timestamp at all, or a state-version skew between handover and run. *A pipeline planned flawlessly from a snapshot of a world that no longer existed.* |
47
+ | FL004 | `count_mismatch` | A reported count that disagrees with the itemized record in the same document, or a total that isn't the sum of its parts. *Reports counted actions attempted; audiences read them as actions completed. Only the list was ever true.* |
48
+ | FL005 | `missing_components` | Required fields present-but-empty, and dead stretches inside segmented deliverables (zero-length segments, silent gaps). *A delivered video passed every existence check while one segment inside it was silent.* |
49
+
50
+ These are not eval scores. FailLoud never asks a model whether an output "looks good" — every verdict is deterministic, reproducible, and explainable line by line. No LLM sits in the verdict path.
51
+
52
+ ## Install
53
+
54
+ ```bash
55
+ pip install failloud # CLI + library (stdlib-only core, zero dependencies)
56
+ pip install "failloud[mcp]" # + the MCP server
57
+ ```
58
+
59
+ Or from a clone: `pip install .` (add `.[mcp]` for the server).
60
+
61
+ ## As an agent skill
62
+
63
+ The same five checks also ship as a [`SKILL.md`](skill/SKILL.md) — a judgment playbook an agent runs on its own output before claiming done, for Claude Code, Cursor, Copilot and other SKILL.md-compatible agents. Listing copy and worked examples live in [`skill/`](skill/).
64
+
65
+ ## 60-second demo
66
+
67
+ The repo ships a realistic agent output that commits all five failures (`examples/failing_output.json`) and its corrected twin (`examples/passing_output.json`):
68
+
69
+ ```console
70
+ $ failloud check examples/failing_output.json
71
+ FAILLOUD — agent output verification
72
+ Source: examples/failing_output.json
73
+ Verdict: FAIL (8 error(s), 1 warning(s))
74
+
75
+ [FL001] placeholder_shipped — FAIL
76
+ ✗ $.output.body — Placeholder content shipped as final output: matched 'coming soon', 'lorem ipsum', '{{', '}}'.
77
+ [FL002] unbacked_claim — FAIL
78
+ ✗ $.claims[1] — Claimed action 'card_published' has no evidence — no id, URL, receipt, or artifact in the claim backs that it happened.
79
+ [FL003] stale_state — FAIL
80
+ ✗ $.state.context_updated_at — State timestamp is 77.1h old at run time (limit 24h) — the run worked from context older than allowed.
81
+ ✗ $ — Run based itself on state version 9 but the state it carried is version 7 — the handover and the work disagree about the world.
82
+ [FL004] count_mismatch — FAIL
83
+ ✗ $.counts.delivered — Count for 'delivered' is reported as 12 but the itemized record at $.items.delivered contains 9 item(s) — the report overstates what the record supports.
84
+ ✗ $.counts.sent — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
85
+ [FL005] missing_components — FAIL
86
+ ! $.output.archive_url — Field 'archive_url' is present but empty while its siblings carry content — verify it was meant to be blank.
87
+ ✗ $.segments[1] — Segment 1 has zero length (4s–4s) — a component that occupies a slot and contains nothing.
88
+ ✗ $.segments[2] — Dead stretch of 5.5s before segment 2 (gap limit 2s) — the deliverable goes silent where content should be.
89
+
90
+ $ echo $?
91
+ 1
92
+
93
+ $ failloud check examples/passing_output.json
94
+ FAILLOUD — agent output verification
95
+ Source: examples/passing_output.json
96
+ Verdict: PASS (0 error(s), 0 warning(s))
97
+
98
+ [FL001] placeholder_shipped — PASS
99
+ [FL002] unbacked_claim — PASS
100
+ [FL003] stale_state — PASS
101
+ [FL004] count_mismatch — PASS
102
+ [FL005] missing_components — PASS
103
+ No silent failures found. The output backs its claims.
104
+ ```
105
+
106
+ Exit codes are the CI contract: **0** pass, **1** verification failed (the agent output failed, not the tool), **2** the tool itself could not run.
107
+
108
+ ### The failure report
109
+
110
+ `--format md` produces the artifact you attach to a run, an issue, or a PR:
111
+
112
+ ```console
113
+ $ failloud check examples/failing_output.json --format md --out report.md
114
+ ```
115
+
116
+ ```markdown
117
+ # FAILURE REPORT
118
+
119
+ **Verdict:** **FAIL** — 8 error(s), 1 warning(s)
120
+
121
+ ## Checks
122
+
123
+ | Check | Name | Result | Findings |
124
+ |---|---|---|---|
125
+ | FL001 | placeholder_shipped | FAIL | 1 |
126
+ | FL002 | unbacked_claim | FAIL | 1 |
127
+ | FL003 | stale_state | FAIL | 2 |
128
+ | FL004 | count_mismatch | FAIL | 2 |
129
+ | FL005 | missing_components | FAIL | 3 |
130
+
131
+ ## Findings
132
+
133
+ ### FL004 — count_mismatch
134
+
135
+ *Why it matters: The claimed number is what travels into dashboards and decisions; the itemized record is the only part that was ever true.*
136
+
137
+ - **ERROR** `$.counts.sent` — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
138
+ ```
139
+
140
+ ## Verification receipts (new in 0.2.0)
141
+
142
+ A report says what FailLoud found; a **receipt proves it**. Seal any run into a hash-chained receipt file:
143
+
144
+ ```console
145
+ $ failloud check examples/failing_output.json --receipt-out receipts.jsonl
146
+ $ failloud check examples/passing_output.json --receipt-out receipts.jsonl
147
+ $ failloud verify-receipts receipts.jsonl
148
+ Receipt chain OK — 2 receipt(s) verified in receipts.jsonl.
149
+ ```
150
+
151
+ Each receipt embeds the full result and two SHA-256 hashes: `result_hash` seals the result and its source, and `receipt_hash` seals every receipt field including `prev_hash` — the previous receipt's hash. Edit a verdict, erase a finding, drop or reorder a receipt, and `verify-receipts` (exit 1) names the break. Anyone holding the chain can re-verify it offline, stdlib only — the audit trail an orchestrator needs before one agent's "done" becomes the next agent's input. In Python: `create_receipt`, `verify_receipt`, `verify_chain`; over MCP: `create_verification_receipt`, `verify_receipt_chain`.
152
+
153
+ ## What to feed it
154
+
155
+ FailLoud works on **any JSON an agent produces** — the checks are shape-tolerant heuristics, not a schema straitjacket. They verify the most when the output follows this envelope (every field optional; each one present is one more thing that gets checked instead of trusted):
156
+
157
+ ```json
158
+ {
159
+ "task": "Send the weekly digest and publish the summary card",
160
+ "status": "done",
161
+ "generated_at": "2026-11-11T14:05:00Z",
162
+ "summary": "Sent the digest to 9 subscribers and published the summary card.",
163
+ "claims": [
164
+ {"action": "email_sent", "target": "subscriber-list",
165
+ "evidence": {"message_id": "msg_8f42"}},
166
+ {"action": "card_published", "target": "community-feed",
167
+ "evidence": {"post_id": "post_2481", "url": "https://…/posts/2481"}}
168
+ ],
169
+ "counts": {"sent": 9, "delivered": 9},
170
+ "items": {"sent": [ … ], "delivered": [ … ]},
171
+ "state": {"context_updated_at": "2026-11-11T13:40:00Z", "version": 9},
172
+ "based_on": {"expected_state_version": 9},
173
+ "output": {"subject": "…", "body": "…", "archive_url": "https://…"},
174
+ "segments": [{"start": 0, "end": 4}, {"start": 4, "end": 9}]
175
+ }
176
+ ```
177
+
178
+ Enforce your own required fields with `--require output.archive_url,output.body` or a JSON Schema (`--schema schema.json`; the `required`/`properties`/`items` subset is supported). Thresholds are flags: `--max-state-age-hours`, `--max-segment-gap-seconds`. Run a subset with `--only FL002,FL004`. `--format json` gives machines the same verdict.
179
+
180
+ ## Python API
181
+
182
+ ```python
183
+ import json
184
+ from failloud import check_output, CheckConfig
185
+
186
+ result = check_output(json.load(open("agent_output.json")))
187
+ if not result.passed:
188
+ for finding in result.errors:
189
+ print(finding.check_id, finding.path, finding.message)
190
+
191
+ # Tuned run:
192
+ result = check_output(doc, CheckConfig(max_state_age_hours=4,
193
+ required_paths=("output.url",)))
194
+ ```
195
+
196
+ ## MCP server
197
+
198
+ FailLoud also ships as an [MCP](https://modelcontextprotocol.io) server, so an agent or orchestrator can verify a sub-agent's output *inside the loop*, one tool call before acting on it:
199
+
200
+ ```json
201
+ {
202
+ "mcpServers": {
203
+ "failloud": { "command": "failloud-mcp" }
204
+ }
205
+ }
206
+ ```
207
+
208
+ Tools: `check_output` (all checks, JSON result), `check_placeholder`, `check_evidence`, `check_stale_state`, `check_counts`, `check_completeness`, `failure_report` (the Markdown report), `create_verification_receipt`, and `verify_receipt_chain`. Works with both the 1.x and 2.x Python MCP SDKs.
209
+
210
+ ## Design principles
211
+
212
+ - **Deterministic.** Same input, same verdict, every time. No model in the verdict path.
213
+ - **Zero-dependency core.** A verification tool should not expand your supply chain.
214
+ - **Fail loud, specifically.** Every finding carries a JSON path, a plain-language explanation, and the numbers behind it. "Something seems off" is not a finding.
215
+ - **Honest scope.** Heuristics are labelled as heuristics. A claim with no itemized record is *unverifiable*, and FailLoud says so instead of inventing a verdict.
216
+
217
+ ## Development
218
+
219
+ ```bash
220
+ pip install -e ".[mcp,dev]"
221
+ python -m pytest # 90 tests, including a live MCP stdio end-to-end test
222
+ ```
223
+
224
+ ## Roadmap
225
+
226
+ - PyPI release and a public GitHub home (this repo is release-ready; publishing is a maintainer decision, not a code gap).
227
+ - Framework adapters that capture outputs directly from popular agent frameworks instead of reading files.
228
+ - A `failloud watch` mode for streaming/runner integration.
229
+
230
+ ## License
231
+
232
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,205 @@
1
+ # FailLoud
2
+
3
+ **A verification harness for LLM agent outputs. It catches the silent failures agents ship as `done`.**
4
+
5
+ Agents rarely fail loudly. They return `status: "done"` with an empty payload. They claim they sent, posted, and published — and nothing in the output proves any of it happened. They plan from state that went stale two steps ago, report counts their own itemized record contradicts, and deliver files with silent holes inside them. Every dashboard stays green, because the agent graded its own homework.
6
+
7
+ FailLoud reads an agent's output the way a skeptical reviewer would — and produces a **failure report** instead of a shrug.
8
+
9
+ > Built by a Forward Deployed Engineer from production failures — every check in this tool exists because a real agent failed this way in production and the miss became a rule.
10
+
11
+ **▶ Try it in your browser: [FailLoud Playground](https://yalakala-naveen.github.io/failloud/)** — paste an agent output, get the failure report. No install, nothing uploaded.
12
+
13
+ ## The five silent failures
14
+
15
+ | Check | Name | What it catches |
16
+ |---|---|---|
17
+ | FL001 | `placeholder_shipped` | Output declared done while the payload is empty or still holds draft scaffolding — lorem ipsum, unrendered `{{template}}` braces, "coming soon". *A page once went live, launch marked complete, with placeholder copy still in the body. Nothing crashed; every signal said success.* |
18
+ | FL002 | `unbacked_claim` | A claimed action (sent, posted, published, updated…) with no evidence — no message id, URL, receipt, or artifact. *A run once reported a batch as sent while a bounce sat unrecorded; "sent" was asserted by the same component that did the sending.* |
19
+ | FL003 | `stale_state` | Work based on state/context already older than allowed, state with no freshness timestamp at all, or a state-version skew between handover and run. *A pipeline planned flawlessly from a snapshot of a world that no longer existed.* |
20
+ | FL004 | `count_mismatch` | A reported count that disagrees with the itemized record in the same document, or a total that isn't the sum of its parts. *Reports counted actions attempted; audiences read them as actions completed. Only the list was ever true.* |
21
+ | FL005 | `missing_components` | Required fields present-but-empty, and dead stretches inside segmented deliverables (zero-length segments, silent gaps). *A delivered video passed every existence check while one segment inside it was silent.* |
22
+
23
+ These are not eval scores. FailLoud never asks a model whether an output "looks good" — every verdict is deterministic, reproducible, and explainable line by line. No LLM sits in the verdict path.
24
+
25
+ ## Install
26
+
27
+ ```bash
28
+ pip install failloud # CLI + library (stdlib-only core, zero dependencies)
29
+ pip install "failloud[mcp]" # + the MCP server
30
+ ```
31
+
32
+ Or from a clone: `pip install .` (add `.[mcp]` for the server).
33
+
34
+ ## As an agent skill
35
+
36
+ The same five checks also ship as a [`SKILL.md`](skill/SKILL.md) — a judgment playbook an agent runs on its own output before claiming done, for Claude Code, Cursor, Copilot and other SKILL.md-compatible agents. Listing copy and worked examples live in [`skill/`](skill/).
37
+
38
+ ## 60-second demo
39
+
40
+ The repo ships a realistic agent output that commits all five failures (`examples/failing_output.json`) and its corrected twin (`examples/passing_output.json`):
41
+
42
+ ```console
43
+ $ failloud check examples/failing_output.json
44
+ FAILLOUD — agent output verification
45
+ Source: examples/failing_output.json
46
+ Verdict: FAIL (8 error(s), 1 warning(s))
47
+
48
+ [FL001] placeholder_shipped — FAIL
49
+ ✗ $.output.body — Placeholder content shipped as final output: matched 'coming soon', 'lorem ipsum', '{{', '}}'.
50
+ [FL002] unbacked_claim — FAIL
51
+ ✗ $.claims[1] — Claimed action 'card_published' has no evidence — no id, URL, receipt, or artifact in the claim backs that it happened.
52
+ [FL003] stale_state — FAIL
53
+ ✗ $.state.context_updated_at — State timestamp is 77.1h old at run time (limit 24h) — the run worked from context older than allowed.
54
+ ✗ $ — Run based itself on state version 9 but the state it carried is version 7 — the handover and the work disagree about the world.
55
+ [FL004] count_mismatch — FAIL
56
+ ✗ $.counts.delivered — Count for 'delivered' is reported as 12 but the itemized record at $.items.delivered contains 9 item(s) — the report overstates what the record supports.
57
+ ✗ $.counts.sent — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
58
+ [FL005] missing_components — FAIL
59
+ ! $.output.archive_url — Field 'archive_url' is present but empty while its siblings carry content — verify it was meant to be blank.
60
+ ✗ $.segments[1] — Segment 1 has zero length (4s–4s) — a component that occupies a slot and contains nothing.
61
+ ✗ $.segments[2] — Dead stretch of 5.5s before segment 2 (gap limit 2s) — the deliverable goes silent where content should be.
62
+
63
+ $ echo $?
64
+ 1
65
+
66
+ $ failloud check examples/passing_output.json
67
+ FAILLOUD — agent output verification
68
+ Source: examples/passing_output.json
69
+ Verdict: PASS (0 error(s), 0 warning(s))
70
+
71
+ [FL001] placeholder_shipped — PASS
72
+ [FL002] unbacked_claim — PASS
73
+ [FL003] stale_state — PASS
74
+ [FL004] count_mismatch — PASS
75
+ [FL005] missing_components — PASS
76
+ No silent failures found. The output backs its claims.
77
+ ```
78
+
79
+ Exit codes are the CI contract: **0** pass, **1** verification failed (the agent output failed, not the tool), **2** the tool itself could not run.
80
+
81
+ ### The failure report
82
+
83
+ `--format md` produces the artifact you attach to a run, an issue, or a PR:
84
+
85
+ ```console
86
+ $ failloud check examples/failing_output.json --format md --out report.md
87
+ ```
88
+
89
+ ```markdown
90
+ # FAILURE REPORT
91
+
92
+ **Verdict:** **FAIL** — 8 error(s), 1 warning(s)
93
+
94
+ ## Checks
95
+
96
+ | Check | Name | Result | Findings |
97
+ |---|---|---|---|
98
+ | FL001 | placeholder_shipped | FAIL | 1 |
99
+ | FL002 | unbacked_claim | FAIL | 1 |
100
+ | FL003 | stale_state | FAIL | 2 |
101
+ | FL004 | count_mismatch | FAIL | 2 |
102
+ | FL005 | missing_components | FAIL | 3 |
103
+
104
+ ## Findings
105
+
106
+ ### FL004 — count_mismatch
107
+
108
+ *Why it matters: The claimed number is what travels into dashboards and decisions; the itemized record is the only part that was ever true.*
109
+
110
+ - **ERROR** `$.counts.sent` — Count for 'sent' is reported as 12 but the itemized record at $.items.sent contains 9 item(s) — the report overstates what the record supports.
111
+ ```
112
+
113
+ ## Verification receipts (new in 0.2.0)
114
+
115
+ A report says what FailLoud found; a **receipt proves it**. Seal any run into a hash-chained receipt file:
116
+
117
+ ```console
118
+ $ failloud check examples/failing_output.json --receipt-out receipts.jsonl
119
+ $ failloud check examples/passing_output.json --receipt-out receipts.jsonl
120
+ $ failloud verify-receipts receipts.jsonl
121
+ Receipt chain OK — 2 receipt(s) verified in receipts.jsonl.
122
+ ```
123
+
124
+ Each receipt embeds the full result and two SHA-256 hashes: `result_hash` seals the result and its source, and `receipt_hash` seals every receipt field including `prev_hash` — the previous receipt's hash. Edit a verdict, erase a finding, drop or reorder a receipt, and `verify-receipts` (exit 1) names the break. Anyone holding the chain can re-verify it offline, stdlib only — the audit trail an orchestrator needs before one agent's "done" becomes the next agent's input. In Python: `create_receipt`, `verify_receipt`, `verify_chain`; over MCP: `create_verification_receipt`, `verify_receipt_chain`.
125
+
126
+ ## What to feed it
127
+
128
+ FailLoud works on **any JSON an agent produces** — the checks are shape-tolerant heuristics, not a schema straitjacket. They verify the most when the output follows this envelope (every field optional; each one present is one more thing that gets checked instead of trusted):
129
+
130
+ ```json
131
+ {
132
+ "task": "Send the weekly digest and publish the summary card",
133
+ "status": "done",
134
+ "generated_at": "2026-11-11T14:05:00Z",
135
+ "summary": "Sent the digest to 9 subscribers and published the summary card.",
136
+ "claims": [
137
+ {"action": "email_sent", "target": "subscriber-list",
138
+ "evidence": {"message_id": "msg_8f42"}},
139
+ {"action": "card_published", "target": "community-feed",
140
+ "evidence": {"post_id": "post_2481", "url": "https://…/posts/2481"}}
141
+ ],
142
+ "counts": {"sent": 9, "delivered": 9},
143
+ "items": {"sent": [ … ], "delivered": [ … ]},
144
+ "state": {"context_updated_at": "2026-11-11T13:40:00Z", "version": 9},
145
+ "based_on": {"expected_state_version": 9},
146
+ "output": {"subject": "…", "body": "…", "archive_url": "https://…"},
147
+ "segments": [{"start": 0, "end": 4}, {"start": 4, "end": 9}]
148
+ }
149
+ ```
150
+
151
+ Enforce your own required fields with `--require output.archive_url,output.body` or a JSON Schema (`--schema schema.json`; the `required`/`properties`/`items` subset is supported). Thresholds are flags: `--max-state-age-hours`, `--max-segment-gap-seconds`. Run a subset with `--only FL002,FL004`. `--format json` gives machines the same verdict.
152
+
153
+ ## Python API
154
+
155
+ ```python
156
+ import json
157
+ from failloud import check_output, CheckConfig
158
+
159
+ result = check_output(json.load(open("agent_output.json")))
160
+ if not result.passed:
161
+ for finding in result.errors:
162
+ print(finding.check_id, finding.path, finding.message)
163
+
164
+ # Tuned run:
165
+ result = check_output(doc, CheckConfig(max_state_age_hours=4,
166
+ required_paths=("output.url",)))
167
+ ```
168
+
169
+ ## MCP server
170
+
171
+ FailLoud also ships as an [MCP](https://modelcontextprotocol.io) server, so an agent or orchestrator can verify a sub-agent's output *inside the loop*, one tool call before acting on it:
172
+
173
+ ```json
174
+ {
175
+ "mcpServers": {
176
+ "failloud": { "command": "failloud-mcp" }
177
+ }
178
+ }
179
+ ```
180
+
181
+ Tools: `check_output` (all checks, JSON result), `check_placeholder`, `check_evidence`, `check_stale_state`, `check_counts`, `check_completeness`, `failure_report` (the Markdown report), `create_verification_receipt`, and `verify_receipt_chain`. Works with both the 1.x and 2.x Python MCP SDKs.
182
+
183
+ ## Design principles
184
+
185
+ - **Deterministic.** Same input, same verdict, every time. No model in the verdict path.
186
+ - **Zero-dependency core.** A verification tool should not expand your supply chain.
187
+ - **Fail loud, specifically.** Every finding carries a JSON path, a plain-language explanation, and the numbers behind it. "Something seems off" is not a finding.
188
+ - **Honest scope.** Heuristics are labelled as heuristics. A claim with no itemized record is *unverifiable*, and FailLoud says so instead of inventing a verdict.
189
+
190
+ ## Development
191
+
192
+ ```bash
193
+ pip install -e ".[mcp,dev]"
194
+ python -m pytest # 90 tests, including a live MCP stdio end-to-end test
195
+ ```
196
+
197
+ ## Roadmap
198
+
199
+ - PyPI release and a public GitHub home (this repo is release-ready; publishing is a maintainer decision, not a code gap).
200
+ - Framework adapters that capture outputs directly from popular agent frameworks instead of reading files.
201
+ - A `failloud watch` mode for streaming/runner integration.
202
+
203
+ ## License
204
+
205
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,53 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "failloud"
7
+ version = "0.2.0"
8
+ description = "Verification harness for LLM agent outputs — catches the silent failures agents ship as 'done'."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ requires-python = ">=3.10"
12
+ authors = [{ name = "Naveen Yalakala" }]
13
+ keywords = [
14
+ "llm",
15
+ "agents",
16
+ "verification",
17
+ "evaluation",
18
+ "mcp",
19
+ "reliability",
20
+ "silent-failures",
21
+ ]
22
+ classifiers = [
23
+ "Development Status :: 4 - Beta",
24
+ "Intended Audience :: Developers",
25
+ "License :: OSI Approved :: MIT License",
26
+ "Programming Language :: Python :: 3",
27
+ "Programming Language :: Python :: 3.10",
28
+ "Programming Language :: Python :: 3.11",
29
+ "Programming Language :: Python :: 3.12",
30
+ "Topic :: Software Development :: Quality Assurance",
31
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
32
+ ]
33
+ # The core is stdlib-only on purpose: a verification tool should add no
34
+ # supply-chain surface of its own.
35
+ dependencies = []
36
+
37
+ [project.optional-dependencies]
38
+ mcp = ["mcp>=1.2"]
39
+ dev = ["pytest>=7"]
40
+
41
+ [project.scripts]
42
+ failloud = "failloud.cli:main"
43
+ failloud-mcp = "failloud.mcp_server:main"
44
+
45
+ [project.urls]
46
+ Homepage = "https://github.com/Yalakala-Naveen/failloud"
47
+ Issues = "https://github.com/Yalakala-Naveen/failloud/issues"
48
+
49
+ [tool.setuptools.packages.find]
50
+ where = ["src"]
51
+
52
+ [tool.pytest.ini_options]
53
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,51 @@
1
+ """FailLoud — a verification harness for LLM agent outputs.
2
+
3
+ Agents rarely fail loudly. They return ``status: done`` with an empty
4
+ payload, claim an action no artifact backs, plan from state that went stale
5
+ two steps ago, report counts their own itemized record contradicts, and
6
+ ship deliverables with silent holes in them. FailLoud reads the output the
7
+ way a skeptical reviewer would and produces a failure report instead of a
8
+ shrug.
9
+
10
+ Quick start::
11
+
12
+ import json
13
+ from failloud import check_output
14
+
15
+ result = check_output(json.load(open("agent_output.json")))
16
+ if not result.passed:
17
+ print(result.verdict) # "FAIL"
18
+ for finding in result.errors:
19
+ print(finding.check_id, finding.path, finding.message)
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from typing import Any
25
+
26
+ from .engine import run_checks, run_single
27
+ from .models import CheckConfig, CheckOutcome, CheckResult, Finding, Severity
28
+ from .receipts import Receipt, create_receipt, verify_chain, verify_receipt
29
+
30
+ __version__ = "0.2.0"
31
+
32
+ __all__ = [
33
+ "check_output",
34
+ "run_checks",
35
+ "run_single",
36
+ "CheckConfig",
37
+ "CheckOutcome",
38
+ "CheckResult",
39
+ "Finding",
40
+ "Severity",
41
+ "Receipt",
42
+ "create_receipt",
43
+ "verify_receipt",
44
+ "verify_chain",
45
+ "__version__",
46
+ ]
47
+
48
+
49
+ def check_output(doc: Any, config: CheckConfig | None = None) -> CheckResult:
50
+ """Verify one agent output document. The main entry point."""
51
+ return run_checks(doc, config)
@@ -0,0 +1,17 @@
1
+ """The check registry.
2
+
3
+ Each module in this package detects one class of silent failure and exposes
4
+ ``CHECK_ID``, ``NAME``, ``DESCRIPTION`` and ``check(doc, config)``.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from types import ModuleType
10
+
11
+ from . import completeness, counts, evidence, placeholder, stale
12
+
13
+ MODULES: list[ModuleType] = [placeholder, evidence, stale, counts, completeness]
14
+
15
+ BY_ID: dict[str, ModuleType] = {m.CHECK_ID: m for m in MODULES}
16
+
17
+ __all__ = ["MODULES", "BY_ID"]