agents-toolkit 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ __pycache__/
2
+ *.pyc
3
+ .pytest_cache/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .venv/
@@ -0,0 +1,29 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ahmed Mribai
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
23
+ ---
24
+
25
+ This MIT licence covers the CODE in agents_kit/ and tests/ only.
26
+
27
+ The written post-mortem sold separately (POSTMORTEM.md and the incident
28
+ analysis it contains) is not covered by this licence and may not be
29
+ redistributed. See LICENSE-POSTMORTEM.txt.
@@ -0,0 +1,13 @@
1
+ Agents Kit — post-mortem licence (the written incident analysis, sold separately).
2
+
3
+ You may read it, quote it with attribution, and apply everything in it to your own
4
+ systems, commercial or not.
5
+
6
+ You may not redistribute or resell the post-mortem itself, in whole or in
7
+ substantial part.
8
+
9
+ The CODE in agents_kit/ is separately licensed under the MIT licence (see LICENSE)
10
+ and carries no such restriction — copy it freely.
11
+
12
+ No warranty. It is extracted from a production system and tested, but your money rail
13
+ is your responsibility — verify it end to end before you rely on it.
@@ -0,0 +1,188 @@
1
+ Metadata-Version: 2.5
2
+ Name: agents-toolkit
3
+ Version: 1.0.0
4
+ Summary: Five things an autonomous agent gets wrong silently, and the code that stops each one. Dependency-free stdlib.
5
+ Project-URL: Homepage, https://get-agents-kit.com/agents-kit/
6
+ Author-email: Ahmed Mribai <hello@get-agents-kit.com>
7
+ License: MIT License
8
+
9
+ Copyright (c) 2026 Ahmed Mribai
10
+
11
+ Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ of this software and associated documentation files (the "Software"), to deal
13
+ in the Software without restriction, including without limitation the rights
14
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ copies of the Software, and to permit persons to whom the Software is
16
+ furnished to do so, subject to the following conditions:
17
+
18
+ The above copyright notice and this permission notice shall be included in all
19
+ copies or substantial portions of the Software.
20
+
21
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
+ SOFTWARE.
28
+
29
+ ---
30
+
31
+ This MIT licence covers the CODE in agents_kit/ and tests/ only.
32
+
33
+ The written post-mortem sold separately (POSTMORTEM.md and the incident
34
+ analysis it contains) is not covered by this licence and may not be
35
+ redistributed. See LICENSE-POSTMORTEM.txt.
36
+ License-File: LICENSE
37
+ License-File: LICENSE-POSTMORTEM.txt
38
+ Keywords: agents,autonomous,fulfilment,idempotency,llm,observability,quality-gates,reliability,webhook
39
+ Classifier: Development Status :: 4 - Beta
40
+ Classifier: Intended Audience :: Developers
41
+ Classifier: License :: OSI Approved :: MIT License
42
+ Classifier: Operating System :: OS Independent
43
+ Classifier: Programming Language :: Python :: 3
44
+ Classifier: Programming Language :: Python :: 3.9
45
+ Classifier: Programming Language :: Python :: 3.10
46
+ Classifier: Programming Language :: Python :: 3.11
47
+ Classifier: Programming Language :: Python :: 3.12
48
+ Classifier: Programming Language :: Python :: 3.13
49
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
50
+ Classifier: Topic :: Software Development :: Quality Assurance
51
+ Requires-Python: >=3.9
52
+ Provides-Extra: test
53
+ Requires-Dist: pytest>=7; extra == 'test'
54
+ Description-Content-Type: text/markdown
55
+
56
+ # The Agents Kit
57
+
58
+ **Five things an autonomous agent gets wrong silently, and the code that stops each one.**
59
+
60
+ Every module here came out of a system that ran continuously for months, made real decisions,
61
+ published real pages, sent real email — and earned exactly **$0**. Not because it crashed.
62
+ Because each of these five failures is invisible from the outside: the logs stay green, the
63
+ uptime stays 100%, the dashboards keep moving, and nothing works.
64
+
65
+ This is not a guide to building agents. There are plenty of those. This is the list of ways
66
+ mine failed *while reporting success*, each one reduced to a standalone module with tests.
67
+
68
+ Every module is dependency-free standard library. Install it, or copy the one file you need —
69
+ both are fine, it's MIT.
70
+
71
+ ```bash
72
+ pip install agents-toolkit
73
+ ```
74
+
75
+ The distribution is `agents-toolkit`; the import is `agents_kit`.
76
+
77
+ ```
78
+ agents_kit.webhook_rail a payment rail that cannot silently lose money
79
+ agents_kit.gates gates that fail OPEN on "couldn't measure", CLOSED on "measured bad"
80
+ agents_kit.staleness catch loops that run forever and produce nothing
81
+ agents_kit.attention budget arbitration that proves nothing starves
82
+ agents_kit.delivery idempotent fulfilment that never drops a paid order
83
+ ```
84
+
85
+ 29 tests cover all five incidents:
86
+
87
+ ```bash
88
+ pip install pytest && python -m pytest tests/ -q
89
+ ```
90
+
91
+ ---
92
+
93
+ ## The first incident, in full
94
+
95
+ Below is one of the five, complete — the diagnosis, not just the fix — so you can judge the
96
+ rest by it.
97
+
98
+ ### 1. The money rail that had never once fired
99
+
100
+ **Symptom:** a live product, a working checkout, a correct webhook handler, and $0 recorded.
101
+
102
+ Four independent breaks, each individually silent, stacked:
103
+
104
+ - The webhook was registered against **the wrong service** — a sibling backend that did not
105
+ own fulfilment. `last_sent_at: null`. It had never fired in its life.
106
+ - It was registered in **test mode**, so a real purchase would fire nothing at all.
107
+ - The signing secret was **empty** in the app's vault. `verify()` returned False for every
108
+ call, so even correctly-routed webhooks 401'd. The secret existed — in a `.env` file forty
109
+ feet away, under a different key name.
110
+ - Retries were **not deduplicated**, so anything that did get through would double-count.
111
+
112
+ Any monitoring you would plausibly have — endpoint uptime, error rate, latency — was green
113
+ throughout. The endpoint was *up*. Nothing ever asked it to do anything.
114
+
115
+ **The fix is an order of operations**, in `kit/webhook_rail.py`:
116
+
117
+ ```
118
+ verify -> parse -> dedupe -> record -> fulfil
119
+ ```
120
+
121
+ Fulfilment runs **last**, and its failure does not roll back the recording:
122
+
123
+ > A sale you recorded but failed to deliver is a support ticket.
124
+ > A sale you delivered but failed to record is a hole in your books that nothing will surface.
125
+
126
+ Three rules inside `verify()` that are each easy to get wrong:
127
+
128
+ - Verify the **raw bytes**, never a re-serialised dict. `json.loads` then `json.dumps`
129
+ reorders keys and changes whitespace; the signature will never match again.
130
+ - An **empty secret returns False**. It must never mean "skip the check" — an internet-facing
131
+ revenue route that mints on an unverified call is a free-money endpoint for whoever finds it.
132
+ - Compare with `hmac.compare_digest`, not `==`, so you do not leak the expected digest
133
+ one byte at a time.
134
+
135
+ **Test-mode money must never be income.** `Event.live` is the most important field on the
136
+ struct. Processor test orders, sandbox checkouts and your own smoke tests have to land in a
137
+ separate ledger. Mine did not, once: a hand-fired test webhook put **$98.99** into the
138
+ briefings, the P&L and the fitness function that decided what to build next. The system spent
139
+ weeks optimising toward a number that was a rehearsal.
140
+
141
+ **How to verify yours actually works — do this today:**
142
+
143
+ ```bash
144
+ # 1. bad signature must be rejected
145
+ curl -s -o /dev/null -w "%{http_code}\n" -X POST https://your-host/webhook/provider \
146
+ -H "X-Signature: deadbeef" --data-binary @payload.json # expect 401
147
+
148
+ # 2. good signature must record exactly one row
149
+ # (compute the HMAC over the exact bytes you send)
150
+ ```
151
+
152
+ Then check your processor's webhook list for `last_sent_at`. If it is null, your rail has
153
+ never run, regardless of how good the handler code is.
154
+
155
+ > **Trap I lost an hour to:** if your endpoint is behind Cloudflare, it may return **403 error
156
+ > 1010** to `Python-urllib` while accepting browsers and your processor perfectly well. Test
157
+ > with a realistic `User-Agent` or you will debug a rail that was fine.
158
+
159
+ ---
160
+
161
+ ## The other four
162
+
163
+ Same shape, all of them: the system reported success and produced nothing.
164
+
165
+ - **The gate that fails closed and deadlocks everything** — why "the user said no" and "I could
166
+ not ask the user" must be different verdicts, and what happens for months when they aren't.
167
+ - **The loop that ran for three weeks and produced nothing** — health checks answer *did it
168
+ run?*. The question that matters is *did running it change anything?*
169
+ - **The tunable that was secretly an off-switch** — one number quietly starved a third of the
170
+ system, and no error was ever raised.
171
+ - **Delivering the sales page to the person who just bought it** — the fulfilment bug that is
172
+ invisible until someone has actually paid you.
173
+
174
+ The code for all four is in this package, free, above. The full write-ups — the specific
175
+ diagnoses, the numbers each was caught by, and the method that found them — are the paid
176
+ post-mortem:
177
+
178
+ **→ [Get the full post-mortem](https://get-agents-kit.com/agents-kit/)**
179
+
180
+ That is the part that isn't reproducible from the code: what the symptom looked like, every
181
+ wrong theory ruled out first, and the one measurement that finally showed what was happening.
182
+
183
+ ---
184
+
185
+ ## Licence
186
+
187
+ Code (`agents_kit/`, `tests/`): **MIT** — copy it, ship it, sell what you build with it.
188
+ The written post-mortem is sold separately and is not MIT; see `LICENSE-POSTMORTEM.txt`.
@@ -0,0 +1,133 @@
1
+ # The Agents Kit
2
+
3
+ **Five things an autonomous agent gets wrong silently, and the code that stops each one.**
4
+
5
+ Every module here came out of a system that ran continuously for months, made real decisions,
6
+ published real pages, sent real email — and earned exactly **$0**. Not because it crashed.
7
+ Because each of these five failures is invisible from the outside: the logs stay green, the
8
+ uptime stays 100%, the dashboards keep moving, and nothing works.
9
+
10
+ This is not a guide to building agents. There are plenty of those. This is the list of ways
11
+ mine failed *while reporting success*, each one reduced to a standalone module with tests.
12
+
13
+ Every module is dependency-free standard library. Install it, or copy the one file you need —
14
+ both are fine, it's MIT.
15
+
16
+ ```bash
17
+ pip install agents-toolkit
18
+ ```
19
+
20
+ The distribution is `agents-toolkit`; the import is `agents_kit`.
21
+
22
+ ```
23
+ agents_kit.webhook_rail a payment rail that cannot silently lose money
24
+ agents_kit.gates gates that fail OPEN on "couldn't measure", CLOSED on "measured bad"
25
+ agents_kit.staleness catch loops that run forever and produce nothing
26
+ agents_kit.attention budget arbitration that proves nothing starves
27
+ agents_kit.delivery idempotent fulfilment that never drops a paid order
28
+ ```
29
+
30
+ 29 tests cover all five incidents:
31
+
32
+ ```bash
33
+ pip install pytest && python -m pytest tests/ -q
34
+ ```
35
+
36
+ ---
37
+
38
+ ## The first incident, in full
39
+
40
+ Below is one of the five, complete — the diagnosis, not just the fix — so you can judge the
41
+ rest by it.
42
+
43
+ ### 1. The money rail that had never once fired
44
+
45
+ **Symptom:** a live product, a working checkout, a correct webhook handler, and $0 recorded.
46
+
47
+ Four independent breaks, each individually silent, stacked:
48
+
49
+ - The webhook was registered against **the wrong service** — a sibling backend that did not
50
+ own fulfilment. `last_sent_at: null`. It had never fired in its life.
51
+ - It was registered in **test mode**, so a real purchase would fire nothing at all.
52
+ - The signing secret was **empty** in the app's vault. `verify()` returned False for every
53
+ call, so even correctly-routed webhooks 401'd. The secret existed — in a `.env` file forty
54
+ feet away, under a different key name.
55
+ - Retries were **not deduplicated**, so anything that did get through would double-count.
56
+
57
+ Any monitoring you would plausibly have — endpoint uptime, error rate, latency — was green
58
+ throughout. The endpoint was *up*. Nothing ever asked it to do anything.
59
+
60
+ **The fix is an order of operations**, in `kit/webhook_rail.py`:
61
+
62
+ ```
63
+ verify -> parse -> dedupe -> record -> fulfil
64
+ ```
65
+
66
+ Fulfilment runs **last**, and its failure does not roll back the recording:
67
+
68
+ > A sale you recorded but failed to deliver is a support ticket.
69
+ > A sale you delivered but failed to record is a hole in your books that nothing will surface.
70
+
71
+ Three rules inside `verify()` that are each easy to get wrong:
72
+
73
+ - Verify the **raw bytes**, never a re-serialised dict. `json.loads` then `json.dumps`
74
+ reorders keys and changes whitespace; the signature will never match again.
75
+ - An **empty secret returns False**. It must never mean "skip the check" — an internet-facing
76
+ revenue route that mints on an unverified call is a free-money endpoint for whoever finds it.
77
+ - Compare with `hmac.compare_digest`, not `==`, so you do not leak the expected digest
78
+ one byte at a time.
79
+
80
+ **Test-mode money must never be income.** `Event.live` is the most important field on the
81
+ struct. Processor test orders, sandbox checkouts and your own smoke tests have to land in a
82
+ separate ledger. Mine did not, once: a hand-fired test webhook put **$98.99** into the
83
+ briefings, the P&L and the fitness function that decided what to build next. The system spent
84
+ weeks optimising toward a number that was a rehearsal.
85
+
86
+ **How to verify yours actually works — do this today:**
87
+
88
+ ```bash
89
+ # 1. bad signature must be rejected
90
+ curl -s -o /dev/null -w "%{http_code}\n" -X POST https://your-host/webhook/provider \
91
+ -H "X-Signature: deadbeef" --data-binary @payload.json # expect 401
92
+
93
+ # 2. good signature must record exactly one row
94
+ # (compute the HMAC over the exact bytes you send)
95
+ ```
96
+
97
+ Then check your processor's webhook list for `last_sent_at`. If it is null, your rail has
98
+ never run, regardless of how good the handler code is.
99
+
100
+ > **Trap I lost an hour to:** if your endpoint is behind Cloudflare, it may return **403 error
101
+ > 1010** to `Python-urllib` while accepting browsers and your processor perfectly well. Test
102
+ > with a realistic `User-Agent` or you will debug a rail that was fine.
103
+
104
+ ---
105
+
106
+ ## The other four
107
+
108
+ Same shape, all of them: the system reported success and produced nothing.
109
+
110
+ - **The gate that fails closed and deadlocks everything** — why "the user said no" and "I could
111
+ not ask the user" must be different verdicts, and what happens for months when they aren't.
112
+ - **The loop that ran for three weeks and produced nothing** — health checks answer *did it
113
+ run?*. The question that matters is *did running it change anything?*
114
+ - **The tunable that was secretly an off-switch** — one number quietly starved a third of the
115
+ system, and no error was ever raised.
116
+ - **Delivering the sales page to the person who just bought it** — the fulfilment bug that is
117
+ invisible until someone has actually paid you.
118
+
119
+ The code for all four is in this package, free, above. The full write-ups — the specific
120
+ diagnoses, the numbers each was caught by, and the method that found them — are the paid
121
+ post-mortem:
122
+
123
+ **→ [Get the full post-mortem](https://get-agents-kit.com/agents-kit/)**
124
+
125
+ That is the part that isn't reproducible from the code: what the symptom looked like, every
126
+ wrong theory ruled out first, and the one measurement that finally showed what was happening.
127
+
128
+ ---
129
+
130
+ ## Licence
131
+
132
+ Code (`agents_kit/`, `tests/`): **MIT** — copy it, ship it, sell what you build with it.
133
+ The written post-mortem is sold separately and is not MIT; see `LICENSE-POSTMORTEM.txt`.
@@ -0,0 +1,15 @@
1
+ """Agents Kit — the parts of an autonomous agent that fail silently, and how to build them so
2
+ they don't.
3
+
4
+ Every module here was extracted from a running system after the corresponding failure had
5
+ already cost me weeks. Each one is standalone: copy the file, it has no dependencies beyond
6
+ the standard library.
7
+
8
+ webhook_rail a payment rail that cannot silently lose money
9
+ gates quality gates that fail OPEN on "couldn't measure", CLOSED on "measured bad"
10
+ staleness catch loops that run forever and produce nothing
11
+ attention budget arbitration that proves nothing starves
12
+ delivery idempotent fulfilment that never drops a paid order
13
+ """
14
+
15
+ __version__ = "1.0.0"
@@ -0,0 +1,102 @@
1
+ """agents_kit/attention.py — spend a tick's compute on what is worth doing, and prove nothing starves.
2
+
3
+ An agent with more things to do than budget needs an arbiter. Mine scored bids as
4
+
5
+ score = (0.4*value + 0.4*info + 0.2*urgency) / cost
6
+
7
+ and spent greedily under a per-tick budget. Reasonable. It also silently disabled a third of
8
+ the system for months.
9
+
10
+ The 13 registered bids cost 6.5 units in total. The budget was 3.0. Because greedy-by-ratio
11
+ picks cheap work first, the three most expensive bids — the deep reasoning, the simulation,
12
+ the dreaming, i.e. the entire reason the system was interesting — won **0 of 81** consecutive
13
+ arbitrations. Not "rarely": never. No flag said so, no log said so; the one number that
14
+ disabled them was a tunable nobody thought of as a switch.
15
+
16
+ Two defences, both here:
17
+ * `starving()` names any bid that has never won. Alarm on it.
18
+ * urgency rises the longer a bid goes unchosen, so an expensive bid eventually outbids
19
+ cheap ones instead of losing on ratio forever.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import time
25
+ from dataclasses import dataclass, field
26
+
27
+
28
+ @dataclass
29
+ class Bid:
30
+ """One candidate action competing for this tick."""
31
+
32
+ name: str
33
+ run: object # callable, invoked if chosen
34
+ value: float = 0.5 # expected payoff, 0..1
35
+ info: float = 0.5 # expected information gain, 0..1
36
+ cost: float = 1.0 # budget units consumed if it runs
37
+ stale_after_s: float = 7 * 86400 # urgency saturates at 1.0 after this long unchosen
38
+
39
+
40
+ @dataclass
41
+ class Arbiter:
42
+ budget: float = 5.0
43
+ w_value: float = 0.4
44
+ w_info: float = 0.4
45
+ w_urgency: float = 0.2
46
+ min_score: float = 0.05
47
+ _last_run: dict[str, float] = field(default_factory=dict)
48
+ _wins: dict[str, int] = field(default_factory=dict)
49
+ _seen: set[str] = field(default_factory=set)
50
+
51
+ def urgency(self, bid: Bid) -> float:
52
+ last = self._last_run.get(bid.name)
53
+ if last is None:
54
+ return 1.0 # never run ⇒ maximally urgent
55
+ return min(1.0, (time.time() - last) / max(1.0, bid.stale_after_s))
56
+
57
+ def score(self, bid: Bid) -> float:
58
+ raw = (self.w_value * bid.value + self.w_info * bid.info
59
+ + self.w_urgency * self.urgency(bid))
60
+ return raw / max(0.01, bid.cost)
61
+
62
+ def choose(self, bids: list[Bid]) -> list[Bid]:
63
+ """Greedy under budget. Pure — does not run anything."""
64
+ for b in bids:
65
+ self._seen.add(b.name)
66
+ ranked = sorted(bids, key=self.score, reverse=True)
67
+ chosen, spent = [], 0.0
68
+ for bid in ranked:
69
+ if self.score(bid) < self.min_score:
70
+ continue
71
+ if spent + bid.cost > self.budget:
72
+ continue
73
+ chosen.append(bid)
74
+ spent += bid.cost
75
+ return chosen
76
+
77
+ def run(self, bids: list[Bid]) -> dict:
78
+ chosen = self.choose(bids)
79
+ results = {}
80
+ for bid in chosen:
81
+ self._last_run[bid.name] = time.time()
82
+ self._wins[bid.name] = self._wins.get(bid.name, 0) + 1
83
+ try:
84
+ results[bid.name] = bid.run() if callable(bid.run) else None
85
+ except Exception as exc: # noqa: BLE001
86
+ results[bid.name] = f"error: {str(exc)[:120]}"
87
+ return {"chosen": [b.name for b in chosen], "results": results,
88
+ "starving": self.starving()}
89
+
90
+ def starving(self) -> list[str]:
91
+ """Bids that have competed but NEVER won. If this is non-empty, either raise the
92
+ budget or delete the bid — do not leave it registered and dead."""
93
+ return sorted(n for n in self._seen if not self._wins.get(n))
94
+
95
+ def feasible(self, bids: list[Bid]) -> dict:
96
+ """Sanity check to run at startup, not in production. Compares the budget against what
97
+ the registered bids actually cost, and warns when the most expensive can never fit."""
98
+ total = sum(b.cost for b in bids)
99
+ unaffordable = sorted(b.name for b in bids if b.cost > self.budget)
100
+ return {"budget": self.budget, "total_cost": round(total, 2),
101
+ "coverage": round(self.budget / total, 2) if total else 1.0,
102
+ "never_affordable": unaffordable}
@@ -0,0 +1,107 @@
1
+ """agents_kit/delivery.py — deliver what was paid for, exactly once, or tell somebody.
2
+
3
+ The three ways I have actually failed a paying customer:
4
+
5
+ 1. **Delivered nothing.** Revenue was recorded, no delivery step existed. The buyer paid and
6
+ got silence. Nothing in the system knew anything was wrong, because the sale looked fine.
7
+
8
+ 2. **Delivered twice.** The processor retried the webhook and the buyer got two emails.
9
+ Harmless here; if the deliverable had been a licence key or a credit top-up, it would not
10
+ have been.
11
+
12
+ 3. **Delivered a link back to the sales page they had just bought from.** This one is my
13
+ favourite, because it passed every test. The code collected "the venture's best URL", and
14
+ the venture's best URL was its own landing page. To the buyer it reads exactly like a
15
+ scam. It shipped because "a link was produced" was the success condition.
16
+
17
+ The rules encoded below:
18
+ * Idempotent per (order, product) — a retry is a no-op, not a second delivery.
19
+ * A deliverable must be *verified* to be a deliverable, not merely to exist.
20
+ * NEVER fail silently. If delivery cannot happen, record it as PENDING and alert a human.
21
+ A paid customer must never be left with nothing and no trace.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ from dataclasses import dataclass
27
+ from typing import Callable, Iterable, Protocol
28
+
29
+
30
+ class Store(Protocol):
31
+ def delivered(self, order_ref: str) -> bool: ...
32
+ def mark_delivered(self, order_ref: str, to: str, links: list[str]) -> None: ...
33
+ def mark_pending(self, order_ref: str, to: str, reason: str) -> None: ...
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class Result:
38
+ status: str # delivered | duplicate | pending
39
+ links: list[str]
40
+ reason: str = ""
41
+
42
+
43
+ def usable_links(candidates: Iterable[str], sales_pages: Iterable[str]) -> list[str]:
44
+ """Filter candidate URLs down to ones a buyer can actually use.
45
+
46
+ Rejects, in order of how badly each one burned me:
47
+ * the sales page itself (and its directory/index twin)
48
+ * `file://` and localhost paths, which resolve only on the machine that made them —
49
+ these appear when a publisher silently falls back to a local "dry" mode
50
+ * empties and duplicates, order preserved
51
+ """
52
+ blocked = set()
53
+ for page in sales_pages:
54
+ p = (page or "").strip()
55
+ if not p:
56
+ continue
57
+ blocked.add(p)
58
+ if p.endswith("/index.html"):
59
+ blocked.add(p[: -len("index.html")])
60
+ elif p.endswith("/"):
61
+ blocked.add(p + "index.html")
62
+
63
+ out, seen = [], set()
64
+ for c in candidates:
65
+ u = (c or "").strip()
66
+ if not u or u in blocked or u in seen:
67
+ continue
68
+ if u.lower().startswith(("file://", "http://127.0.0.1", "http://localhost")):
69
+ continue
70
+ seen.add(u)
71
+ out.append(u)
72
+ return out
73
+
74
+
75
+ def deliver(order_ref: str, to: str, candidates: Iterable[str], sales_pages: Iterable[str],
76
+ store: Store, send: Callable[[str, list[str]], bool]) -> Result:
77
+ """Deliver once. Never raises.
78
+
79
+ Note what happens when there is nothing good to send: it does NOT fall back to "send the
80
+ least-bad link". It records PENDING so a human finishes the job. An honest pending beats a
81
+ delivery the buyer will read as a scam — and unlike the scam, somebody finds out about it.
82
+ """
83
+ if store.delivered(order_ref):
84
+ return Result("duplicate", [], "already delivered")
85
+
86
+ if not to or "@" not in to:
87
+ store.mark_pending(order_ref, to, "no buyer address")
88
+ return Result("pending", [], "no buyer address")
89
+
90
+ links = usable_links(candidates, sales_pages)
91
+ if not links:
92
+ store.mark_pending(order_ref, to, "no usable deliverable")
93
+ return Result("pending", [], "no usable deliverable")
94
+
95
+ try:
96
+ ok = send(to, links)
97
+ except Exception as exc: # noqa: BLE001
98
+ store.mark_pending(order_ref, to, f"send raised: {str(exc)[:120]}")
99
+ return Result("pending", links, "send raised")
100
+
101
+ if not ok:
102
+ store.mark_pending(order_ref, to, "send rejected")
103
+ return Result("pending", links, "send rejected")
104
+
105
+ # Mark only AFTER a confirmed send, so a failure retries instead of being sealed as done.
106
+ store.mark_delivered(order_ref, to, links)
107
+ return Result("delivered", links)
@@ -0,0 +1,97 @@
1
+ """agents_kit/gates.py — quality gates that fail OPEN on "couldn't measure" and CLOSED on "measured bad".
2
+
3
+ The bug this exists to prevent cost me weeks of a pipeline that looked healthy and shipped
4
+ nothing.
5
+
6
+ A gate scored products with an LLM panel. Its judge looked like this:
7
+
8
+ try:
9
+ verdict = llm.judge(...)
10
+ return {"success": verdict["success"], "reuse": verdict["reuse"]}
11
+ except Exception:
12
+ return {"success": False, "reuse": False} # <-- the bug
13
+
14
+ That `except` conflates two completely different facts: "the user said no" and "I could not
15
+ ask the user". When the LLM pool started returning 429s, every judgement became a rejection,
16
+ the score pinned to 0.0, the threshold was 0.5, and the gate blocked every launch — forever.
17
+ The logs showed a busy, green system. 47 blocks, 3 passes, and nobody could see why.
18
+
19
+ Worse, it was self-sealing: no launch → no page → no traffic → no real usage data → the gate
20
+ fell back to the broken proxy → no launch.
21
+
22
+ The rule: a gate may only block on evidence. Absence of evidence is UNKNOWN, and UNKNOWN
23
+ must pass through while being loudly recorded.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ from dataclasses import dataclass, field
29
+ from enum import Enum
30
+ from typing import Callable, Sequence
31
+
32
+
33
+ class Verdict(str, Enum):
34
+ PASS = "pass" # measured, and good enough
35
+ BLOCK = "block" # measured, and not good enough
36
+ UNKNOWN = "unknown" # could not measure — passes through, but says so
37
+
38
+
39
+ @dataclass
40
+ class Judgement:
41
+ """One judge's answer. `error=True` means the judge never ran."""
42
+
43
+ ok: bool = False
44
+ error: bool = False
45
+
46
+
47
+ @dataclass
48
+ class GateResult:
49
+ verdict: Verdict
50
+ score: float
51
+ threshold: float
52
+ judged: int # how many judgements actually happened
53
+ errors: int # how many failed to run
54
+ reasons: list[str] = field(default_factory=list)
55
+
56
+ @property
57
+ def allowed(self) -> bool:
58
+ """UNKNOWN is allowed. This is the whole point of the module."""
59
+ return self.verdict in (Verdict.PASS, Verdict.UNKNOWN)
60
+
61
+
62
+ def evaluate(judges: Sequence[Callable[[], Judgement]], threshold: float = 0.5,
63
+ enabled: bool = True) -> GateResult:
64
+ """Run every judge, then decide. Never raises.
65
+
66
+ `enabled=False` gives you measure-only mode: the score is still computed and returned, but
67
+ the verdict is always PASS. Run a new gate this way for a week before you let it block —
68
+ you want to know what it *would* have done while it can't hurt you.
69
+ """
70
+ judged = errors = passed = 0
71
+ reasons: list[str] = []
72
+
73
+ for judge in judges:
74
+ try:
75
+ j = judge()
76
+ except Exception as exc: # noqa: BLE001
77
+ errors += 1
78
+ reasons.append(f"judge raised: {str(exc)[:80]}")
79
+ continue
80
+ if j.error:
81
+ errors += 1
82
+ reasons.append("judge could not run")
83
+ continue
84
+ judged += 1
85
+ passed += 1 if j.ok else 0
86
+
87
+ if judged == 0:
88
+ # Nothing was actually measured. Do NOT score this 0.0 and block on it.
89
+ return GateResult(Verdict.UNKNOWN, 0.0, threshold, 0, errors,
90
+ reasons or ["no judge produced a verdict"])
91
+
92
+ score = round(passed / judged, 4)
93
+ if not enabled:
94
+ return GateResult(Verdict.PASS, score, threshold, judged, errors,
95
+ ["gate in measure-only mode"])
96
+ verdict = Verdict.PASS if score >= threshold else Verdict.BLOCK
97
+ return GateResult(verdict, score, threshold, judged, errors, reasons)
@@ -0,0 +1,83 @@
1
+ """agents_kit/staleness.py — catch the machinery that runs and produces nothing.
2
+
3
+ This is the failure mode that hides best, because every signal you normally watch says fine.
4
+
5
+ For three weeks a discovery loop logged, every single tick:
6
+
7
+ discovery scan ok :: candidates=17 recorded=0
8
+
9
+ Status `ok`. No exception, no error rate, no latency spike, uptime 100%, dashboards green.
10
+ It evaluated the same 17 candidates and recorded none of them, forever. Alongside it a
11
+ perception loop logged `new=0` for seven days and a falsification loop logged `refuted=0`
12
+ on every tick it had ever run.
13
+
14
+ Health checks answer "did it run?". Almost nothing answers "did running it change anything?" —
15
+ and for an autonomous system, a step that changes nothing is indistinguishable from a step
16
+ that never ran, except that it also burns your budget.
17
+
18
+ Track the OUTPUT DELTA, and alarm when it is zero N times running.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import time
24
+ from dataclasses import dataclass, field
25
+
26
+
27
+ @dataclass
28
+ class _Track:
29
+ zero_streak: int = 0
30
+ last_output_at: float = 0.0
31
+ total_ticks: int = 0
32
+ total_output: float = 0.0
33
+
34
+
35
+ @dataclass
36
+ class StalenessMonitor:
37
+ """Records per-task output volume and reports which tasks have gone inert.
38
+
39
+ >>> m = StalenessMonitor(patience=3)
40
+ >>> for _ in range(3): _ = m.record("discovery", produced=0)
41
+ >>> m.stale()
42
+ ['discovery']
43
+
44
+ `produced` is whatever "this tick did something" means for the task: rows written, bytes
45
+ published, decisions taken. It must be a count of NEW output — an idempotent upsert that
46
+ rewrites the same 7 rows every tick produces 0, not 7. Getting this wrong is exactly how
47
+ `found=7 stored=7` read as healthy for a month.
48
+ """
49
+
50
+ patience: int = 5
51
+ _tracks: dict[str, _Track] = field(default_factory=dict)
52
+
53
+ def record(self, task: str, produced: float) -> bool:
54
+ """Log one tick. Returns True if `task` is now considered stale."""
55
+ t = self._tracks.setdefault(task, _Track())
56
+ t.total_ticks += 1
57
+ if produced > 0:
58
+ t.zero_streak = 0
59
+ t.last_output_at = time.time()
60
+ t.total_output += produced
61
+ else:
62
+ t.zero_streak += 1
63
+ return t.zero_streak >= self.patience
64
+
65
+ def stale(self) -> list[str]:
66
+ """Every task whose last `patience` ticks all produced nothing."""
67
+ return sorted(k for k, t in self._tracks.items() if t.zero_streak >= self.patience)
68
+
69
+ def report(self) -> list[dict]:
70
+ """Full picture, worst first — drop this straight into a daily brief."""
71
+ rows = [
72
+ {
73
+ "task": k,
74
+ "zero_streak": t.zero_streak,
75
+ "ticks": t.total_ticks,
76
+ "total_output": t.total_output,
77
+ "idle_hours": round((time.time() - t.last_output_at) / 3600.0, 1)
78
+ if t.last_output_at else None,
79
+ "stale": t.zero_streak >= self.patience,
80
+ }
81
+ for k, t in self._tracks.items()
82
+ ]
83
+ return sorted(rows, key=lambda r: (-r["zero_streak"], r["task"]))
@@ -0,0 +1,144 @@
1
+ """agents_kit/webhook_rail.py — a payment webhook that cannot silently lose money.
2
+
3
+ Every failure mode here is one I hit in production, in the order I hit it.
4
+
5
+ 1. The webhook was registered against the WRONG SERVICE. It had never fired once.
6
+ 2. The signing secret was empty, so verification returned False and every call 401'd —
7
+ money arrived at the processor, the app recorded nothing, the buyer got nothing.
8
+ 3. Retries double-counted, because "did we already handle this order?" was never asked.
9
+ 4. Test-mode orders booked as real income, so the dashboard showed revenue that did not
10
+ exist and every downstream gate keyed off a lie.
11
+
12
+ The rail below is ~120 lines and closes all four. Copy it whole.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import hashlib
18
+ import hmac
19
+ import json
20
+ from dataclasses import dataclass
21
+ from typing import Callable, Protocol
22
+
23
+
24
+ class AlreadyHandled(Exception):
25
+ """Raised by a Ledger when an event id has been seen before."""
26
+
27
+
28
+ class Ledger(Protocol):
29
+ """Your storage. Two operations, both of which must be atomic."""
30
+
31
+ def seen(self, event_id: str) -> bool: ...
32
+ def record(self, event_id: str, amount: float, currency: str, live: bool) -> None: ...
33
+
34
+
35
+ @dataclass(frozen=True)
36
+ class Event:
37
+ """A normalised payment event. `live` is the single most important field on it."""
38
+
39
+ id: str
40
+ kind: str
41
+ amount: float
42
+ currency: str
43
+ email: str
44
+ live: bool # False for processor test-mode. NEVER book these as income.
45
+ reference: str # your own id (venture/product/customer) from checkout metadata
46
+ raw: dict
47
+
48
+ @property
49
+ def is_refund(self) -> bool:
50
+ return self.kind in ("order_refunded", "subscription_payment_refunded")
51
+
52
+
53
+ def verify(raw_body: bytes, signature: str, secret: str) -> bool:
54
+ """Constant-time HMAC-SHA256 check over the RAW body.
55
+
56
+ Three rules that are easy to get wrong:
57
+
58
+ * Verify the raw bytes, not a re-serialised dict. `json.loads` then `json.dumps` will
59
+ reorder keys and change whitespace, and the signature will never match again.
60
+ * An empty secret returns False. It must never mean "skip the check" — an internet-facing
61
+ revenue route that mints on an unverified call is a free-money endpoint for anyone who
62
+ finds it.
63
+ * Compare with `hmac.compare_digest`, not `==`, so the comparison does not leak the
64
+ expected digest one byte at a time.
65
+ """
66
+ if not secret or not signature:
67
+ return False
68
+ expected = hmac.new(secret.encode("utf-8"), raw_body or b"", hashlib.sha256).hexdigest()
69
+ return hmac.compare_digest(expected, signature.strip().lower())
70
+
71
+
72
+ def parse(raw_body: bytes, event_name: str) -> Event | None:
73
+ """Normalise a Lemon Squeezy payload. Adapt `attributes` for another processor.
74
+
75
+ Returns None rather than raising: a malformed body is a 400, not a 500, and it must not
76
+ take down the listener that healthy retries depend on.
77
+ """
78
+ try:
79
+ payload = json.loads((raw_body or b"").decode("utf-8", "replace"))
80
+ except Exception:
81
+ return None
82
+ if not isinstance(payload, dict):
83
+ return None
84
+
85
+ data = payload.get("data") or {}
86
+ attrs = data.get("attributes") or {}
87
+ meta = payload.get("meta") or {}
88
+ custom = meta.get("custom_data") or {}
89
+
90
+ total = attrs.get("total")
91
+ try:
92
+ amount = round(float(total) / 100.0, 2) if total is not None else 0.0
93
+ except (TypeError, ValueError):
94
+ amount = 0.0
95
+
96
+ return Event(
97
+ id=str(data.get("id") or attrs.get("identifier") or "").strip(),
98
+ kind=(event_name or meta.get("event_name") or "").strip(),
99
+ amount=amount,
100
+ currency=str(attrs.get("currency") or "USD").upper(),
101
+ email=str(attrs.get("user_email") or "").strip(),
102
+ # Missing test_mode is treated as LIVE. A processor always sends it; a payload without
103
+ # it is not a test order, and defaulting to "test" would hide real income.
104
+ live=not _truthy(attrs.get("test_mode")),
105
+ reference=str(custom.get("reference") or custom.get("venture_id") or "").strip(),
106
+ raw=payload,
107
+ )
108
+
109
+
110
+ def _truthy(value) -> bool:
111
+ return str(value).strip().lower() in ("1", "true", "yes", "t")
112
+
113
+
114
+ def handle(raw_body: bytes, signature: str, event_name: str, secret: str,
115
+ ledger: Ledger, fulfil: Callable[[Event], None]) -> tuple[int, dict]:
116
+ """The whole rail. Returns (http_status, body) — wire it straight into your handler.
117
+
118
+ Order matters and is not negotiable:
119
+ verify → parse → dedupe → record → fulfil
120
+
121
+ Fulfilment runs LAST and its failure does not roll back the recording. A sale you recorded
122
+ but failed to deliver is a support ticket; a sale you delivered but failed to record is a
123
+ hole in your books that nothing will ever surface.
124
+ """
125
+ if not verify(raw_body, signature, secret):
126
+ return 401, {"error": "bad signature"}
127
+
128
+ event = parse(raw_body, event_name)
129
+ if event is None or not event.id:
130
+ return 400, {"error": "unparseable"}
131
+
132
+ if ledger.seen(event.id):
133
+ # 200, not 409: the processor is retrying and a non-2xx makes it retry harder.
134
+ return 200, {"status": "duplicate", "id": event.id}
135
+
136
+ amount = -event.amount if event.is_refund else event.amount
137
+ ledger.record(event.id, amount, event.currency, event.live)
138
+
139
+ try:
140
+ fulfil(event)
141
+ except Exception as exc: # noqa: BLE001 — reported, not raised
142
+ return 200, {"status": "recorded", "fulfilment": f"pending: {exc}"[:200]}
143
+
144
+ return 200, {"status": "ok", "recorded": amount, "live": event.live}
@@ -0,0 +1,47 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "agents-toolkit"
7
+ version = "1.0.0"
8
+ description = "Five things an autonomous agent gets wrong silently, and the code that stops each one. Dependency-free stdlib."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { file = "LICENSE" }
12
+ authors = [{ name = "Ahmed Mribai", email = "hello@get-agents-kit.com" }]
13
+ keywords = [
14
+ "agents", "llm", "autonomous", "webhook", "idempotency",
15
+ "reliability", "observability", "fulfilment", "quality-gates",
16
+ ]
17
+ classifiers = [
18
+ "Development Status :: 4 - Beta",
19
+ "Intended Audience :: Developers",
20
+ "License :: OSI Approved :: MIT License",
21
+ "Operating System :: OS Independent",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.9",
24
+ "Programming Language :: Python :: 3.10",
25
+ "Programming Language :: Python :: 3.11",
26
+ "Programming Language :: Python :: 3.12",
27
+ "Programming Language :: Python :: 3.13",
28
+ "Topic :: Software Development :: Libraries :: Python Modules",
29
+ "Topic :: Software Development :: Quality Assurance",
30
+ ]
31
+ # Deliberately empty, and it is the selling point: every module is standard library only.
32
+ dependencies = []
33
+
34
+ [project.urls]
35
+ Homepage = "https://get-agents-kit.com/agents-kit/"
36
+ # Source deliberately omitted until the public repo exists — a 404 on the PyPI
37
+ # sidebar reads worse than no link at all.
38
+
39
+ [project.optional-dependencies]
40
+ test = ["pytest>=7"]
41
+
42
+ [tool.hatch.build.targets.wheel]
43
+ packages = ["agents_kit"]
44
+
45
+ [tool.hatch.build.targets.sdist]
46
+ # The post-mortem is sold separately — it must never ride along in a published artifact.
47
+ exclude = ["POSTMORTEM.md", "dist/", ".pytest_cache/", "__pycache__/"]
@@ -0,0 +1,290 @@
1
+ """Every claim the kit makes, as an executable test.
2
+
3
+ These are the regression tests for the production incidents the kit exists to prevent.
4
+ If you change a module, these must stay green.
5
+ """
6
+
7
+ import hashlib
8
+ import hmac
9
+ import json
10
+ import time
11
+
12
+ from agents_kit import attention, delivery, gates, staleness, webhook_rail
13
+
14
+ SECRET = "test-signing-secret"
15
+
16
+
17
+ def _signed(payload: dict) -> tuple[bytes, str]:
18
+ raw = json.dumps(payload).encode()
19
+ sig = hmac.new(SECRET.encode(), raw, hashlib.sha256).hexdigest()
20
+ return raw, sig
21
+
22
+
23
+ def _order(oid="ord_1", total=9899, test_mode=False, email="buyer@example.com"):
24
+ return {"meta": {"custom_data": {"reference": "42"}},
25
+ "data": {"id": oid, "attributes": {"total": total, "test_mode": test_mode,
26
+ "user_email": email, "currency": "USD"}}}
27
+
28
+
29
+ class FakeLedger:
30
+ def __init__(self):
31
+ self.rows = {}
32
+
33
+ def seen(self, event_id):
34
+ return event_id in self.rows
35
+
36
+ def record(self, event_id, amount, currency, live):
37
+ self.rows[event_id] = {"amount": amount, "currency": currency, "live": live}
38
+
39
+
40
+ # -- webhook rail -------------------------------------------------------------
41
+
42
+ def test_bad_signature_is_rejected():
43
+ raw, _ = _signed(_order())
44
+ status, _ = webhook_rail.handle(raw, "deadbeef", "order_created", SECRET,
45
+ FakeLedger(), lambda e: None)
46
+ assert status == 401
47
+
48
+
49
+ def test_empty_secret_never_passes():
50
+ """An unset secret must not mean 'skip verification'."""
51
+ raw, sig = _signed(_order())
52
+ assert webhook_rail.verify(raw, sig, "") is False
53
+ status, _ = webhook_rail.handle(raw, sig, "order_created", "", FakeLedger(), lambda e: None)
54
+ assert status == 401
55
+
56
+
57
+ def test_good_signature_records_once():
58
+ raw, sig = _signed(_order())
59
+ ledger = FakeLedger()
60
+ status, body = webhook_rail.handle(raw, sig, "order_created", SECRET, ledger, lambda e: None)
61
+ assert status == 200 and body["recorded"] == 98.99
62
+ assert ledger.rows["ord_1"]["live"] is True
63
+
64
+
65
+ def test_retry_does_not_double_count():
66
+ raw, sig = _signed(_order())
67
+ ledger = FakeLedger()
68
+ webhook_rail.handle(raw, sig, "order_created", SECRET, ledger, lambda e: None)
69
+ status, body = webhook_rail.handle(raw, sig, "order_created", SECRET, ledger, lambda e: None)
70
+ assert status == 200 and body["status"] == "duplicate"
71
+ assert len(ledger.rows) == 1
72
+
73
+
74
+ def test_test_mode_is_not_live_revenue():
75
+ """The rehearsal must never be reportable as income."""
76
+ raw, sig = _signed(_order(test_mode=True))
77
+ ledger = FakeLedger()
78
+ webhook_rail.handle(raw, sig, "order_created", SECRET, ledger, lambda e: None)
79
+ assert ledger.rows["ord_1"]["live"] is False
80
+
81
+
82
+ def test_refund_books_negative():
83
+ raw, sig = _signed(_order(oid="ord_r"))
84
+ ledger = FakeLedger()
85
+ webhook_rail.handle(raw, sig, "order_refunded", SECRET, ledger, lambda e: None)
86
+ assert ledger.rows["ord_r"]["amount"] == -98.99
87
+
88
+
89
+ def test_fulfilment_failure_still_records_the_sale():
90
+ def boom(_event):
91
+ raise RuntimeError("mail server down")
92
+
93
+ raw, sig = _signed(_order())
94
+ ledger = FakeLedger()
95
+ status, body = webhook_rail.handle(raw, sig, "order_created", SECRET, ledger, boom)
96
+ assert status == 200 and "pending" in body["fulfilment"]
97
+ assert ledger.rows["ord_1"]["amount"] == 98.99 # the money is still on the books
98
+
99
+
100
+ def test_malformed_body_is_400_not_a_crash():
101
+ sig = hmac.new(SECRET.encode(), b"not json", hashlib.sha256).hexdigest()
102
+ status, _ = webhook_rail.handle(b"not json", sig, "order_created", SECRET,
103
+ FakeLedger(), lambda e: None)
104
+ assert status == 400
105
+
106
+
107
+ # -- gates --------------------------------------------------------------------
108
+
109
+ def _judge(ok=True, error=False):
110
+ return lambda: gates.Judgement(ok=ok, error=error)
111
+
112
+
113
+ def test_gate_blocks_a_measured_low_score():
114
+ r = gates.evaluate([_judge(ok=False)] * 4, threshold=0.5)
115
+ assert r.verdict is gates.Verdict.BLOCK and r.allowed is False
116
+
117
+
118
+ def test_gate_passes_a_measured_high_score():
119
+ r = gates.evaluate([_judge(ok=True)] * 4, threshold=0.5)
120
+ assert r.verdict is gates.Verdict.PASS and r.allowed is True
121
+
122
+
123
+ def test_gate_returns_unknown_when_every_judge_errors():
124
+ """THE regression: an LLM outage must not read as unanimous rejection."""
125
+ r = gates.evaluate([_judge(error=True)] * 5, threshold=0.5)
126
+ assert r.verdict is gates.Verdict.UNKNOWN
127
+ assert r.allowed is True # unknown passes through
128
+ assert r.errors == 5 and r.judged == 0
129
+
130
+
131
+ def test_gate_unknown_when_a_judge_raises():
132
+ def raising():
133
+ raise TimeoutError("429 rate limited")
134
+
135
+ r = gates.evaluate([raising] * 3, threshold=0.5)
136
+ assert r.verdict is gates.Verdict.UNKNOWN and r.allowed is True
137
+
138
+
139
+ def test_partial_errors_still_score_on_what_was_measured():
140
+ r = gates.evaluate([_judge(ok=True), _judge(ok=True), _judge(error=True)], threshold=0.5)
141
+ assert r.judged == 2 and r.errors == 1
142
+ assert r.score == 1.0 and r.verdict is gates.Verdict.PASS
143
+
144
+
145
+ def test_measure_only_mode_never_blocks():
146
+ r = gates.evaluate([_judge(ok=False)] * 4, threshold=0.5, enabled=False)
147
+ assert r.verdict is gates.Verdict.PASS and r.score == 0.0
148
+
149
+
150
+ # -- staleness ----------------------------------------------------------------
151
+
152
+ def test_zero_output_streak_is_detected():
153
+ m = staleness.StalenessMonitor(patience=3)
154
+ for _ in range(2):
155
+ assert m.record("discovery", produced=0) is False
156
+ assert m.record("discovery", produced=0) is True
157
+ assert m.stale() == ["discovery"]
158
+
159
+
160
+ def test_output_resets_the_streak():
161
+ m = staleness.StalenessMonitor(patience=2)
162
+ m.record("perception", 0)
163
+ m.record("perception", 5)
164
+ assert m.stale() == []
165
+
166
+
167
+ def test_productive_task_never_flagged():
168
+ m = staleness.StalenessMonitor(patience=2)
169
+ for _ in range(10):
170
+ m.record("publisher", produced=3)
171
+ assert m.stale() == [] and m.report()[0]["total_output"] == 30
172
+
173
+
174
+ def test_report_ranks_worst_first():
175
+ m = staleness.StalenessMonitor(patience=2)
176
+ for _ in range(4):
177
+ m.record("dead", 0)
178
+ m.record("alive", 1)
179
+ assert m.report()[0]["task"] == "dead" and m.report()[0]["stale"] is True
180
+
181
+
182
+ # -- attention ----------------------------------------------------------------
183
+
184
+ def test_expensive_bid_is_reported_as_starving():
185
+ """THE regression: a bid that can never win must be visible, not silently dead."""
186
+ a = attention.Arbiter(budget=3.0)
187
+ bids = [attention.Bid("cheap", lambda: 1, cost=0.3),
188
+ attention.Bid("expensive", lambda: 1, cost=5.0)]
189
+ for _ in range(5):
190
+ a.run(bids)
191
+ assert "expensive" in a.starving()
192
+
193
+
194
+ def test_feasibility_check_flags_an_unaffordable_bid():
195
+ a = attention.Arbiter(budget=3.0)
196
+ bids = [attention.Bid("a", lambda: 1, cost=1.0), attention.Bid("b", lambda: 1, cost=4.0)]
197
+ f = a.feasible(bids)
198
+ assert f["never_affordable"] == ["b"] and f["total_cost"] == 5.0
199
+
200
+
201
+ def test_budget_is_respected():
202
+ a = attention.Arbiter(budget=1.0)
203
+ bids = [attention.Bid("b%d" % i, lambda: 1, cost=0.4) for i in range(5)]
204
+ assert len(a.choose(bids)) == 2
205
+
206
+
207
+ def test_urgency_lets_a_neglected_bid_eventually_win():
208
+ a = attention.Arbiter(budget=1.0)
209
+ fresh = attention.Bid("fresh", lambda: 1, cost=0.5, value=0.9, info=0.9)
210
+ neglected = attention.Bid("neglected", lambda: 1, cost=0.5, value=0.1, info=0.1)
211
+ a._last_run["fresh"] = time.time() # just ran
212
+ # neglected has never run, so its urgency is 1.0
213
+ assert "neglected" in [b.name for b in a.choose([fresh, neglected])]
214
+
215
+
216
+ def test_a_raising_bid_does_not_kill_the_tick():
217
+ def boom():
218
+ raise ValueError("nope")
219
+
220
+ a = attention.Arbiter(budget=5.0)
221
+ out = a.run([attention.Bid("bad", boom, cost=1.0),
222
+ attention.Bid("good", lambda: "ok", cost=1.0)])
223
+ assert out["results"]["good"] == "ok" and "error" in out["results"]["bad"]
224
+
225
+
226
+ # -- delivery -----------------------------------------------------------------
227
+
228
+ class FakeStore:
229
+ def __init__(self):
230
+ self.done, self.pending = {}, {}
231
+
232
+ def delivered(self, ref):
233
+ return ref in self.done
234
+
235
+ def mark_delivered(self, ref, to, links):
236
+ self.done[ref] = (to, links)
237
+
238
+ def mark_pending(self, ref, to, reason):
239
+ self.pending[ref] = (to, reason)
240
+
241
+
242
+ def test_sales_page_is_never_delivered():
243
+ """THE regression: the buyer must not be emailed the page they just bought from."""
244
+ links = delivery.usable_links(
245
+ ["https://x.com/product/guide.pdf", "https://x.com/sale/index.html"],
246
+ ["https://x.com/sale/index.html"])
247
+ assert links == ["https://x.com/product/guide.pdf"]
248
+
249
+
250
+ def test_sales_page_directory_twin_is_also_blocked():
251
+ assert delivery.usable_links(["https://x.com/sale/"], ["https://x.com/sale/index.html"]) == []
252
+
253
+
254
+ def test_local_paths_are_never_delivered():
255
+ assert delivery.usable_links(["file:///C:/Users/me/out.html"], []) == []
256
+ assert delivery.usable_links(["http://127.0.0.1:8765/x"], []) == []
257
+
258
+
259
+ def test_no_usable_link_records_pending_rather_than_sending_junk():
260
+ store = FakeStore()
261
+ sent = []
262
+
263
+ def send(to, links):
264
+ sent.append(links)
265
+ return True
266
+
267
+ r = delivery.deliver("ord_1", "b@example.com", ["https://x.com/sale/"],
268
+ ["https://x.com/sale/"], store, send)
269
+ assert r.status == "pending" and not sent
270
+ assert "ord_1" in store.pending
271
+
272
+
273
+ def test_happy_path_delivers_and_is_idempotent():
274
+ store, sent = FakeStore(), []
275
+
276
+ def send(to, links):
277
+ sent.append(links)
278
+ return True
279
+
280
+ args = ("ord_2", "b@example.com", ["https://x.com/guide.pdf"], ["https://x.com/sale/"])
281
+ assert delivery.deliver(*args, store, send).status == "delivered"
282
+ assert delivery.deliver(*args, store, send).status == "duplicate"
283
+ assert len(sent) == 1
284
+
285
+
286
+ def test_failed_send_is_pending_and_retryable():
287
+ store = FakeStore()
288
+ r = delivery.deliver("ord_3", "b@example.com", ["https://x.com/g.pdf"], [],
289
+ store, lambda to, links: False)
290
+ assert r.status == "pending" and not store.delivered("ord_3") # retryable, not sealed