cachecanary 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cachecanary-0.3.0 → cachecanary-0.3.2}/PKG-INFO +14 -4
- {cachecanary-0.3.0 → cachecanary-0.3.2}/README.md +12 -2
- {cachecanary-0.3.0 → cachecanary-0.3.2}/pyproject.toml +1 -1
- cachecanary-0.3.2/scripts/_fake_bedrock_stream.py +135 -0
- cachecanary-0.3.2/scripts/capture_langchain_requests.py +86 -0
- cachecanary-0.3.2/scripts/capture_litellm_requests.py +117 -0
- cachecanary-0.3.2/scripts/capture_strands_requests.py +68 -0
- cachecanary-0.3.2/scripts/live_langchain_checks.py +203 -0
- cachecanary-0.3.2/scripts/live_litellm_checks.py +170 -0
- cachecanary-0.3.2/scripts/live_strands_checks.py +245 -0
- cachecanary-0.3.2/scripts/og-cards/og-langchain.html +9 -0
- cachecanary-0.3.2/scripts/og-cards/og-litellm.html +9 -0
- cachecanary-0.3.2/scripts/og-cards/og-strands.html +9 -0
- cachecanary-0.3.2/scripts/og-cards/og.html +9 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/site/404.html +3 -0
- cachecanary-0.3.2/site/apple-touch-icon.png +0 -0
- cachecanary-0.3.2/site/favicon.ico +0 -0
- cachecanary-0.3.2/site/icon-192.png +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/site/index.html +76 -6
- cachecanary-0.3.2/site/langchain/index.html +265 -0
- cachecanary-0.3.2/site/litellm/index.html +260 -0
- cachecanary-0.3.2/site/llms.txt +36 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/site/main.js +19 -0
- cachecanary-0.3.2/site/og-langchain.png +0 -0
- cachecanary-0.3.2/site/og-litellm.png +0 -0
- cachecanary-0.3.2/site/og-strands.png +0 -0
- cachecanary-0.3.2/site/og.png +0 -0
- cachecanary-0.3.2/site/robots.txt +4 -0
- cachecanary-0.3.2/site/sitemap.xml +19 -0
- cachecanary-0.3.2/site/strands/index.html +322 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/site/style.css +26 -3
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/__init__.py +1 -1
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/diff.py +6 -1
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/models.py +2 -1
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/pricing.py +3 -2
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary.egg-info/PKG-INFO +14 -4
- cachecanary-0.3.2/src/cachecanary.egg-info/SOURCES.txt +94 -0
- cachecanary-0.3.2/tests/fixtures/langchain/date-day1.json +58 -0
- cachecanary-0.3.2/tests/fixtures/langchain/date-day2.json +58 -0
- cachecanary-0.3.2/tests/fixtures/langchain/edit-call8.json +303 -0
- cachecanary-0.3.2/tests/fixtures/langchain/edit-call9.json +333 -0
- cachecanary-0.3.2/tests/fixtures/langchain/invoke-turn1.json +37 -0
- cachecanary-0.3.2/tests/fixtures/langchain/invoke-turn2.json +185 -0
- cachecanary-0.3.2/tests/fixtures/litellm/agent-turn1.json +51 -0
- cachecanary-0.3.2/tests/fixtures/litellm/agent-turn2.json +289 -0
- cachecanary-0.3.2/tests/fixtures/litellm/chat-turn6.json +120 -0
- cachecanary-0.3.2/tests/fixtures/litellm/chat-turn7.json +136 -0
- cachecanary-0.3.2/tests/fixtures/litellm/with-download.json +26 -0
- cachecanary-0.3.2/tests/fixtures/litellm/without-download.json +25 -0
- cachecanary-0.3.2/tests/fixtures/strands/agent-turn1.json +55 -0
- cachecanary-0.3.2/tests/fixtures/strands/agent-turn2.json +285 -0
- cachecanary-0.3.2/tests/fixtures/strands/chat-turn20.json +375 -0
- cachecanary-0.3.2/tests/fixtures/strands/chat-turn21.json +375 -0
- cachecanary-0.3.2/tests/fixtures/strands/ttl-order.json +57 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_cachecanary.py +30 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_pricing.py +9 -3
- cachecanary-0.3.2/tests/test_site.py +489 -0
- cachecanary-0.3.0/site/robots.txt +0 -2
- cachecanary-0.3.0/src/cachecanary.egg-info/SOURCES.txt +0 -54
- cachecanary-0.3.0/tests/test_site.py +0 -73
- {cachecanary-0.3.0 → cachecanary-0.3.2}/.github/workflows/ci.yml +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/.github/workflows/publish.yml +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/LICENSE +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/MANIFEST.in +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/NOTICE +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/action.yml +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/examples/request_bad.json +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/examples/request_good.json +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/examples/request_next.json +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/examples/request_too_short.json +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/scripts/check_site.mjs +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/scripts/live_logs_test.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/scripts/live_lookback_boundary.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/scripts/live_test.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/scripts/live_v03_checks.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/scripts/make_log_fixture.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/scripts/serve_site.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/setup.cfg +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/site/_headers +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/site/favicon.svg +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/cli.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/github.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/lint.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/logs.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/probe.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/request.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary/usage.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary.egg-info/dependency_links.txt +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary.egg-info/entry_points.txt +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary.egg-info/requires.txt +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/src/cachecanary.egg-info/top_level.txt +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/fixtures/bedrock_invocation_logs_2026-10.json +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_action.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_edge_cases.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_github.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_readme.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_scripts_and_formats.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/tests/test_v03_rules.py +0 -0
- {cachecanary-0.3.0 → cachecanary-0.3.2}/wrangler.jsonc +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cachecanary
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock, not when the bill arrives.
|
|
5
5
|
Author-email: Haris Farooq <harisf98@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
7
|
-
Project-URL: Homepage, https://
|
|
7
|
+
Project-URL: Homepage, https://cachecanary.com
|
|
8
8
|
Project-URL: Source, https://github.com/Haarris/cachecanary
|
|
9
9
|
Project-URL: Issues, https://github.com/Haarris/cachecanary/issues
|
|
10
10
|
Keywords: claude,anthropic,amazon-bedrock,bedrock,prompt-caching,llm,cost,observability,ci
|
|
@@ -42,6 +42,12 @@ Cache reads on Bedrock cost about a tenth of normal input tokens, so a long syst
|
|
|
42
42
|
|
|
43
43
|
LiteLLM hit this in [July 2026](https://docs.litellm.ai/blog/bedrock-invoke-prompt-caching-incident). After an upgrade, Claude Code traffic on Bedrock went from about 90% cache hits to 25-45%, and daily spend went up 2-3x for six days before anyone noticed. Anthropic has a cache diagnostics feature, but it only works on their own API, not on Bedrock.
|
|
44
44
|
|
|
45
|
+
I built CacheCanary after running into these failures while running Claude agents in production at [Ruby](https://heyruby.io).
|
|
46
|
+
|
|
47
|
+
I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
|
|
48
|
+
|
|
49
|
+
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
|
|
50
|
+
|
|
45
51
|
What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
|
|
46
52
|
|
|
47
53
|
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
|
|
@@ -170,10 +176,14 @@ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe`
|
|
|
170
176
|
|
|
171
177
|
## What's next
|
|
172
178
|
|
|
173
|
-
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
179
|
+
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
174
180
|
|
|
175
|
-
|
|
181
|
+
Using CacheCanary? I'd love to hear how it's going, good or bad, and whether you'd want the dashboard. Email me at [hello@cachecanary.com](mailto:hello@cachecanary.com). I read every message.
|
|
182
|
+
|
|
183
|
+
Vertex AI support is also planned.
|
|
176
184
|
|
|
177
185
|
## License
|
|
178
186
|
|
|
179
187
|
Apache 2.0. See [LICENSE](https://github.com/Haarris/cachecanary/blob/main/LICENSE). Copyright 2026 Haris Farooq.
|
|
188
|
+
|
|
189
|
+
Made by [Haris Farooq](https://www.linkedin.com/in/haris-farooq), an engineer who runs Claude agents on Bedrock in production.
|
|
@@ -9,6 +9,12 @@ Cache reads on Bedrock cost about a tenth of normal input tokens, so a long syst
|
|
|
9
9
|
|
|
10
10
|
LiteLLM hit this in [July 2026](https://docs.litellm.ai/blog/bedrock-invoke-prompt-caching-incident). After an upgrade, Claude Code traffic on Bedrock went from about 90% cache hits to 25-45%, and daily spend went up 2-3x for six days before anyone noticed. Anthropic has a cache diagnostics feature, but it only works on their own API, not on Bedrock.
|
|
11
11
|
|
|
12
|
+
I built CacheCanary after running into these failures while running Claude agents in production at [Ruby](https://heyruby.io).
|
|
13
|
+
|
|
14
|
+
I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
|
|
15
|
+
|
|
16
|
+
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
|
|
17
|
+
|
|
12
18
|
What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
|
|
13
19
|
|
|
14
20
|
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
|
|
@@ -137,10 +143,14 @@ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe`
|
|
|
137
143
|
|
|
138
144
|
## What's next
|
|
139
145
|
|
|
140
|
-
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
146
|
+
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
141
147
|
|
|
142
|
-
|
|
148
|
+
Using CacheCanary? I'd love to hear how it's going, good or bad, and whether you'd want the dashboard. Email me at [hello@cachecanary.com](mailto:hello@cachecanary.com). I read every message.
|
|
149
|
+
|
|
150
|
+
Vertex AI support is also planned.
|
|
143
151
|
|
|
144
152
|
## License
|
|
145
153
|
|
|
146
154
|
Apache 2.0. See [LICENSE](https://github.com/Haarris/cachecanary/blob/main/LICENSE). Copyright 2026 Haris Farooq.
|
|
155
|
+
|
|
156
|
+
Made by [Haris Farooq](https://www.linkedin.com/in/haris-farooq), an engineer who runs Claude agents on Bedrock in production.
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""A local stand-in for bedrock-runtime that speaks the real Converse, ConverseStream and InvokeModel formats.
|
|
2
|
+
|
|
3
|
+
Lets an agent framework run its real agent loop offline: each request body is saved, and replies follow
|
|
4
|
+
a script of turns: "tools:N" (N parallel toolUse blocks for a lookup_order tool), "text" (a short answer),
|
|
5
|
+
"text:N" (an N-line answer) or "use:NAME:JSON" (one call to the named tool, e.g. a structured-output tool).
|
|
6
|
+
InvokeModel replies use the Anthropic Messages format. Point a client at it with endpoint_url=Fake().url
|
|
7
|
+
and fake credentials.
|
|
8
|
+
"""
|
|
9
|
+
import json, struct, threading, zlib
|
|
10
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
11
|
+
|
|
12
|
+
def _header(name, value):
|
|
13
|
+
n, v = name.encode(), value.encode()
|
|
14
|
+
return struct.pack(">B", len(n)) + n + b"\x07" + struct.pack(">H", len(v)) + v
|
|
15
|
+
|
|
16
|
+
def event(event_type, payload):
|
|
17
|
+
headers = _header(":event-type", event_type) + _header(":content-type", "application/json") + _header(":message-type", "event")
|
|
18
|
+
body = json.dumps(payload).encode()
|
|
19
|
+
total = 12 + len(headers) + len(body) + 4
|
|
20
|
+
prelude = struct.pack(">II", total, len(headers))
|
|
21
|
+
prelude += struct.pack(">I", zlib.crc32(prelude) & 0xFFFFFFFF)
|
|
22
|
+
msg = prelude + headers + body
|
|
23
|
+
return msg + struct.pack(">I", zlib.crc32(msg) & 0xFFFFFFFF)
|
|
24
|
+
|
|
25
|
+
class Fake:
|
|
26
|
+
def __init__(self):
|
|
27
|
+
self.requests, self.script, self.turn = [], [], 0
|
|
28
|
+
fake = self
|
|
29
|
+
class H(BaseHTTPRequestHandler):
|
|
30
|
+
def do_POST(self):
|
|
31
|
+
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])) or b"{}")
|
|
32
|
+
path = self.path
|
|
33
|
+
if path.endswith("/count-tokens"):
|
|
34
|
+
return self._json({"inputTokens": 100})
|
|
35
|
+
fake.requests.append({"path": path, "body": body})
|
|
36
|
+
step = fake.script[fake.turn] if fake.turn < len(fake.script) else "text"
|
|
37
|
+
fake.turn += 1
|
|
38
|
+
if path.endswith("/invoke"):
|
|
39
|
+
return self._json(fake.messages_api(step, fake.turn))
|
|
40
|
+
if path.endswith("/invoke-with-response-stream"):
|
|
41
|
+
self.send_response(200)
|
|
42
|
+
self.send_header("Content-Type", "application/vnd.amazon.eventstream")
|
|
43
|
+
data = fake.messages_stream(step, fake.turn)
|
|
44
|
+
self.send_header("Content-Length", str(len(data)))
|
|
45
|
+
self.end_headers(); self.wfile.write(data); return
|
|
46
|
+
if path.endswith("/converse-stream"):
|
|
47
|
+
self.send_response(200)
|
|
48
|
+
self.send_header("Content-Type", "application/vnd.amazon.eventstream")
|
|
49
|
+
data = fake.stream(step, fake.turn)
|
|
50
|
+
self.send_header("Content-Length", str(len(data)))
|
|
51
|
+
self.end_headers(); self.wfile.write(data)
|
|
52
|
+
else:
|
|
53
|
+
self._json(fake.whole(step, fake.turn))
|
|
54
|
+
def _json(self, obj):
|
|
55
|
+
d = json.dumps(obj).encode()
|
|
56
|
+
self.send_response(200); self.send_header("Content-Type", "application/json")
|
|
57
|
+
self.send_header("Content-Length", str(len(d))); self.end_headers(); self.wfile.write(d)
|
|
58
|
+
def log_message(self, *a): pass
|
|
59
|
+
self.server = ThreadingHTTPServer(("127.0.0.1", 0), H)
|
|
60
|
+
threading.Thread(target=self.server.serve_forever, daemon=True).start()
|
|
61
|
+
self.url = f"http://127.0.0.1:{self.server.server_port}"
|
|
62
|
+
|
|
63
|
+
def reset(self, script):
|
|
64
|
+
self.requests, self.script, self.turn = [], list(script), 0
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def _calls(step, turn):
|
|
68
|
+
n = int(step.split(":")[1])
|
|
69
|
+
return [(f"t{turn}_{i}", {"id": f"{turn}-{i}"}) for i in range(n)]
|
|
70
|
+
|
|
71
|
+
def stream(self, step, turn):
|
|
72
|
+
out = event("messageStart", {"role": "assistant"})
|
|
73
|
+
if step.startswith("use:"):
|
|
74
|
+
_, name, raw = step.split(":", 2)
|
|
75
|
+
out += event("contentBlockStart", {"start": {"toolUse": {"toolUseId": f"u{turn}", "name": name}}, "contentBlockIndex": 0})
|
|
76
|
+
out += event("contentBlockDelta", {"delta": {"toolUse": {"input": raw}}, "contentBlockIndex": 0})
|
|
77
|
+
out += event("contentBlockStop", {"contentBlockIndex": 0})
|
|
78
|
+
out += event("messageStop", {"stopReason": "tool_use"})
|
|
79
|
+
out += event("metadata", {"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}})
|
|
80
|
+
return out
|
|
81
|
+
if step.startswith("tools:"):
|
|
82
|
+
for i, (tid, inp) in enumerate(self._calls(step, turn)):
|
|
83
|
+
out += event("contentBlockStart", {"start": {"toolUse": {"toolUseId": tid, "name": "lookup_order"}}, "contentBlockIndex": i})
|
|
84
|
+
out += event("contentBlockDelta", {"delta": {"toolUse": {"input": json.dumps(inp)}}, "contentBlockIndex": i})
|
|
85
|
+
out += event("contentBlockStop", {"contentBlockIndex": i})
|
|
86
|
+
stop = "tool_use"
|
|
87
|
+
else:
|
|
88
|
+
text = "All done." if step == "text" else " ".join(f"Line {j} of answer {turn}: order {j} shipped on day {j}." for j in range(int(step.split(":")[1])))
|
|
89
|
+
out += event("contentBlockDelta", {"delta": {"text": text}, "contentBlockIndex": 0})
|
|
90
|
+
out += event("contentBlockStop", {"contentBlockIndex": 0})
|
|
91
|
+
stop = "end_turn"
|
|
92
|
+
out += event("messageStop", {"stopReason": stop})
|
|
93
|
+
out += event("metadata", {"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}})
|
|
94
|
+
return out
|
|
95
|
+
|
|
96
|
+
def whole(self, step, turn):
|
|
97
|
+
if step.startswith("use:"):
|
|
98
|
+
_, name, raw = step.split(":", 2)
|
|
99
|
+
return {"output": {"message": {"role": "assistant", "content": [{"toolUse": {"toolUseId": f"u{turn}", "name": name, "input": json.loads(raw)}}]}},
|
|
100
|
+
"stopReason": "tool_use", "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}
|
|
101
|
+
if step.startswith("tools:"):
|
|
102
|
+
content = [{"toolUse": {"toolUseId": tid, "name": "lookup_order", "input": inp}} for tid, inp in self._calls(step, turn)]
|
|
103
|
+
stop = "tool_use"
|
|
104
|
+
else:
|
|
105
|
+
text = "All done." if step == "text" else " ".join(f"Line {j} of answer {turn}: order {j} shipped on day {j}." for j in range(int(step.split(":")[1])))
|
|
106
|
+
content, stop = [{"text": text}], "end_turn"
|
|
107
|
+
return {"output": {"message": {"role": "assistant", "content": content}}, "stopReason": stop,
|
|
108
|
+
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}
|
|
109
|
+
|
|
110
|
+
# InvokeModel (Anthropic Messages API) replies, for ChatBedrock and other invoke-route clients.
|
|
111
|
+
def messages_api(self, step, turn):
|
|
112
|
+
if step.startswith("tools:"):
|
|
113
|
+
content = [{"type": "tool_use", "id": tid, "name": "lookup_order", "input": inp} for tid, inp in self._calls(step, turn)]
|
|
114
|
+
stop = "tool_use"
|
|
115
|
+
else:
|
|
116
|
+
text = "All done." if step == "text" else " ".join(f"Line {j} of answer {turn}." for j in range(int(step.split(":")[1])))
|
|
117
|
+
content, stop = [{"type": "text", "text": text}], "end_turn"
|
|
118
|
+
return {"id": f"msg_{turn}", "type": "message", "role": "assistant", "model": "claude", "content": content,
|
|
119
|
+
"stop_reason": stop, "stop_sequence": None, "usage": {"input_tokens": 1, "output_tokens": 1}}
|
|
120
|
+
|
|
121
|
+
def messages_stream(self, step, turn):
|
|
122
|
+
import base64
|
|
123
|
+
whole = self.messages_api(step, turn)
|
|
124
|
+
events = [{"type": "message_start", "message": {**whole, "content": [], "stop_reason": None}}]
|
|
125
|
+
for i, block in enumerate(whole["content"]):
|
|
126
|
+
if block["type"] == "tool_use":
|
|
127
|
+
events.append({"type": "content_block_start", "index": i, "content_block": {"type": "tool_use", "id": block["id"], "name": block["name"], "input": {}}})
|
|
128
|
+
events.append({"type": "content_block_delta", "index": i, "delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}})
|
|
129
|
+
else:
|
|
130
|
+
events.append({"type": "content_block_start", "index": i, "content_block": {"type": "text", "text": ""}})
|
|
131
|
+
events.append({"type": "content_block_delta", "index": i, "delta": {"type": "text_delta", "text": block["text"]}})
|
|
132
|
+
events.append({"type": "content_block_stop", "index": i})
|
|
133
|
+
events.append({"type": "message_delta", "delta": {"stop_reason": whole["stop_reason"]}, "usage": {"output_tokens": 1}})
|
|
134
|
+
events.append({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 1, "outputTokenCount": 1, "invocationLatency": 1, "firstByteLatency": 1}})
|
|
135
|
+
return b"".join(event("chunk", {"bytes": base64.b64encode(json.dumps(e).encode()).decode()}) for e in events)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Capture the exact Bedrock requests LangChain builds, offline, for tests/fixtures/langchain/.
|
|
2
|
+
|
|
3
|
+
LangChain runs its real agent loop (create_agent, tool calls, middleware) against a local stand-in for
|
|
4
|
+
bedrock-runtime (scripts/_fake_bedrock_stream.py) with fake credentials, so nothing leaves the machine.
|
|
5
|
+
|
|
6
|
+
pip install langchain-aws==1.8.1 langchain==1.4.3 langgraph==1.2.14
|
|
7
|
+
python scripts/capture_langchain_requests.py tests/fixtures/langchain
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
16
|
+
from _fake_bedrock_stream import Fake # noqa: E402
|
|
17
|
+
|
|
18
|
+
os.environ.update(AWS_ACCESS_KEY_ID="AKIAFAKE", AWS_SECRET_ACCESS_KEY="fake", AWS_DEFAULT_REGION="us-west-2")
|
|
19
|
+
|
|
20
|
+
from langchain.agents import create_agent # noqa: E402
|
|
21
|
+
from langchain.agents.middleware import ClearToolUsesEdit, ContextEditingMiddleware, dynamic_prompt # noqa: E402
|
|
22
|
+
from langchain_aws import ChatBedrock, ChatBedrockConverse # noqa: E402
|
|
23
|
+
from langchain_aws.middleware.prompt_caching import BedrockPromptCachingMiddleware # noqa: E402
|
|
24
|
+
from langchain_core.tools import tool # noqa: E402
|
|
25
|
+
|
|
26
|
+
MODEL = "us.anthropic.claude-sonnet-4-6"
|
|
27
|
+
SYSTEM = "You are a support agent for Acme. " + " ".join(
|
|
28
|
+
f"Rule {i}: answer politely and cite section {i}." for i in range(150))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@tool
|
|
32
|
+
def lookup_order(id: str) -> str:
|
|
33
|
+
"""Look up an order."""
|
|
34
|
+
return f"order {id} shipped on day {id}. " * 8
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main() -> int:
|
|
38
|
+
out = Path(sys.argv[1])
|
|
39
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
fake = Fake()
|
|
41
|
+
|
|
42
|
+
def converse():
|
|
43
|
+
return ChatBedrockConverse(model=MODEL, region_name="us-west-2", endpoint_url=fake.url, max_tokens=50)
|
|
44
|
+
|
|
45
|
+
def run(script, model, prompts, middleware, system=SYSTEM):
|
|
46
|
+
fake.reset(script)
|
|
47
|
+
agent = create_agent(model=model, tools=[lookup_order], system_prompt=system, middleware=middleware)
|
|
48
|
+
for prompt in prompts:
|
|
49
|
+
agent.invoke({"messages": [{"role": "user", "content": prompt}]})
|
|
50
|
+
return [request["body"] for request in fake.requests]
|
|
51
|
+
|
|
52
|
+
def save(name, body):
|
|
53
|
+
(out / name).write_text(json.dumps(body, indent=2) + "\n")
|
|
54
|
+
|
|
55
|
+
# Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls.
|
|
56
|
+
chat_bedrock = ChatBedrock(model=MODEL, region_name="us-west-2", endpoint_url=fake.url, model_kwargs={"max_tokens": 50})
|
|
57
|
+
bodies = run(["tools:11", "text"], chat_bedrock, ["Check my orders."], [BedrockPromptCachingMiddleware()])
|
|
58
|
+
save("invoke-turn1.json", bodies[0])
|
|
59
|
+
save("invoke-turn2.json", bodies[1])
|
|
60
|
+
|
|
61
|
+
# Finding 3: context editing that clears a sliding window of old tool results on every call.
|
|
62
|
+
editing = ContextEditingMiddleware(edits=[ClearToolUsesEdit(trigger=600, keep=3)])
|
|
63
|
+
bodies = run(["tools:1"] * 9 + ["text"], converse(), ["Check my orders."], [editing, BedrockPromptCachingMiddleware()])
|
|
64
|
+
save("edit-call8.json", bodies[8])
|
|
65
|
+
save("edit-call9.json", bodies[9])
|
|
66
|
+
|
|
67
|
+
# Smaller things: today's date in a dynamic system prompt.
|
|
68
|
+
day = {"date": "2026-10-07"}
|
|
69
|
+
|
|
70
|
+
@dynamic_prompt
|
|
71
|
+
def with_date(request):
|
|
72
|
+
return f"Today is {day['date']}. " + SYSTEM
|
|
73
|
+
|
|
74
|
+
fake.reset(["text", "text"])
|
|
75
|
+
agent = create_agent(model=converse(), tools=[lookup_order], middleware=[with_date, BedrockPromptCachingMiddleware()])
|
|
76
|
+
agent.invoke({"messages": [{"role": "user", "content": "Hi"}]})
|
|
77
|
+
day["date"] = "2026-10-08"
|
|
78
|
+
agent.invoke({"messages": [{"role": "user", "content": "Hi"}]})
|
|
79
|
+
save("date-day1.json", fake.requests[0]["body"])
|
|
80
|
+
save("date-day2.json", fake.requests[1]["body"])
|
|
81
|
+
print(f"Saved {len(list(out.glob('*.json')))} requests to {out}")
|
|
82
|
+
return 0
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
sys.exit(main())
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Capture the exact Bedrock requests LiteLLM builds, offline, for tests/fixtures/litellm/.
|
|
2
|
+
|
|
3
|
+
A small local server stands in for bedrock-runtime and saves every request body. LiteLLM is pointed at it
|
|
4
|
+
with aws_bedrock_runtime_endpoint and fake credentials, so nothing leaves the machine.
|
|
5
|
+
|
|
6
|
+
pip install litellm==1.104.0
|
|
7
|
+
python scripts/capture_litellm_requests.py tests/fixtures/litellm
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import threading
|
|
15
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
REPLY = {"output": {"message": {"role": "assistant", "content": [{"text": "OK"}]}}, "stopReason": "end_turn",
|
|
19
|
+
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}
|
|
20
|
+
SONNET_46 = "bedrock/converse/us.anthropic.claude-sonnet-4-6"
|
|
21
|
+
SYSTEM = "You are a support agent for Acme. " + " ".join(
|
|
22
|
+
f"Rule {i}: answer politely and cite section {i}." for i in range(150))
|
|
23
|
+
TOOLS = [{"type": "function", "function": {"name": "lookup_order", "description": "Look up an order.",
|
|
24
|
+
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}}]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def serve():
|
|
28
|
+
bodies = []
|
|
29
|
+
|
|
30
|
+
class Handler(BaseHTTPRequestHandler):
|
|
31
|
+
def do_POST(self):
|
|
32
|
+
bodies.append(json.loads(self.rfile.read(int(self.headers["Content-Length"]))))
|
|
33
|
+
data = json.dumps(REPLY).encode()
|
|
34
|
+
self.send_response(200)
|
|
35
|
+
self.send_header("Content-Type", "application/json")
|
|
36
|
+
self.send_header("Content-Length", str(len(data)))
|
|
37
|
+
self.end_headers()
|
|
38
|
+
self.wfile.write(data)
|
|
39
|
+
|
|
40
|
+
def log_message(self, *args):
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
server = HTTPServer(("127.0.0.1", 0), Handler)
|
|
44
|
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
45
|
+
return f"http://127.0.0.1:{server.server_port}", bodies
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def tool_round(n, prefix="t"):
|
|
49
|
+
calls = {"role": "assistant", "content": None, "tool_calls": [
|
|
50
|
+
{"id": f"{prefix}{i}", "type": "function",
|
|
51
|
+
"function": {"name": "lookup_order", "arguments": json.dumps({"id": f"{prefix}{i}"})}} for i in range(n)]}
|
|
52
|
+
results = [{"role": "tool", "tool_call_id": f"{prefix}{i}", "content": f"order {prefix}{i} shipped"} for i in range(n)]
|
|
53
|
+
return [calls] + results
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def capture_ttl(out: Path, name: str, local_map: bool) -> None:
|
|
57
|
+
"""The cost map is read at import, so each setting needs its own process."""
|
|
58
|
+
env = dict(os.environ)
|
|
59
|
+
if local_map:
|
|
60
|
+
env["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
|
61
|
+
subprocess.run([sys.executable, __file__, "--ttl-only", str(out / name)], env=env, check=True)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def main() -> int:
|
|
65
|
+
os.environ.update(AWS_ACCESS_KEY_ID="AKIAFAKE", AWS_SECRET_ACCESS_KEY="fake", AWS_REGION_NAME="us-west-2")
|
|
66
|
+
endpoint, bodies = serve()
|
|
67
|
+
import litellm
|
|
68
|
+
|
|
69
|
+
def capture(path: Path, **kwargs) -> None:
|
|
70
|
+
count = len(bodies)
|
|
71
|
+
litellm.completion(aws_bedrock_runtime_endpoint=endpoint, max_tokens=5, **kwargs)
|
|
72
|
+
assert len(bodies) == count + 1, f"no request captured for {path.name}"
|
|
73
|
+
path.write_text(json.dumps(bodies[-1], indent=2) + "\n")
|
|
74
|
+
|
|
75
|
+
if sys.argv[1] == "--ttl-only":
|
|
76
|
+
capture(Path(sys.argv[2]), model="bedrock/converse/us.anthropic.claude-sonnet-5-5",
|
|
77
|
+
messages=[{"role": "system", "content": [{"type": "text", "text": SYSTEM,
|
|
78
|
+
"cache_control": {"type": "ephemeral", "ttl": "1h"}}]},
|
|
79
|
+
{"role": "user", "content": "Hi"}])
|
|
80
|
+
return 0
|
|
81
|
+
|
|
82
|
+
out = Path(sys.argv[1])
|
|
83
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
84
|
+
last_message = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]
|
|
85
|
+
|
|
86
|
+
# Finding 2: an agent step with 12 parallel tool calls.
|
|
87
|
+
turn1 = [{"role": "system", "content": SYSTEM},
|
|
88
|
+
{"role": "user", "content": "Check orders 1 to 12 and tell me which shipped."}]
|
|
89
|
+
capture(out / "agent-turn1.json", model=SONNET_46, tools=TOOLS, messages=turn1,
|
|
90
|
+
cache_control_injection_points=last_message)
|
|
91
|
+
capture(out / "agent-turn2.json", model=SONNET_46, tools=TOOLS, messages=turn1 + tool_round(12),
|
|
92
|
+
cache_control_injection_points=last_message)
|
|
93
|
+
|
|
94
|
+
# Finding 1: role "user" on a chat with more than three user messages.
|
|
95
|
+
by_role = [{"location": "message", "role": "system"}, {"location": "message", "role": "user"}]
|
|
96
|
+
chat = [{"role": "system", "content": SYSTEM}]
|
|
97
|
+
for i in range(5):
|
|
98
|
+
chat += [{"role": "user", "content": f"Question {i} about order {i}."},
|
|
99
|
+
{"role": "assistant", "content": f"Answer {i}."}]
|
|
100
|
+
capture(out / "chat-turn6.json", model=SONNET_46, messages=chat + [{"role": "user", "content": "Last question."}],
|
|
101
|
+
cache_control_injection_points=by_role)
|
|
102
|
+
capture(out / "chat-turn7.json", model=SONNET_46, cache_control_injection_points=by_role,
|
|
103
|
+
messages=chat + [{"role": "user", "content": "Last question."}, {"role": "assistant", "content": "Answer 5."},
|
|
104
|
+
{"role": "user", "content": "One more question."}])
|
|
105
|
+
|
|
106
|
+
# Finding 4: the same Sonnet 5.5 request with the downloaded and the built-in price list.
|
|
107
|
+
capture_ttl(out, "with-download.json", local_map=False)
|
|
108
|
+
capture_ttl(out, "without-download.json", local_map=True)
|
|
109
|
+
if '"ttl"' not in (out / "with-download.json").read_text():
|
|
110
|
+
sys.exit("LiteLLM couldn't download its price list (no network?), so with-download.json is wrong. Run again online.")
|
|
111
|
+
assert '"ttl"' not in (out / "without-download.json").read_text()
|
|
112
|
+
print(f"Saved {len(list(out.glob('*.json')))} requests to {out}")
|
|
113
|
+
return 0
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__":
|
|
117
|
+
sys.exit(main())
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Capture the exact Bedrock requests Strands Agents builds, offline, for tests/fixtures/strands/.
|
|
2
|
+
|
|
3
|
+
Strands runs its real agent loop (tool calls, streaming, conversation trimming) against a local stand-in
|
|
4
|
+
for bedrock-runtime (scripts/_fake_bedrock_stream.py) with fake credentials, so nothing leaves the machine.
|
|
5
|
+
|
|
6
|
+
pip install strands-agents==1.58.1
|
|
7
|
+
python scripts/capture_strands_requests.py tests/fixtures/strands
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
16
|
+
from _fake_bedrock_stream import Fake # noqa: E402
|
|
17
|
+
|
|
18
|
+
os.environ.update(AWS_ACCESS_KEY_ID="AKIAFAKE", AWS_SECRET_ACCESS_KEY="fake", AWS_REGION="us-west-2")
|
|
19
|
+
|
|
20
|
+
from strands import Agent, tool # noqa: E402
|
|
21
|
+
from strands.models import BedrockModel # noqa: E402
|
|
22
|
+
from strands.models.model import CacheConfig # noqa: E402
|
|
23
|
+
|
|
24
|
+
MODEL = "us.anthropic.claude-sonnet-4-6"
|
|
25
|
+
SYSTEM = "You are a support agent for Acme. " + " ".join(
|
|
26
|
+
f"Rule {i}: answer politely and cite section {i}." for i in range(150))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@tool
|
|
30
|
+
def lookup_order(id: str) -> str:
|
|
31
|
+
"""Look up an order."""
|
|
32
|
+
return f"order {id} shipped"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def main() -> int:
|
|
36
|
+
out = Path(sys.argv[1])
|
|
37
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
fake = Fake()
|
|
39
|
+
|
|
40
|
+
def run(script, prompts, cache_config):
|
|
41
|
+
fake.reset(script)
|
|
42
|
+
model = BedrockModel(model_id=MODEL, endpoint_url=fake.url, region_name="us-west-2", cache_config=cache_config)
|
|
43
|
+
agent = Agent(model=model, tools=[lookup_order], system_prompt=SYSTEM, callback_handler=None)
|
|
44
|
+
for prompt in prompts:
|
|
45
|
+
agent(prompt)
|
|
46
|
+
return [request["body"] for request in fake.requests]
|
|
47
|
+
|
|
48
|
+
def save(name, body):
|
|
49
|
+
(out / name).write_text(json.dumps(body, indent=2) + "\n")
|
|
50
|
+
|
|
51
|
+
auto = CacheConfig(strategy="auto")
|
|
52
|
+
# Finding 2: one agent step with 11 parallel tool calls.
|
|
53
|
+
bodies = run(["tools:11", "text"], ["Check my orders."], auto)
|
|
54
|
+
save("agent-turn1.json", bodies[0])
|
|
55
|
+
save("agent-turn2.json", bodies[1])
|
|
56
|
+
# Finding 3: a long chat with the default sliding window (40 messages); turn 21 trims the first messages.
|
|
57
|
+
bodies = run(["text"] * 22, [f"Question {i} about order {i}." for i in range(22)], auto)
|
|
58
|
+
save("chat-turn20.json", bodies[20])
|
|
59
|
+
save("chat-turn21.json", bodies[21])
|
|
60
|
+
# Finding 5: a 1-hour default lifetime with a 5-minute system prompt point.
|
|
61
|
+
bodies = run(["text"], ["Hi"], CacheConfig(strategy="auto", ttl="1h", system_prompt_ttl="5m"))
|
|
62
|
+
save("ttl-order.json", bodies[0])
|
|
63
|
+
print(f"Saved {len(list(out.glob('*.json')))} requests to {out}")
|
|
64
|
+
return 0
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
sys.exit(main())
|