cachecanary 0.3.1__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cachecanary-0.3.1 → cachecanary-0.3.2}/PKG-INFO +12 -4
- {cachecanary-0.3.1 → cachecanary-0.3.2}/README.md +10 -2
- {cachecanary-0.3.1 → cachecanary-0.3.2}/pyproject.toml +1 -1
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/_fake_bedrock_stream.py +52 -3
- cachecanary-0.3.2/scripts/capture_langchain_requests.py +86 -0
- cachecanary-0.3.2/scripts/live_langchain_checks.py +203 -0
- cachecanary-0.3.2/scripts/og-cards/og-langchain.html +9 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/404.html +3 -0
- cachecanary-0.3.2/site/apple-touch-icon.png +0 -0
- cachecanary-0.3.2/site/favicon.ico +0 -0
- cachecanary-0.3.2/site/icon-192.png +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/index.html +34 -7
- cachecanary-0.3.2/site/langchain/index.html +265 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/litellm/index.html +72 -7
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/llms.txt +1 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/main.js +19 -0
- cachecanary-0.3.2/site/og-langchain.png +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/sitemap.xml +4 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/strands/index.html +69 -4
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/style.css +16 -5
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/__init__.py +1 -1
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/models.py +2 -1
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/pricing.py +3 -2
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary.egg-info/PKG-INFO +12 -4
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary.egg-info/SOURCES.txt +14 -0
- cachecanary-0.3.2/tests/fixtures/langchain/date-day1.json +58 -0
- cachecanary-0.3.2/tests/fixtures/langchain/date-day2.json +58 -0
- cachecanary-0.3.2/tests/fixtures/langchain/edit-call8.json +303 -0
- cachecanary-0.3.2/tests/fixtures/langchain/edit-call9.json +333 -0
- cachecanary-0.3.2/tests/fixtures/langchain/invoke-turn1.json +37 -0
- cachecanary-0.3.2/tests/fixtures/langchain/invoke-turn2.json +185 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_cachecanary.py +7 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_pricing.py +9 -3
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_site.py +162 -7
- {cachecanary-0.3.1 → cachecanary-0.3.2}/.github/workflows/ci.yml +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/.github/workflows/publish.yml +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/LICENSE +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/MANIFEST.in +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/NOTICE +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/action.yml +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/examples/request_bad.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/examples/request_good.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/examples/request_next.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/examples/request_too_short.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/capture_litellm_requests.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/capture_strands_requests.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/check_site.mjs +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/live_litellm_checks.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/live_logs_test.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/live_lookback_boundary.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/live_strands_checks.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/live_test.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/live_v03_checks.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/make_log_fixture.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/og-cards/og-litellm.html +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/og-cards/og-strands.html +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/og-cards/og.html +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/scripts/serve_site.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/setup.cfg +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/_headers +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/favicon.svg +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/og-litellm.png +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/og-strands.png +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/og.png +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/site/robots.txt +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/cli.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/diff.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/github.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/lint.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/logs.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/probe.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/request.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary/usage.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary.egg-info/dependency_links.txt +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary.egg-info/entry_points.txt +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary.egg-info/requires.txt +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/src/cachecanary.egg-info/top_level.txt +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/bedrock_invocation_logs_2026-10.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/litellm/agent-turn1.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/litellm/agent-turn2.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/litellm/chat-turn6.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/litellm/chat-turn7.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/litellm/with-download.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/litellm/without-download.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/strands/agent-turn1.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/strands/agent-turn2.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/strands/chat-turn20.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/strands/chat-turn21.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/fixtures/strands/ttl-order.json +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_action.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_edge_cases.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_github.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_readme.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_scripts_and_formats.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/tests/test_v03_rules.py +0 -0
- {cachecanary-0.3.1 → cachecanary-0.3.2}/wrangler.jsonc +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cachecanary
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock, not when the bill arrives.
|
|
5
5
|
Author-email: Haris Farooq <harisf98@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
7
|
-
Project-URL: Homepage, https://
|
|
7
|
+
Project-URL: Homepage, https://cachecanary.com
|
|
8
8
|
Project-URL: Source, https://github.com/Haarris/cachecanary
|
|
9
9
|
Project-URL: Issues, https://github.com/Haarris/cachecanary/issues
|
|
10
10
|
Keywords: claude,anthropic,amazon-bedrock,bedrock,prompt-caching,llm,cost,observability,ci
|
|
@@ -42,8 +42,12 @@ Cache reads on Bedrock cost about a tenth of normal input tokens, so a long syst
|
|
|
42
42
|
|
|
43
43
|
LiteLLM hit this in [July 2026](https://docs.litellm.ai/blog/bedrock-invoke-prompt-caching-incident). After an upgrade, Claude Code traffic on Bedrock went from about 90% cache hits to 25-45%, and daily spend went up 2-3x for six days before anyone noticed. Anthropic has a cache diagnostics feature, but it only works on their own API, not on Bedrock.
|
|
44
44
|
|
|
45
|
+
I built CacheCanary after running into these failures while running Claude agents in production at [Ruby](https://heyruby.io).
|
|
46
|
+
|
|
45
47
|
I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
|
|
46
48
|
|
|
49
|
+
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
|
|
50
|
+
|
|
47
51
|
What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
|
|
48
52
|
|
|
49
53
|
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
|
|
@@ -172,10 +176,14 @@ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe`
|
|
|
172
176
|
|
|
173
177
|
## What's next
|
|
174
178
|
|
|
175
|
-
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
179
|
+
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
176
180
|
|
|
177
|
-
|
|
181
|
+
Using CacheCanary? I'd love to hear how it's going, good or bad, and whether you'd want the dashboard. Email me at [hello@cachecanary.com](mailto:hello@cachecanary.com). I read every message.
|
|
182
|
+
|
|
183
|
+
Vertex AI support is also planned.
|
|
178
184
|
|
|
179
185
|
## License
|
|
180
186
|
|
|
181
187
|
Apache 2.0. See [LICENSE](https://github.com/Haarris/cachecanary/blob/main/LICENSE). Copyright 2026 Haris Farooq.
|
|
188
|
+
|
|
189
|
+
Made by [Haris Farooq](https://www.linkedin.com/in/haris-farooq), an engineer who runs Claude agents on Bedrock in production.
|
|
@@ -9,8 +9,12 @@ Cache reads on Bedrock cost about a tenth of normal input tokens, so a long syst
|
|
|
9
9
|
|
|
10
10
|
LiteLLM hit this in [July 2026](https://docs.litellm.ai/blog/bedrock-invoke-prompt-caching-incident). After an upgrade, Claude Code traffic on Bedrock went from about 90% cache hits to 25-45%, and daily spend went up 2-3x for six days before anyone noticed. Anthropic has a cache diagnostics feature, but it only works on their own API, not on Bedrock.
|
|
11
11
|
|
|
12
|
+
I built CacheCanary after running into these failures while running Claude agents in production at [Ruby](https://heyruby.io).
|
|
13
|
+
|
|
12
14
|
I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
|
|
13
15
|
|
|
16
|
+
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
|
|
17
|
+
|
|
14
18
|
What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
|
|
15
19
|
|
|
16
20
|
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
|
|
@@ -139,10 +143,14 @@ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe`
|
|
|
139
143
|
|
|
140
144
|
## What's next
|
|
141
145
|
|
|
142
|
-
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
146
|
+
A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.
|
|
143
147
|
|
|
144
|
-
|
|
148
|
+
Using CacheCanary? I'd love to hear how it's going, good or bad, and whether you'd want the dashboard. Email me at [hello@cachecanary.com](mailto:hello@cachecanary.com). I read every message.
|
|
149
|
+
|
|
150
|
+
Vertex AI support is also planned.
|
|
145
151
|
|
|
146
152
|
## License
|
|
147
153
|
|
|
148
154
|
Apache 2.0. See [LICENSE](https://github.com/Haarris/cachecanary/blob/main/LICENSE). Copyright 2026 Haris Farooq.
|
|
155
|
+
|
|
156
|
+
Made by [Haris Farooq](https://www.linkedin.com/in/haris-farooq), an engineer who runs Claude agents on Bedrock in production.
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
"""A local stand-in for bedrock-runtime that speaks the real Converse and
|
|
1
|
+
"""A local stand-in for bedrock-runtime that speaks the real Converse, ConverseStream and InvokeModel formats.
|
|
2
2
|
|
|
3
3
|
Lets an agent framework run its real agent loop offline: each request body is saved, and replies follow
|
|
4
|
-
a script of turns: "tools:N" (N parallel toolUse blocks for a lookup_order tool), "text" (a short answer)
|
|
5
|
-
|
|
4
|
+
a script of turns: "tools:N" (N parallel toolUse blocks for a lookup_order tool), "text" (a short answer),
|
|
5
|
+
"text:N" (an N-line answer) or "use:NAME:JSON" (one call to the named tool, e.g. a structured-output tool).
|
|
6
|
+
InvokeModel replies use the Anthropic Messages format. Point a client at it with endpoint_url=Fake().url
|
|
7
|
+
and fake credentials.
|
|
6
8
|
"""
|
|
7
9
|
import json, struct, threading, zlib
|
|
8
10
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
@@ -33,6 +35,14 @@ class Fake:
|
|
|
33
35
|
fake.requests.append({"path": path, "body": body})
|
|
34
36
|
step = fake.script[fake.turn] if fake.turn < len(fake.script) else "text"
|
|
35
37
|
fake.turn += 1
|
|
38
|
+
if path.endswith("/invoke"):
|
|
39
|
+
return self._json(fake.messages_api(step, fake.turn))
|
|
40
|
+
if path.endswith("/invoke-with-response-stream"):
|
|
41
|
+
self.send_response(200)
|
|
42
|
+
self.send_header("Content-Type", "application/vnd.amazon.eventstream")
|
|
43
|
+
data = fake.messages_stream(step, fake.turn)
|
|
44
|
+
self.send_header("Content-Length", str(len(data)))
|
|
45
|
+
self.end_headers(); self.wfile.write(data); return
|
|
36
46
|
if path.endswith("/converse-stream"):
|
|
37
47
|
self.send_response(200)
|
|
38
48
|
self.send_header("Content-Type", "application/vnd.amazon.eventstream")
|
|
@@ -60,6 +70,14 @@ class Fake:
|
|
|
60
70
|
|
|
61
71
|
def stream(self, step, turn):
|
|
62
72
|
out = event("messageStart", {"role": "assistant"})
|
|
73
|
+
if step.startswith("use:"):
|
|
74
|
+
_, name, raw = step.split(":", 2)
|
|
75
|
+
out += event("contentBlockStart", {"start": {"toolUse": {"toolUseId": f"u{turn}", "name": name}}, "contentBlockIndex": 0})
|
|
76
|
+
out += event("contentBlockDelta", {"delta": {"toolUse": {"input": raw}}, "contentBlockIndex": 0})
|
|
77
|
+
out += event("contentBlockStop", {"contentBlockIndex": 0})
|
|
78
|
+
out += event("messageStop", {"stopReason": "tool_use"})
|
|
79
|
+
out += event("metadata", {"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}})
|
|
80
|
+
return out
|
|
63
81
|
if step.startswith("tools:"):
|
|
64
82
|
for i, (tid, inp) in enumerate(self._calls(step, turn)):
|
|
65
83
|
out += event("contentBlockStart", {"start": {"toolUse": {"toolUseId": tid, "name": "lookup_order"}}, "contentBlockIndex": i})
|
|
@@ -76,6 +94,10 @@ class Fake:
|
|
|
76
94
|
return out
|
|
77
95
|
|
|
78
96
|
def whole(self, step, turn):
|
|
97
|
+
if step.startswith("use:"):
|
|
98
|
+
_, name, raw = step.split(":", 2)
|
|
99
|
+
return {"output": {"message": {"role": "assistant", "content": [{"toolUse": {"toolUseId": f"u{turn}", "name": name, "input": json.loads(raw)}}]}},
|
|
100
|
+
"stopReason": "tool_use", "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}
|
|
79
101
|
if step.startswith("tools:"):
|
|
80
102
|
content = [{"toolUse": {"toolUseId": tid, "name": "lookup_order", "input": inp}} for tid, inp in self._calls(step, turn)]
|
|
81
103
|
stop = "tool_use"
|
|
@@ -84,3 +106,30 @@ class Fake:
|
|
|
84
106
|
content, stop = [{"text": text}], "end_turn"
|
|
85
107
|
return {"output": {"message": {"role": "assistant", "content": content}}, "stopReason": stop,
|
|
86
108
|
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}
|
|
109
|
+
|
|
110
|
+
# InvokeModel (Anthropic Messages API) replies, for ChatBedrock and other invoke-route clients.
|
|
111
|
+
def messages_api(self, step, turn):
|
|
112
|
+
if step.startswith("tools:"):
|
|
113
|
+
content = [{"type": "tool_use", "id": tid, "name": "lookup_order", "input": inp} for tid, inp in self._calls(step, turn)]
|
|
114
|
+
stop = "tool_use"
|
|
115
|
+
else:
|
|
116
|
+
text = "All done." if step == "text" else " ".join(f"Line {j} of answer {turn}." for j in range(int(step.split(":")[1])))
|
|
117
|
+
content, stop = [{"type": "text", "text": text}], "end_turn"
|
|
118
|
+
return {"id": f"msg_{turn}", "type": "message", "role": "assistant", "model": "claude", "content": content,
|
|
119
|
+
"stop_reason": stop, "stop_sequence": None, "usage": {"input_tokens": 1, "output_tokens": 1}}
|
|
120
|
+
|
|
121
|
+
def messages_stream(self, step, turn):
|
|
122
|
+
import base64
|
|
123
|
+
whole = self.messages_api(step, turn)
|
|
124
|
+
events = [{"type": "message_start", "message": {**whole, "content": [], "stop_reason": None}}]
|
|
125
|
+
for i, block in enumerate(whole["content"]):
|
|
126
|
+
if block["type"] == "tool_use":
|
|
127
|
+
events.append({"type": "content_block_start", "index": i, "content_block": {"type": "tool_use", "id": block["id"], "name": block["name"], "input": {}}})
|
|
128
|
+
events.append({"type": "content_block_delta", "index": i, "delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}})
|
|
129
|
+
else:
|
|
130
|
+
events.append({"type": "content_block_start", "index": i, "content_block": {"type": "text", "text": ""}})
|
|
131
|
+
events.append({"type": "content_block_delta", "index": i, "delta": {"type": "text_delta", "text": block["text"]}})
|
|
132
|
+
events.append({"type": "content_block_stop", "index": i})
|
|
133
|
+
events.append({"type": "message_delta", "delta": {"stop_reason": whole["stop_reason"]}, "usage": {"output_tokens": 1}})
|
|
134
|
+
events.append({"type": "message_stop", "amazon-bedrock-invocationMetrics": {"inputTokenCount": 1, "outputTokenCount": 1, "invocationLatency": 1, "firstByteLatency": 1}})
|
|
135
|
+
return b"".join(event("chunk", {"bytes": base64.b64encode(json.dumps(e).encode()).decode()}) for e in events)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Capture the exact Bedrock requests LangChain builds, offline, for tests/fixtures/langchain/.
|
|
2
|
+
|
|
3
|
+
LangChain runs its real agent loop (create_agent, tool calls, middleware) against a local stand-in for
|
|
4
|
+
bedrock-runtime (scripts/_fake_bedrock_stream.py) with fake credentials, so nothing leaves the machine.
|
|
5
|
+
|
|
6
|
+
pip install langchain-aws==1.8.1 langchain==1.4.3 langgraph==1.2.14
|
|
7
|
+
python scripts/capture_langchain_requests.py tests/fixtures/langchain
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
16
|
+
from _fake_bedrock_stream import Fake # noqa: E402
|
|
17
|
+
|
|
18
|
+
os.environ.update(AWS_ACCESS_KEY_ID="AKIAFAKE", AWS_SECRET_ACCESS_KEY="fake", AWS_DEFAULT_REGION="us-west-2")
|
|
19
|
+
|
|
20
|
+
from langchain.agents import create_agent # noqa: E402
|
|
21
|
+
from langchain.agents.middleware import ClearToolUsesEdit, ContextEditingMiddleware, dynamic_prompt # noqa: E402
|
|
22
|
+
from langchain_aws import ChatBedrock, ChatBedrockConverse # noqa: E402
|
|
23
|
+
from langchain_aws.middleware.prompt_caching import BedrockPromptCachingMiddleware # noqa: E402
|
|
24
|
+
from langchain_core.tools import tool # noqa: E402
|
|
25
|
+
|
|
26
|
+
MODEL = "us.anthropic.claude-sonnet-4-6"
|
|
27
|
+
SYSTEM = "You are a support agent for Acme. " + " ".join(
|
|
28
|
+
f"Rule {i}: answer politely and cite section {i}." for i in range(150))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@tool
|
|
32
|
+
def lookup_order(id: str) -> str:
|
|
33
|
+
"""Look up an order."""
|
|
34
|
+
return f"order {id} shipped on day {id}. " * 8
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main() -> int:
|
|
38
|
+
out = Path(sys.argv[1])
|
|
39
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
fake = Fake()
|
|
41
|
+
|
|
42
|
+
def converse():
|
|
43
|
+
return ChatBedrockConverse(model=MODEL, region_name="us-west-2", endpoint_url=fake.url, max_tokens=50)
|
|
44
|
+
|
|
45
|
+
def run(script, model, prompts, middleware, system=SYSTEM):
|
|
46
|
+
fake.reset(script)
|
|
47
|
+
agent = create_agent(model=model, tools=[lookup_order], system_prompt=system, middleware=middleware)
|
|
48
|
+
for prompt in prompts:
|
|
49
|
+
agent.invoke({"messages": [{"role": "user", "content": prompt}]})
|
|
50
|
+
return [request["body"] for request in fake.requests]
|
|
51
|
+
|
|
52
|
+
def save(name, body):
|
|
53
|
+
(out / name).write_text(json.dumps(body, indent=2) + "\n")
|
|
54
|
+
|
|
55
|
+
# Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls.
|
|
56
|
+
chat_bedrock = ChatBedrock(model=MODEL, region_name="us-west-2", endpoint_url=fake.url, model_kwargs={"max_tokens": 50})
|
|
57
|
+
bodies = run(["tools:11", "text"], chat_bedrock, ["Check my orders."], [BedrockPromptCachingMiddleware()])
|
|
58
|
+
save("invoke-turn1.json", bodies[0])
|
|
59
|
+
save("invoke-turn2.json", bodies[1])
|
|
60
|
+
|
|
61
|
+
# Finding 3: context editing that clears a sliding window of old tool results on every call.
|
|
62
|
+
editing = ContextEditingMiddleware(edits=[ClearToolUsesEdit(trigger=600, keep=3)])
|
|
63
|
+
bodies = run(["tools:1"] * 9 + ["text"], converse(), ["Check my orders."], [editing, BedrockPromptCachingMiddleware()])
|
|
64
|
+
save("edit-call8.json", bodies[8])
|
|
65
|
+
save("edit-call9.json", bodies[9])
|
|
66
|
+
|
|
67
|
+
# Smaller things: today's date in a dynamic system prompt.
|
|
68
|
+
day = {"date": "2026-10-07"}
|
|
69
|
+
|
|
70
|
+
@dynamic_prompt
|
|
71
|
+
def with_date(request):
|
|
72
|
+
return f"Today is {day['date']}. " + SYSTEM
|
|
73
|
+
|
|
74
|
+
fake.reset(["text", "text"])
|
|
75
|
+
agent = create_agent(model=converse(), tools=[lookup_order], middleware=[with_date, BedrockPromptCachingMiddleware()])
|
|
76
|
+
agent.invoke({"messages": [{"role": "user", "content": "Hi"}]})
|
|
77
|
+
day["date"] = "2026-10-08"
|
|
78
|
+
agent.invoke({"messages": [{"role": "user", "content": "Hi"}]})
|
|
79
|
+
save("date-day1.json", fake.requests[0]["body"])
|
|
80
|
+
save("date-day2.json", fake.requests[1]["body"])
|
|
81
|
+
print(f"Saved {len(list(out.glob('*.json')))} requests to {out}")
|
|
82
|
+
return 0
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
sys.exit(main())
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Re-run the live numbers on cachecanary.com/langchain/ against real Amazon Bedrock.
|
|
2
|
+
|
|
3
|
+
Settings checks run LangChain fully live. Agent-loop checks need an exact number of parallel tool calls,
|
|
4
|
+
which a live model won't do on request, so LangChain runs its real agent loop against a local stand-in
|
|
5
|
+
(scripts/_fake_bedrock_stream.py) and the exact requests it built are then sent, in order, to Bedrock.
|
|
6
|
+
Caching depends only on the request, so the numbers are the same.
|
|
7
|
+
|
|
8
|
+
Synthetic prompts with a fresh random tag per case. About 70 small calls on Claude Sonnet 4.6 (about $1).
|
|
9
|
+
Uses your normal AWS credentials, always in the region you pass.
|
|
10
|
+
|
|
11
|
+
pip install langchain-aws==1.8.1 langchain==1.4.3 langgraph==1.2.14
|
|
12
|
+
python scripts/live_langchain_checks.py --region us-west-2
|
|
13
|
+
|
|
14
|
+
To include the application inference profile check, create a profile for Sonnet 4.6 and pass its ARN
|
|
15
|
+
(see scripts/live_litellm_checks.py for the create and delete commands):
|
|
16
|
+
|
|
17
|
+
python scripts/live_langchain_checks.py --region us-west-2 --profile-arn ARN
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import sys
|
|
24
|
+
import time
|
|
25
|
+
import uuid
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
30
|
+
from _fake_bedrock_stream import Fake # noqa: E402
|
|
31
|
+
|
|
32
|
+
from langchain.agents.middleware import ClearToolUsesEdit # noqa: E402
|
|
33
|
+
from langchain_core.messages import ToolMessage # noqa: E402
|
|
34
|
+
|
|
35
|
+
MODEL = "us.anthropic.claude-sonnet-4-6"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Same as on the page (tests/test_site.py checks they match).
|
|
39
|
+
@dataclass(slots=True)
|
|
40
|
+
class ClearToolUsesInChunks(ClearToolUsesEdit):
|
|
41
|
+
"""Clear old tool results in whole batches, not one per call."""
|
|
42
|
+
|
|
43
|
+
chunk: int = 10
|
|
44
|
+
|
|
45
|
+
def apply(self, messages, *, count_tokens) -> None:
|
|
46
|
+
results = sum(isinstance(m, ToolMessage) for m in messages)
|
|
47
|
+
cleared = max(0, results - self.keep) // self.chunk * self.chunk
|
|
48
|
+
keep, self.keep = self.keep, results - cleared # whole batches only
|
|
49
|
+
try:
|
|
50
|
+
ClearToolUsesEdit.apply(self, messages, count_tokens=count_tokens)
|
|
51
|
+
finally:
|
|
52
|
+
self.keep = keep
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def main() -> int:
|
|
56
|
+
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
57
|
+
parser.add_argument("--region", required=True)
|
|
58
|
+
parser.add_argument("--profile-arn", help="an application inference profile for Sonnet 4.6 (optional)")
|
|
59
|
+
args = parser.parse_args()
|
|
60
|
+
|
|
61
|
+
import boto3
|
|
62
|
+
from langchain.agents import create_agent
|
|
63
|
+
from langchain.agents.middleware import ContextEditingMiddleware
|
|
64
|
+
from langchain_aws import ChatBedrock, ChatBedrockConverse
|
|
65
|
+
from langchain_aws.middleware.prompt_caching import BedrockPromptCachingMiddleware
|
|
66
|
+
from langchain_core.messages import SystemMessage
|
|
67
|
+
from langchain_core.tools import tool
|
|
68
|
+
|
|
69
|
+
bedrock = boto3.client("bedrock-runtime", region_name=args.region)
|
|
70
|
+
assert bedrock.meta.region_name == args.region
|
|
71
|
+
tool_output = {"repeat": 8}
|
|
72
|
+
|
|
73
|
+
@tool
|
|
74
|
+
def lookup_order(id: str) -> str:
|
|
75
|
+
"""Look up one order by its id. Returns its shipping status."""
|
|
76
|
+
return f"order {id} shipped on day {id}. " * tool_output["repeat"]
|
|
77
|
+
|
|
78
|
+
def live_model(model_id=MODEL, **kwargs):
|
|
79
|
+
# boto3 prefers the region in ~/.aws/config over environment variables, so pass it and check.
|
|
80
|
+
model = ChatBedrockConverse(model=model_id, region_name=args.region, max_tokens=kwargs.pop("max_tokens", 50), **kwargs)
|
|
81
|
+
assert model.client.meta.region_name == args.region, model.client.meta.region_name
|
|
82
|
+
return model
|
|
83
|
+
|
|
84
|
+
def tag_prompts():
|
|
85
|
+
tag = uuid.uuid4().hex
|
|
86
|
+
system = f"[run {tag}] You are a support agent for Acme. " + " ".join(
|
|
87
|
+
f"Rule {i}: answer politely and cite section {i}." for i in range(150))
|
|
88
|
+
history = f"[doc {tag}] Customer history: " + " ".join(
|
|
89
|
+
f"Order {i} was placed on day {i} and contained item {i * 7}." for i in range(250))
|
|
90
|
+
return system, history
|
|
91
|
+
|
|
92
|
+
def usage(messages):
|
|
93
|
+
total = {"read": 0, "write": 0}
|
|
94
|
+
for message in messages:
|
|
95
|
+
details = (getattr(message, "usage_metadata", None) or {}).get("input_token_details", {}) or {}
|
|
96
|
+
total["read"] += details.get("cache_read", 0)
|
|
97
|
+
# LangChain moves writes into the per-lifetime keys when Bedrock reports them.
|
|
98
|
+
total["write"] += (details.get("cache_creation", 0) + details.get("ephemeral_5m_input_tokens", 0)
|
|
99
|
+
+ details.get("ephemeral_1h_input_tokens", 0))
|
|
100
|
+
return total
|
|
101
|
+
|
|
102
|
+
def show(row):
|
|
103
|
+
print(json.dumps(row), flush=True)
|
|
104
|
+
return row
|
|
105
|
+
|
|
106
|
+
checks = []
|
|
107
|
+
|
|
108
|
+
# Settings, fully live: the same system prompt in two fresh agents.
|
|
109
|
+
def twice(case, middleware, **model_kwargs):
|
|
110
|
+
system = tag_prompts()[0] + " ".join(f"Extra rule {i}." for i in range(150))
|
|
111
|
+
rows = []
|
|
112
|
+
for call in (1, 2):
|
|
113
|
+
agent = create_agent(model=live_model(**model_kwargs), tools=[lookup_order], system_prompt=system,
|
|
114
|
+
middleware=middleware)
|
|
115
|
+
out = agent.invoke({"messages": [{"role": "user", "content": "Reply with OK."}]})
|
|
116
|
+
rows.append(show({"case": case, "call": call, **usage(out["messages"])}))
|
|
117
|
+
time.sleep(2)
|
|
118
|
+
return rows
|
|
119
|
+
|
|
120
|
+
off = twice("no middleware (the default)", [])
|
|
121
|
+
on = twice("BedrockPromptCachingMiddleware", [BedrockPromptCachingMiddleware()])
|
|
122
|
+
checks += [off[1]["read"] == 0 and off[1]["write"] == 0, on[1]["read"] > 1000]
|
|
123
|
+
if args.profile_arn:
|
|
124
|
+
looked_up = twice("profile ARN, provider anthropic", [BedrockPromptCachingMiddleware()],
|
|
125
|
+
model_id=args.profile_arn, provider="anthropic")
|
|
126
|
+
named = twice("profile ARN, provider anthropic, base_model_id", [BedrockPromptCachingMiddleware()],
|
|
127
|
+
model_id=args.profile_arn, provider="anthropic", base_model_id="anthropic.claude-sonnet-4-6")
|
|
128
|
+
checks += [looked_up[1]["read"] > 1000, named[1]["read"] > 1000]
|
|
129
|
+
|
|
130
|
+
# Agent loops: LangChain's real loop on the stand-in, then the exact requests replayed to Bedrock.
|
|
131
|
+
os.environ.setdefault("AWS_ACCESS_KEY_ID", "unused-by-the-stand-in")
|
|
132
|
+
fake = Fake()
|
|
133
|
+
|
|
134
|
+
def capture(script, *, invoke_route=False, middleware=None, manual_point=False):
|
|
135
|
+
system, history = tag_prompts()
|
|
136
|
+
fake.reset(script)
|
|
137
|
+
if invoke_route:
|
|
138
|
+
model = ChatBedrock(model=MODEL, region_name=args.region, endpoint_url=fake.url, model_kwargs={"max_tokens": 50})
|
|
139
|
+
else:
|
|
140
|
+
model = ChatBedrockConverse(model=MODEL, region_name=args.region, endpoint_url=fake.url, max_tokens=50)
|
|
141
|
+
if manual_point:
|
|
142
|
+
system = SystemMessage(content=[{"type": "text", "text": system}, ChatBedrockConverse.create_cache_point()])
|
|
143
|
+
agent = create_agent(model=model, tools=[lookup_order], system_prompt=system,
|
|
144
|
+
middleware=middleware or [BedrockPromptCachingMiddleware()])
|
|
145
|
+
agent.invoke({"messages": [{"role": "user", "content": history + " Check my orders."}]})
|
|
146
|
+
return [(request["path"], request["body"]) for request in fake.requests]
|
|
147
|
+
|
|
148
|
+
def replay(case, requests):
|
|
149
|
+
rows = []
|
|
150
|
+
for i, (path, body) in enumerate(requests):
|
|
151
|
+
if path.endswith("/invoke"):
|
|
152
|
+
out = json.loads(bedrock.invoke_model(modelId=MODEL, body=json.dumps({**body, "max_tokens": 5}))["body"].read())
|
|
153
|
+
u = out["usage"]
|
|
154
|
+
read, write, uncached = u.get("cache_read_input_tokens", 0), u.get("cache_creation_input_tokens", 0), u.get("input_tokens", 0)
|
|
155
|
+
else:
|
|
156
|
+
u = bedrock.converse(modelId=MODEL, **{**body, "inferenceConfig": {"maxTokens": 5}})["usage"]
|
|
157
|
+
read, write, uncached = u.get("cacheReadInputTokens", 0), u.get("cacheWriteInputTokens", 0), u.get("inputTokens", 0)
|
|
158
|
+
rows.append(show({"case": case, "call": i, "read": read, "write": write, "prompt": read + write + uncached}))
|
|
159
|
+
time.sleep(2)
|
|
160
|
+
return rows
|
|
161
|
+
|
|
162
|
+
def kept(rows): # the second call read (almost) the whole first request
|
|
163
|
+
return rows[1]["read"] >= rows[0]["prompt"] - 5
|
|
164
|
+
|
|
165
|
+
def kept_with_retry(case, script, **kwargs):
|
|
166
|
+
# With a us. or global. model ID, Bedrock can route two calls to different Regions, and caches
|
|
167
|
+
# are per Region: the second call then reads nothing, not even the system prompt. That is not
|
|
168
|
+
# LangChain, so try once more before counting it.
|
|
169
|
+
for attempt in (1, 2):
|
|
170
|
+
rows = replay(case if attempt == 1 else f"{case} (retry after a full miss)", capture(script, **kwargs))
|
|
171
|
+
if rows[1]["read"] > 0:
|
|
172
|
+
return kept(rows)
|
|
173
|
+
return False
|
|
174
|
+
|
|
175
|
+
checks.append(kept_with_retry("ChatBedrockConverse, 21 parallel calls", ["tools:21", "text"]))
|
|
176
|
+
checks.append(not kept(replay("ChatBedrockConverse, 22 parallel calls", capture(["tools:22", "text"]))))
|
|
177
|
+
checks.append(kept_with_retry("ChatBedrock, 10 parallel calls", ["tools:10", "text"], invoke_route=True))
|
|
178
|
+
invoke_11 = replay("ChatBedrock, 11 parallel calls", capture(["tools:11", "text"], invoke_route=True))
|
|
179
|
+
checks.append(invoke_11[1]["read"] == 0) # nothing read: no separate system prompt marker on this route
|
|
180
|
+
checks.append(kept_with_retry("middleware only, 11 parallel calls", ["tools:11", "text"]))
|
|
181
|
+
checks.append(not kept(replay("manual system cache point + middleware, 11 parallel calls",
|
|
182
|
+
capture(["tools:11", "text"], manual_point=True))))
|
|
183
|
+
|
|
184
|
+
# Context editing: stock clearing vs clearing in batches vs none, 13 calls with long tool results.
|
|
185
|
+
tool_output["repeat"] = 40
|
|
186
|
+
|
|
187
|
+
def cost(row): # in full-price input tokens: reads 0.1x, 5-minute writes 1.25x
|
|
188
|
+
return row["read"] * 0.1 + row["write"] * 1.25 + (row["prompt"] - row["read"] - row["write"])
|
|
189
|
+
|
|
190
|
+
stock = replay("ClearToolUsesEdit (stock)", capture(["tools:1"] * 12 + ["text"], middleware=[
|
|
191
|
+
ContextEditingMiddleware(edits=[ClearToolUsesEdit(trigger=5800, keep=3)]), BedrockPromptCachingMiddleware()]))
|
|
192
|
+
chunked = replay("ClearToolUsesInChunks(chunk=5)", capture(["tools:1"] * 12 + ["text"], middleware=[
|
|
193
|
+
ContextEditingMiddleware(edits=[ClearToolUsesInChunks(trigger=5800, keep=3, chunk=5)]), BedrockPromptCachingMiddleware()]))
|
|
194
|
+
none = replay("no context editing", capture(["tools:1"] * 12 + ["text"]))
|
|
195
|
+
show({"case": "cost of call 12", "stock": round(cost(stock[12])), "chunks": round(cost(chunked[12])), "none": round(cost(none[12]))})
|
|
196
|
+
checks += [cost(stock[12]) > 3 * cost(none[12]), cost(chunked[12]) < cost(none[12])]
|
|
197
|
+
|
|
198
|
+
print(f"\n{sum(checks)}/{len(checks)} checks matched the page.")
|
|
199
|
+
return 0 if all(checks) else 1
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
if __name__ == "__main__":
|
|
203
|
+
sys.exit(main())
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
<!-- Social preview card. Render: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless=new --hide-scrollbars --force-device-scale-factor=2 --window-size=1200,630 --screenshot=site/og-langchain.png scripts/og-cards/og-langchain.html -->
|
|
2
|
+
<!doctype html><html><head><meta charset="utf-8"><style>
|
|
3
|
+
*{box-sizing:border-box;margin:0}html,body{width:1200px;height:630px;overflow:hidden}
|
|
4
|
+
body{background:#fbfaf7;color:#1d1d1b;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Arial,sans-serif;padding:72px 84px;display:flex;flex-direction:column;border-top:14px solid #f2c230}
|
|
5
|
+
.brand{display:flex;align-items:center;gap:18px;font-size:38px;font-weight:700}.brand svg{width:64px;height:64px}
|
|
6
|
+
h1{font-size:64px;line-height:1.12;letter-spacing:-0.02em;margin-top:52px;max-width:1030px}
|
|
7
|
+
p{font-size:30px;line-height:1.4;color:#5f5d57;margin-top:26px;max-width:1000px}
|
|
8
|
+
.foot{margin-top:auto;display:flex;justify-content:space-between;font-size:26px;color:#5f5d57}.foot b{color:#7a5200}
|
|
9
|
+
</style></head><body><div class="brand"><svg viewBox="0 0 32 32" aria-hidden="true"><path d="M3 29.2h26" stroke="#8a5a2b" stroke-width="1.8" stroke-linecap="round"/><path d="M14.6 24.4l-.8 4.6M18.2 24l.4 5" stroke="#e8892b" stroke-width="1.4" stroke-linecap="round"/><path d="M10.5 19.5L2.2 25.4l4.4.4L12.8 22z" fill="#d9a514"/><ellipse cx="15.5" cy="18.6" rx="8.4" ry="6.2" transform="rotate(-24 15.5 18.6)" fill="#f2c230"/><circle cx="21.8" cy="11.2" r="4.7" fill="#f2c230"/><path d="M26 10l4.4 1.6-4.4 1.8z" fill="#e8892b"/><circle cx="23.2" cy="10.4" r="1.1" fill="#1d1d1b"/><path d="M9.6 19.6c2.2-4.4 6.8-5.6 9.6-4.2-1.2 3.8-5.4 6.4-9.6 4.2z" fill="#d9a514"/></svg> CacheCanary</div><h1>Prompt caching with LangChain on Amazon Bedrock</h1><p>What LangChain actually sends, where the cache is lost, and a setup that keeps it. Tested on live Bedrock.</p><div class="foot"><span>Free and open source</span><b>cachecanary.com/langchain</b></div></body></html>
|
|
@@ -5,7 +5,10 @@
|
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
6
|
<title>Page not found · CacheCanary</title>
|
|
7
7
|
<meta name="robots" content="noindex">
|
|
8
|
+
<link rel="icon" href="/favicon.ico" sizes="48x48">
|
|
8
9
|
<link rel="icon" href="/favicon.svg" type="image/svg+xml">
|
|
10
|
+
<link rel="icon" href="/icon-192.png" type="image/png" sizes="192x192">
|
|
11
|
+
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
|
|
9
12
|
<link rel="stylesheet" href="/style.css">
|
|
10
13
|
</head>
|
|
11
14
|
<body>
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -16,7 +16,10 @@
|
|
|
16
16
|
<meta property="og:site_name" content="CacheCanary">
|
|
17
17
|
<meta name="twitter:card" content="summary_large_image">
|
|
18
18
|
<link rel="canonical" href="https://cachecanary.com/">
|
|
19
|
+
<link rel="icon" href="/favicon.ico" sizes="48x48">
|
|
19
20
|
<link rel="icon" href="/favicon.svg" type="image/svg+xml">
|
|
21
|
+
<link rel="icon" href="/icon-192.png" type="image/png" sizes="192x192">
|
|
22
|
+
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
|
|
20
23
|
<meta name="color-scheme" content="light dark">
|
|
21
24
|
<link rel="stylesheet" href="/style.css">
|
|
22
25
|
<script type="application/ld+json">
|
|
@@ -28,7 +31,7 @@
|
|
|
28
31
|
"url": "https://cachecanary.com/",
|
|
29
32
|
"applicationCategory": "DeveloperApplication",
|
|
30
33
|
"operatingSystem": "macOS, Linux, Windows",
|
|
31
|
-
"softwareVersion": "0.3.
|
|
34
|
+
"softwareVersion": "0.3.2",
|
|
32
35
|
"downloadUrl": "https://pypi.org/project/cachecanary/",
|
|
33
36
|
"license": "https://www.apache.org/licenses/LICENSE-2.0",
|
|
34
37
|
"offers": {
|
|
@@ -39,7 +42,12 @@
|
|
|
39
42
|
"author": {
|
|
40
43
|
"@type": "Person",
|
|
41
44
|
"name": "Haris Farooq",
|
|
42
|
-
"url": "https://
|
|
45
|
+
"url": "https://www.linkedin.com/in/haris-farooq",
|
|
46
|
+
"sameAs": [
|
|
47
|
+
"https://www.linkedin.com/in/haris-farooq",
|
|
48
|
+
"https://github.com/Haarris",
|
|
49
|
+
"https://harisfarooq.substack.com"
|
|
50
|
+
]
|
|
43
51
|
},
|
|
44
52
|
"sameAs": [
|
|
45
53
|
"https://github.com/Haarris/cachecanary",
|
|
@@ -48,6 +56,15 @@
|
|
|
48
56
|
]
|
|
49
57
|
}
|
|
50
58
|
</script>
|
|
59
|
+
<script type="application/ld+json">
|
|
60
|
+
{
|
|
61
|
+
"@context": "https://schema.org",
|
|
62
|
+
"@type": "WebSite",
|
|
63
|
+
"name": "CacheCanary",
|
|
64
|
+
"alternateName": "cachecanary.com",
|
|
65
|
+
"url": "https://cachecanary.com/"
|
|
66
|
+
}
|
|
67
|
+
</script>
|
|
51
68
|
</head>
|
|
52
69
|
<body>
|
|
53
70
|
<div class="wrap">
|
|
@@ -59,7 +76,15 @@
|
|
|
59
76
|
</a>
|
|
60
77
|
<nav>
|
|
61
78
|
<a href="#how">How it works</a>
|
|
62
|
-
<
|
|
79
|
+
<details class="guides">
|
|
80
|
+
<summary>Guides</summary>
|
|
81
|
+
<div class="menu">
|
|
82
|
+
<a href="/litellm/">LiteLLM</a>
|
|
83
|
+
<a href="/strands/">Strands Agents</a>
|
|
84
|
+
<a href="/langchain/">LangChain</a>
|
|
85
|
+
<a href="/#guides">All guides</a>
|
|
86
|
+
</div>
|
|
87
|
+
</details>
|
|
63
88
|
<a href="https://github.com/Haarris/cachecanary">GitHub</a>
|
|
64
89
|
</nav>
|
|
65
90
|
</header>
|
|
@@ -75,6 +100,7 @@
|
|
|
75
100
|
<a class="btn" href="https://github.com/Haarris/cachecanary#readme">Read the docs</a>
|
|
76
101
|
</div>
|
|
77
102
|
<p class="note under-actions">Free and open source (Apache 2.0). The tool runs on your machine and sends nothing anywhere.</p>
|
|
103
|
+
<p class="note">Framework guides: <a href="/litellm/">LiteLLM</a> · <a href="/strands/">Strands Agents</a> · <a href="/langchain/">LangChain</a></p>
|
|
78
104
|
|
|
79
105
|
<h2>What a broken cache costs</h2>
|
|
80
106
|
<div class="facts">
|
|
@@ -152,6 +178,7 @@ system-changed: The system prompt changed inside the cached prefix (look for dat
|
|
|
152
178
|
<ul>
|
|
153
179
|
<li><a href="/litellm/">Prompt caching with LiteLLM on Amazon Bedrock</a>: <code>role: "user"</code> caching your oldest messages, wide tool calls, and a 1-hour cache lifetime that is quietly dropped.</li>
|
|
154
180
|
<li><a href="/strands/">Prompt caching with Strands Agents on Amazon Bedrock</a>: caching that is off by default, wide tool calls, and long chats that lose the cache on every turn.</li>
|
|
181
|
+
<li><a href="/langchain/">Prompt caching with LangChain on Amazon Bedrock</a>: ChatBedrock losing the whole cache after 10 tool calls, context editing that rewrites the conversation on every call, and your own cache points turning off LangChain's extra one.</li>
|
|
155
182
|
</ul>
|
|
156
183
|
|
|
157
184
|
<h2>Built on how Bedrock actually behaves</h2>
|
|
@@ -163,13 +190,13 @@ system-changed: The system prompt changed inside the cached prefix (look for dat
|
|
|
163
190
|
</div>
|
|
164
191
|
|
|
165
192
|
<h2>Coming next</h2>
|
|
166
|
-
<p>A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It
|
|
167
|
-
<p>
|
|
193
|
+
<p>A hosted dashboard: cache hit rate and wasted spend per app, an alert when the rate drops, and the reason for each miss. It would run inside your own AWS account, so prompts never leave it.</p>
|
|
194
|
+
<p>Using CacheCanary? I'd love to hear how it's going, good or bad, and whether you'd want the dashboard. Email me at <a href="mailto:hello@cachecanary.com">hello@cachecanary.com</a>. I read every message.</p>
|
|
168
195
|
</main>
|
|
169
196
|
|
|
170
197
|
<footer>
|
|
171
|
-
CacheCanary is free and open source under the Apache 2.0 license. Made by Haris Farooq
|
|
172
|
-
<a href="https://github.com/Haarris/cachecanary">GitHub</a> · <a href="mailto:hello@cachecanary.com">hello@cachecanary.com</a>
|
|
198
|
+
CacheCanary is free and open source under the Apache 2.0 license. Made by <a href="https://www.linkedin.com/in/haris-farooq">Haris Farooq</a>, an engineer who runs Claude agents on Bedrock in production.<br>
|
|
199
|
+
<a href="https://github.com/Haarris/cachecanary">GitHub</a> · <a href="https://www.linkedin.com/in/haris-farooq">LinkedIn</a> · <a href="mailto:hello@cachecanary.com">hello@cachecanary.com</a>
|
|
173
200
|
</footer>
|
|
174
201
|
|
|
175
202
|
</div>
|