cachecanary 0.3.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {cachecanary-0.3.0 → cachecanary-0.3.1}/PKG-INFO +3 -1
  2. {cachecanary-0.3.0 → cachecanary-0.3.1}/README.md +2 -0
  3. cachecanary-0.3.1/scripts/_fake_bedrock_stream.py +86 -0
  4. cachecanary-0.3.1/scripts/capture_litellm_requests.py +117 -0
  5. cachecanary-0.3.1/scripts/capture_strands_requests.py +68 -0
  6. cachecanary-0.3.1/scripts/live_litellm_checks.py +170 -0
  7. cachecanary-0.3.1/scripts/live_strands_checks.py +245 -0
  8. cachecanary-0.3.1/scripts/og-cards/og-litellm.html +9 -0
  9. cachecanary-0.3.1/scripts/og-cards/og-strands.html +9 -0
  10. cachecanary-0.3.1/scripts/og-cards/og.html +9 -0
  11. {cachecanary-0.3.0 → cachecanary-0.3.1}/site/index.html +45 -2
  12. cachecanary-0.3.1/site/litellm/index.html +195 -0
  13. cachecanary-0.3.1/site/llms.txt +35 -0
  14. cachecanary-0.3.1/site/og-litellm.png +0 -0
  15. cachecanary-0.3.1/site/og-strands.png +0 -0
  16. cachecanary-0.3.1/site/og.png +0 -0
  17. cachecanary-0.3.1/site/robots.txt +4 -0
  18. cachecanary-0.3.1/site/sitemap.xml +15 -0
  19. cachecanary-0.3.1/site/strands/index.html +257 -0
  20. {cachecanary-0.3.0 → cachecanary-0.3.1}/site/style.css +12 -0
  21. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/__init__.py +1 -1
  22. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/diff.py +6 -1
  23. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary.egg-info/PKG-INFO +3 -1
  24. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary.egg-info/SOURCES.txt +27 -1
  25. cachecanary-0.3.1/tests/fixtures/litellm/agent-turn1.json +51 -0
  26. cachecanary-0.3.1/tests/fixtures/litellm/agent-turn2.json +289 -0
  27. cachecanary-0.3.1/tests/fixtures/litellm/chat-turn6.json +120 -0
  28. cachecanary-0.3.1/tests/fixtures/litellm/chat-turn7.json +136 -0
  29. cachecanary-0.3.1/tests/fixtures/litellm/with-download.json +26 -0
  30. cachecanary-0.3.1/tests/fixtures/litellm/without-download.json +25 -0
  31. cachecanary-0.3.1/tests/fixtures/strands/agent-turn1.json +55 -0
  32. cachecanary-0.3.1/tests/fixtures/strands/agent-turn2.json +285 -0
  33. cachecanary-0.3.1/tests/fixtures/strands/chat-turn20.json +375 -0
  34. cachecanary-0.3.1/tests/fixtures/strands/chat-turn21.json +375 -0
  35. cachecanary-0.3.1/tests/fixtures/strands/ttl-order.json +57 -0
  36. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_cachecanary.py +23 -0
  37. cachecanary-0.3.1/tests/test_site.py +334 -0
  38. cachecanary-0.3.0/site/robots.txt +0 -2
  39. cachecanary-0.3.0/tests/test_site.py +0 -73
  40. {cachecanary-0.3.0 → cachecanary-0.3.1}/.github/workflows/ci.yml +0 -0
  41. {cachecanary-0.3.0 → cachecanary-0.3.1}/.github/workflows/publish.yml +0 -0
  42. {cachecanary-0.3.0 → cachecanary-0.3.1}/LICENSE +0 -0
  43. {cachecanary-0.3.0 → cachecanary-0.3.1}/MANIFEST.in +0 -0
  44. {cachecanary-0.3.0 → cachecanary-0.3.1}/NOTICE +0 -0
  45. {cachecanary-0.3.0 → cachecanary-0.3.1}/action.yml +0 -0
  46. {cachecanary-0.3.0 → cachecanary-0.3.1}/examples/request_bad.json +0 -0
  47. {cachecanary-0.3.0 → cachecanary-0.3.1}/examples/request_good.json +0 -0
  48. {cachecanary-0.3.0 → cachecanary-0.3.1}/examples/request_next.json +0 -0
  49. {cachecanary-0.3.0 → cachecanary-0.3.1}/examples/request_too_short.json +0 -0
  50. {cachecanary-0.3.0 → cachecanary-0.3.1}/pyproject.toml +0 -0
  51. {cachecanary-0.3.0 → cachecanary-0.3.1}/scripts/check_site.mjs +0 -0
  52. {cachecanary-0.3.0 → cachecanary-0.3.1}/scripts/live_logs_test.py +0 -0
  53. {cachecanary-0.3.0 → cachecanary-0.3.1}/scripts/live_lookback_boundary.py +0 -0
  54. {cachecanary-0.3.0 → cachecanary-0.3.1}/scripts/live_test.py +0 -0
  55. {cachecanary-0.3.0 → cachecanary-0.3.1}/scripts/live_v03_checks.py +0 -0
  56. {cachecanary-0.3.0 → cachecanary-0.3.1}/scripts/make_log_fixture.py +0 -0
  57. {cachecanary-0.3.0 → cachecanary-0.3.1}/scripts/serve_site.py +0 -0
  58. {cachecanary-0.3.0 → cachecanary-0.3.1}/setup.cfg +0 -0
  59. {cachecanary-0.3.0 → cachecanary-0.3.1}/site/404.html +0 -0
  60. {cachecanary-0.3.0 → cachecanary-0.3.1}/site/_headers +0 -0
  61. {cachecanary-0.3.0 → cachecanary-0.3.1}/site/favicon.svg +0 -0
  62. {cachecanary-0.3.0 → cachecanary-0.3.1}/site/main.js +0 -0
  63. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/cli.py +0 -0
  64. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/github.py +0 -0
  65. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/lint.py +0 -0
  66. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/logs.py +0 -0
  67. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/models.py +0 -0
  68. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/pricing.py +0 -0
  69. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/probe.py +0 -0
  70. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/request.py +0 -0
  71. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary/usage.py +0 -0
  72. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary.egg-info/dependency_links.txt +0 -0
  73. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary.egg-info/entry_points.txt +0 -0
  74. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary.egg-info/requires.txt +0 -0
  75. {cachecanary-0.3.0 → cachecanary-0.3.1}/src/cachecanary.egg-info/top_level.txt +0 -0
  76. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/fixtures/bedrock_invocation_logs_2026-10.json +0 -0
  77. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_action.py +0 -0
  78. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_edge_cases.py +0 -0
  79. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_github.py +0 -0
  80. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_pricing.py +0 -0
  81. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_readme.py +0 -0
  82. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_scripts_and_formats.py +0 -0
  83. {cachecanary-0.3.0 → cachecanary-0.3.1}/tests/test_v03_rules.py +0 -0
  84. {cachecanary-0.3.0 → cachecanary-0.3.1}/wrangler.jsonc +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cachecanary
3
- Version: 0.3.0
3
+ Version: 0.3.1
4
4
  Summary: CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock, not when the bill arrives.
5
5
  Author-email: Haris Farooq <harisf98@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -42,6 +42,8 @@ Cache reads on Bedrock cost about a tenth of normal input tokens, so a long syst
42
42
 
43
43
  LiteLLM hit this in [July 2026](https://docs.litellm.ai/blog/bedrock-invoke-prompt-caching-incident). After an upgrade, Claude Code traffic on Bedrock went from about 90% cache hits to 25-45%, and daily spend went up 2-3x for six days before anyone noticed. Anthropic has a cache diagnostics feature, but it only works on their own API, not on Bedrock.
44
44
 
45
+ I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
46
+
45
47
  What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
46
48
 
47
49
  CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
@@ -9,6 +9,8 @@ Cache reads on Bedrock cost about a tenth of normal input tokens, so a long syst
9
9
 
10
10
  LiteLLM hit this in [July 2026](https://docs.litellm.ai/blog/bedrock-invoke-prompt-caching-incident). After an upgrade, Claude Code traffic on Bedrock went from about 90% cache hits to 25-45%, and daily spend went up 2-3x for six days before anyone noticed. Anthropic has a cache diagnostics feature, but it only works on their own API, not on Bedrock.
11
11
 
12
+ I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
13
+
12
14
  What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
13
15
 
14
16
  CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
@@ -0,0 +1,86 @@
1
+ """A local stand-in for bedrock-runtime that speaks the real Converse and ConverseStream formats.
2
+
3
+ Lets an agent framework run its real agent loop offline: each request body is saved, and replies follow
4
+ a script of turns: "tools:N" (N parallel toolUse blocks for a lookup_order tool), "text" (a short answer)
5
+ or "text:N" (an N-line answer). Point a client at it with endpoint_url=Fake().url and fake credentials.
6
+ """
7
+ import json, struct, threading, zlib
8
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
9
+
10
+ def _header(name, value):
11
+ n, v = name.encode(), value.encode()
12
+ return struct.pack(">B", len(n)) + n + b"\x07" + struct.pack(">H", len(v)) + v
13
+
14
+ def event(event_type, payload):
15
+ headers = _header(":event-type", event_type) + _header(":content-type", "application/json") + _header(":message-type", "event")
16
+ body = json.dumps(payload).encode()
17
+ total = 12 + len(headers) + len(body) + 4
18
+ prelude = struct.pack(">II", total, len(headers))
19
+ prelude += struct.pack(">I", zlib.crc32(prelude) & 0xFFFFFFFF)
20
+ msg = prelude + headers + body
21
+ return msg + struct.pack(">I", zlib.crc32(msg) & 0xFFFFFFFF)
22
+
23
+ class Fake:
24
+ def __init__(self):
25
+ self.requests, self.script, self.turn = [], [], 0
26
+ fake = self
27
+ class H(BaseHTTPRequestHandler):
28
+ def do_POST(self):
29
+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])) or b"{}")
30
+ path = self.path
31
+ if path.endswith("/count-tokens"):
32
+ return self._json({"inputTokens": 100})
33
+ fake.requests.append({"path": path, "body": body})
34
+ step = fake.script[fake.turn] if fake.turn < len(fake.script) else "text"
35
+ fake.turn += 1
36
+ if path.endswith("/converse-stream"):
37
+ self.send_response(200)
38
+ self.send_header("Content-Type", "application/vnd.amazon.eventstream")
39
+ data = fake.stream(step, fake.turn)
40
+ self.send_header("Content-Length", str(len(data)))
41
+ self.end_headers(); self.wfile.write(data)
42
+ else:
43
+ self._json(fake.whole(step, fake.turn))
44
+ def _json(self, obj):
45
+ d = json.dumps(obj).encode()
46
+ self.send_response(200); self.send_header("Content-Type", "application/json")
47
+ self.send_header("Content-Length", str(len(d))); self.end_headers(); self.wfile.write(d)
48
+ def log_message(self, *a): pass
49
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), H)
50
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
51
+ self.url = f"http://127.0.0.1:{self.server.server_port}"
52
+
53
+ def reset(self, script):
54
+ self.requests, self.script, self.turn = [], list(script), 0
55
+
56
+ @staticmethod
57
+ def _calls(step, turn):
58
+ n = int(step.split(":")[1])
59
+ return [(f"t{turn}_{i}", {"id": f"{turn}-{i}"}) for i in range(n)]
60
+
61
+ def stream(self, step, turn):
62
+ out = event("messageStart", {"role": "assistant"})
63
+ if step.startswith("tools:"):
64
+ for i, (tid, inp) in enumerate(self._calls(step, turn)):
65
+ out += event("contentBlockStart", {"start": {"toolUse": {"toolUseId": tid, "name": "lookup_order"}}, "contentBlockIndex": i})
66
+ out += event("contentBlockDelta", {"delta": {"toolUse": {"input": json.dumps(inp)}}, "contentBlockIndex": i})
67
+ out += event("contentBlockStop", {"contentBlockIndex": i})
68
+ stop = "tool_use"
69
+ else:
70
+ text = "All done." if step == "text" else " ".join(f"Line {j} of answer {turn}: order {j} shipped on day {j}." for j in range(int(step.split(":")[1])))
71
+ out += event("contentBlockDelta", {"delta": {"text": text}, "contentBlockIndex": 0})
72
+ out += event("contentBlockStop", {"contentBlockIndex": 0})
73
+ stop = "end_turn"
74
+ out += event("messageStop", {"stopReason": stop})
75
+ out += event("metadata", {"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}})
76
+ return out
77
+
78
+ def whole(self, step, turn):
79
+ if step.startswith("tools:"):
80
+ content = [{"toolUse": {"toolUseId": tid, "name": "lookup_order", "input": inp}} for tid, inp in self._calls(step, turn)]
81
+ stop = "tool_use"
82
+ else:
83
+ text = "All done." if step == "text" else " ".join(f"Line {j} of answer {turn}: order {j} shipped on day {j}." for j in range(int(step.split(":")[1])))
84
+ content, stop = [{"text": text}], "end_turn"
85
+ return {"output": {"message": {"role": "assistant", "content": content}}, "stopReason": stop,
86
+ "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}
@@ -0,0 +1,117 @@
1
+ """Capture the exact Bedrock requests LiteLLM builds, offline, for tests/fixtures/litellm/.
2
+
3
+ A small local server stands in for bedrock-runtime and saves every request body. LiteLLM is pointed at it
4
+ with aws_bedrock_runtime_endpoint and fake credentials, so nothing leaves the machine.
5
+
6
+ pip install litellm==1.104.0
7
+ python scripts/capture_litellm_requests.py tests/fixtures/litellm
8
+ """
9
+
10
+ import json
11
+ import os
12
+ import subprocess
13
+ import sys
14
+ import threading
15
+ from http.server import BaseHTTPRequestHandler, HTTPServer
16
+ from pathlib import Path
17
+
18
+ REPLY = {"output": {"message": {"role": "assistant", "content": [{"text": "OK"}]}}, "stopReason": "end_turn",
19
+ "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}
20
+ SONNET_46 = "bedrock/converse/us.anthropic.claude-sonnet-4-6"
21
+ SYSTEM = "You are a support agent for Acme. " + " ".join(
22
+ f"Rule {i}: answer politely and cite section {i}." for i in range(150))
23
+ TOOLS = [{"type": "function", "function": {"name": "lookup_order", "description": "Look up an order.",
24
+ "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}}]
25
+
26
+
27
+ def serve():
28
+ bodies = []
29
+
30
+ class Handler(BaseHTTPRequestHandler):
31
+ def do_POST(self):
32
+ bodies.append(json.loads(self.rfile.read(int(self.headers["Content-Length"]))))
33
+ data = json.dumps(REPLY).encode()
34
+ self.send_response(200)
35
+ self.send_header("Content-Type", "application/json")
36
+ self.send_header("Content-Length", str(len(data)))
37
+ self.end_headers()
38
+ self.wfile.write(data)
39
+
40
+ def log_message(self, *args):
41
+ pass
42
+
43
+ server = HTTPServer(("127.0.0.1", 0), Handler)
44
+ threading.Thread(target=server.serve_forever, daemon=True).start()
45
+ return f"http://127.0.0.1:{server.server_port}", bodies
46
+
47
+
48
+ def tool_round(n, prefix="t"):
49
+ calls = {"role": "assistant", "content": None, "tool_calls": [
50
+ {"id": f"{prefix}{i}", "type": "function",
51
+ "function": {"name": "lookup_order", "arguments": json.dumps({"id": f"{prefix}{i}"})}} for i in range(n)]}
52
+ results = [{"role": "tool", "tool_call_id": f"{prefix}{i}", "content": f"order {prefix}{i} shipped"} for i in range(n)]
53
+ return [calls] + results
54
+
55
+
56
+ def capture_ttl(out: Path, name: str, local_map: bool) -> None:
57
+ """The cost map is read at import, so each setting needs its own process."""
58
+ env = dict(os.environ)
59
+ if local_map:
60
+ env["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
61
+ subprocess.run([sys.executable, __file__, "--ttl-only", str(out / name)], env=env, check=True)
62
+
63
+
64
+ def main() -> int:
65
+ os.environ.update(AWS_ACCESS_KEY_ID="AKIAFAKE", AWS_SECRET_ACCESS_KEY="fake", AWS_REGION_NAME="us-west-2")
66
+ endpoint, bodies = serve()
67
+ import litellm
68
+
69
+ def capture(path: Path, **kwargs) -> None:
70
+ count = len(bodies)
71
+ litellm.completion(aws_bedrock_runtime_endpoint=endpoint, max_tokens=5, **kwargs)
72
+ assert len(bodies) == count + 1, f"no request captured for {path.name}"
73
+ path.write_text(json.dumps(bodies[-1], indent=2) + "\n")
74
+
75
+ if sys.argv[1] == "--ttl-only":
76
+ capture(Path(sys.argv[2]), model="bedrock/converse/us.anthropic.claude-sonnet-5-5",
77
+ messages=[{"role": "system", "content": [{"type": "text", "text": SYSTEM,
78
+ "cache_control": {"type": "ephemeral", "ttl": "1h"}}]},
79
+ {"role": "user", "content": "Hi"}])
80
+ return 0
81
+
82
+ out = Path(sys.argv[1])
83
+ out.mkdir(parents=True, exist_ok=True)
84
+ last_message = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]
85
+
86
+ # Finding 2: an agent step with 12 parallel tool calls.
87
+ turn1 = [{"role": "system", "content": SYSTEM},
88
+ {"role": "user", "content": "Check orders 1 to 12 and tell me which shipped."}]
89
+ capture(out / "agent-turn1.json", model=SONNET_46, tools=TOOLS, messages=turn1,
90
+ cache_control_injection_points=last_message)
91
+ capture(out / "agent-turn2.json", model=SONNET_46, tools=TOOLS, messages=turn1 + tool_round(12),
92
+ cache_control_injection_points=last_message)
93
+
94
+ # Finding 1: role "user" on a chat with more than three user messages.
95
+ by_role = [{"location": "message", "role": "system"}, {"location": "message", "role": "user"}]
96
+ chat = [{"role": "system", "content": SYSTEM}]
97
+ for i in range(5):
98
+ chat += [{"role": "user", "content": f"Question {i} about order {i}."},
99
+ {"role": "assistant", "content": f"Answer {i}."}]
100
+ capture(out / "chat-turn6.json", model=SONNET_46, messages=chat + [{"role": "user", "content": "Last question."}],
101
+ cache_control_injection_points=by_role)
102
+ capture(out / "chat-turn7.json", model=SONNET_46, cache_control_injection_points=by_role,
103
+ messages=chat + [{"role": "user", "content": "Last question."}, {"role": "assistant", "content": "Answer 5."},
104
+ {"role": "user", "content": "One more question."}])
105
+
106
+ # Finding 4: the same Sonnet 5.5 request with the downloaded and the built-in price list.
107
+ capture_ttl(out, "with-download.json", local_map=False)
108
+ capture_ttl(out, "without-download.json", local_map=True)
109
+ if '"ttl"' not in (out / "with-download.json").read_text():
110
+ sys.exit("LiteLLM couldn't download its price list (no network?), so with-download.json is wrong. Run again online.")
111
+ assert '"ttl"' not in (out / "without-download.json").read_text()
112
+ print(f"Saved {len(list(out.glob('*.json')))} requests to {out}")
113
+ return 0
114
+
115
+
116
+ if __name__ == "__main__":
117
+ sys.exit(main())
@@ -0,0 +1,68 @@
1
+ """Capture the exact Bedrock requests Strands Agents builds, offline, for tests/fixtures/strands/.
2
+
3
+ Strands runs its real agent loop (tool calls, streaming, conversation trimming) against a local stand-in
4
+ for bedrock-runtime (scripts/_fake_bedrock_stream.py) with fake credentials, so nothing leaves the machine.
5
+
6
+ pip install strands-agents==1.58.1
7
+ python scripts/capture_strands_requests.py tests/fixtures/strands
8
+ """
9
+
10
+ import json
11
+ import os
12
+ import sys
13
+ from pathlib import Path
14
+
15
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
16
+ from _fake_bedrock_stream import Fake # noqa: E402
17
+
18
+ os.environ.update(AWS_ACCESS_KEY_ID="AKIAFAKE", AWS_SECRET_ACCESS_KEY="fake", AWS_REGION="us-west-2")
19
+
20
+ from strands import Agent, tool # noqa: E402
21
+ from strands.models import BedrockModel # noqa: E402
22
+ from strands.models.model import CacheConfig # noqa: E402
23
+
24
+ MODEL = "us.anthropic.claude-sonnet-4-6"
25
+ SYSTEM = "You are a support agent for Acme. " + " ".join(
26
+ f"Rule {i}: answer politely and cite section {i}." for i in range(150))
27
+
28
+
29
+ @tool
30
+ def lookup_order(id: str) -> str:
31
+ """Look up an order."""
32
+ return f"order {id} shipped"
33
+
34
+
35
+ def main() -> int:
36
+ out = Path(sys.argv[1])
37
+ out.mkdir(parents=True, exist_ok=True)
38
+ fake = Fake()
39
+
40
+ def run(script, prompts, cache_config):
41
+ fake.reset(script)
42
+ model = BedrockModel(model_id=MODEL, endpoint_url=fake.url, region_name="us-west-2", cache_config=cache_config)
43
+ agent = Agent(model=model, tools=[lookup_order], system_prompt=SYSTEM, callback_handler=None)
44
+ for prompt in prompts:
45
+ agent(prompt)
46
+ return [request["body"] for request in fake.requests]
47
+
48
+ def save(name, body):
49
+ (out / name).write_text(json.dumps(body, indent=2) + "\n")
50
+
51
+ auto = CacheConfig(strategy="auto")
52
+ # Finding 2: one agent step with 11 parallel tool calls.
53
+ bodies = run(["tools:11", "text"], ["Check my orders."], auto)
54
+ save("agent-turn1.json", bodies[0])
55
+ save("agent-turn2.json", bodies[1])
56
+ # Finding 3: a long chat with the default sliding window (40 messages); turn 21 trims the first messages.
57
+ bodies = run(["text"] * 22, [f"Question {i} about order {i}." for i in range(22)], auto)
58
+ save("chat-turn20.json", bodies[20])
59
+ save("chat-turn21.json", bodies[21])
60
+ # Finding 5: a 1-hour default lifetime with a 5-minute system prompt point.
61
+ bodies = run(["text"], ["Hi"], CacheConfig(strategy="auto", ttl="1h", system_prompt_ttl="5m"))
62
+ save("ttl-order.json", bodies[0])
63
+ print(f"Saved {len(list(out.glob('*.json')))} requests to {out}")
64
+ return 0
65
+
66
+
67
+ if __name__ == "__main__":
68
+ sys.exit(main())
@@ -0,0 +1,170 @@
1
+ """Re-run the live numbers on cachecanary.com/litellm/ through LiteLLM against real Amazon Bedrock.
2
+
3
+ Synthetic prompts with a fresh random tag per case, so old cache entries can't help. About 20 small calls
4
+ on Claude Sonnet 4.6 and Sonnet 5.5 (a few cents). Uses your normal AWS credentials.
5
+
6
+ pip install litellm==1.104.0
7
+ python scripts/live_litellm_checks.py --region us-west-2
8
+
9
+ To include the application inference profile check, create a profile for Sonnet 4.6 and pass its ARN:
10
+
11
+ aws bedrock create-inference-profile --inference-profile-name cachecanary-test \
12
+ --model-source copyFrom=arn:aws:bedrock:REGION:ACCOUNT:inference-profile/us.anthropic.claude-sonnet-4-6
13
+ python scripts/live_litellm_checks.py --region us-west-2 --profile-arn ARN
14
+ aws bedrock delete-inference-profile --inference-profile-identifier ARN
15
+ """
16
+
17
+ import argparse
18
+ import json
19
+ import os
20
+ import subprocess
21
+ import sys
22
+ import time
23
+ import uuid
24
+
25
+ MODEL = "bedrock/converse/us.anthropic.claude-sonnet-4-6"
26
+ TOOLS = [{"type": "function", "function": {"name": "lookup_order", "description": "Look up an order.",
27
+ "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}}]
28
+ SYSTEM_AND_LAST = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]
29
+
30
+
31
+ # Same helper as on the page (tests/test_site.py checks they match).
32
+ def cache_points(messages):
33
+ """Bedrock cache points for this call (Bedrock allows 4)."""
34
+ # The system prompt and the end of the conversation.
35
+ points = [{"location": "message", "role": "system"},
36
+ {"location": "message", "index": -1}]
37
+ users = [i for i, m in enumerate(messages) if m["role"] == "user"]
38
+ rounds = [i for i, m in enumerate(messages)
39
+ if m["role"] == "assistant" and m.get("tool_calls")]
40
+ # The newest user message.
41
+ if users:
42
+ points.append({"location": "message", "index": users[-1]})
43
+ # The first tool result of the newest round of tool calls.
44
+ if rounds and (not users or rounds[-1] > users[-1]) \
45
+ and rounds[-1] + 1 < len(messages):
46
+ points.append({"location": "message", "index": rounds[-1] + 1})
47
+ return points
48
+
49
+
50
+ def lifetime_check() -> int:
51
+ """One call asking for a 1-hour cache; prints the lifetime Bedrock reports it used."""
52
+ model, extra = sys.argv[2], json.loads(sys.argv[3])
53
+ import litellm
54
+ from litellm.integrations.custom_logger import CustomLogger
55
+ seen = {}
56
+
57
+ class Raw(CustomLogger):
58
+ def log_success_event(self, kwargs, response_obj, start_time, end_time):
59
+ seen["raw"] = kwargs.get("original_response")
60
+
61
+ litellm.callbacks = [Raw()]
62
+ system = f"[run {uuid.uuid4().hex}] " + " ".join(f"Rule {i}: cite section {i}." for i in range(250))
63
+ litellm.completion(model=model, max_tokens=5, **extra,
64
+ messages=[{"role": "system", "content": system}, {"role": "user", "content": "Reply OK."}],
65
+ cache_control_injection_points=[{"location": "message", "role": "system",
66
+ "control": {"type": "ephemeral", "ttl": "1h"}}])
67
+ time.sleep(1)
68
+ raw = seen["raw"]
69
+ details = (json.loads(raw) if isinstance(raw, str) else raw)["usage"].get("cacheDetails") or []
70
+ print(json.dumps({"lifetimes": [d["ttl"] for d in details]}))
71
+ return 0
72
+
73
+
74
+ def main() -> int:
75
+ if sys.argv[1:2] == ["--lifetime-check"]:
76
+ return lifetime_check()
77
+ parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
78
+ parser.add_argument("--region", required=True)
79
+ parser.add_argument("--profile-arn", help="an application inference profile for Sonnet 4.6 (optional)")
80
+ args = parser.parse_args()
81
+ os.environ["AWS_REGION_NAME"] = args.region
82
+ import litellm
83
+
84
+ def call(case, messages, points, tools=None):
85
+ r = litellm.completion(model=MODEL, max_tokens=5, messages=messages, tools=tools,
86
+ cache_control_injection_points=points)
87
+ u = r.usage
88
+ row = {"case": case, "read": u.cache_read_input_tokens, "write": u.cache_creation_input_tokens,
89
+ "uncached": u.prompt_tokens - u.cache_read_input_tokens - u.cache_creation_input_tokens,
90
+ "prompt": u.prompt_tokens}
91
+ print(json.dumps(row), flush=True)
92
+ time.sleep(2)
93
+ return row
94
+
95
+ def fresh():
96
+ tag = uuid.uuid4().hex
97
+ system = f"[run {tag}] You are a support agent for Acme. " + " ".join(
98
+ f"Rule {i}: answer politely and cite section {i}." for i in range(150))
99
+ history = f"[doc {tag}] Customer history: " + " ".join(
100
+ f"Order {i} was placed on day {i} and contained item {i * 7}." for i in range(250))
101
+ return [{"role": "system", "content": system}, {"role": "user", "content": history + " Check my orders."}]
102
+
103
+ def tool_round(r, n):
104
+ calls = {"role": "assistant", "content": None, "tool_calls": [
105
+ {"id": f"r{r}t{i}", "type": "function",
106
+ "function": {"name": "lookup_order", "arguments": json.dumps({"id": f"{r}-{i}"})}} for i in range(n)]}
107
+ return [calls] + [{"role": "tool", "tool_call_id": f"r{r}t{i}", "content": f"order {r}-{i} shipped on day {i}. " * 8}
108
+ for i in range(n)]
109
+
110
+ checks = []
111
+
112
+ # Finding 1: role "user" vs the last message, turn 10 of a chat.
113
+ for label, points in (("role user", [{"location": "message", "role": "system"}, {"location": "message", "role": "user"}]),
114
+ ("last message", SYSTEM_AND_LAST)):
115
+ chat = fresh()[:1]
116
+ for i in range(8):
117
+ chat += [{"role": "user", "content": f"Question {i}: summarise order batch {i}."},
118
+ {"role": "assistant", "content": " ".join(f"Batch {i} line {j}: order {j} shipped on day {j}." for j in range(60))}]
119
+ call(f"{label}, turn 9", chat + [{"role": "user", "content": "Question 8?"}], points)
120
+ chat += [{"role": "user", "content": "Question 8?"}, {"role": "assistant", "content": "Answer 8. " * 200}]
121
+ checks.append((label, call(f"{label}, turn 10", chat + [{"role": "user", "content": "Question 9?"}], points)))
122
+ checks_ok = [checks[0][1]["uncached"] > 5000, checks[1][1]["uncached"] < 20]
123
+
124
+ # Finding 2: 4 vs 12 parallel tool calls with points on the system prompt and the last message.
125
+ start = fresh()
126
+ call("agent start", start, SYSTEM_AND_LAST, TOOLS)
127
+ twelve = call("12 parallel calls", start + tool_round(1, 12), SYSTEM_AND_LAST, TOOLS)
128
+ four = call("4 parallel calls", start + tool_round(1, 4), SYSTEM_AND_LAST, TOOLS)
129
+ checks_ok += [twelve["read"] < four["read"] / 2, four["write"] < 1000]
130
+
131
+ # The helper through a 7-step session, then a 21-call round.
132
+ steps = [fresh()]
133
+ for r, n in ((1, 12), (2, 1), (3, 20)):
134
+ steps.append(steps[-1] + tool_round(r, n))
135
+ steps.append(steps[-1] + [{"role": "assistant", "content": "All checked."}, {"role": "user", "content": "Refund order 3-4."}])
136
+ steps.append(steps[-1] + tool_round(4, 3))
137
+ previous = None
138
+ for i, messages in enumerate(steps):
139
+ row = call(f"helper step {i}", messages, cache_points(messages), TOOLS)
140
+ if previous is not None:
141
+ checks_ok.append(row["read"] >= previous["prompt"] - 5) # read almost the whole previous request
142
+ previous = row
143
+ wide = call("helper, 21 parallel calls", steps[-1] + tool_round(5, 21), cache_points(steps[-1] + tool_round(5, 21)), TOOLS)
144
+ checks_ok.append(wide["read"] < previous["prompt"] - 5)
145
+
146
+ # Finding 4: the 1-hour lifetime. The price list is read at import, so each case runs in its own process.
147
+ def lifetime(case, model, extra=None, built_in_prices=False):
148
+ env = dict(os.environ)
149
+ if built_in_prices:
150
+ env["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
151
+ out = subprocess.run([sys.executable, __file__, "--lifetime-check", model, json.dumps(extra or {})],
152
+ env=env, capture_output=True, text=True, check=True).stdout
153
+ got = json.loads(out.strip().splitlines()[-1])["lifetimes"]
154
+ print(json.dumps({"case": case, "bedrock_cache_lifetime": got}), flush=True)
155
+ return got
156
+
157
+ checks_ok.append(lifetime("Sonnet 5.5, downloaded price list", "bedrock/converse/us.anthropic.claude-sonnet-5-5") == ["1h"])
158
+ checks_ok.append(lifetime("Sonnet 5.5, built-in price list", "bedrock/converse/us.anthropic.claude-sonnet-5-5",
159
+ built_in_prices=True) == ["5m"])
160
+ checks_ok.append(lifetime("Sonnet 4.6, built-in price list", MODEL, built_in_prices=True) == ["1h"])
161
+ if args.profile_arn:
162
+ checks_ok.append(lifetime("profile ARN as the model", f"bedrock/converse/{args.profile_arn}") == ["5m"])
163
+ checks_ok.append(lifetime("model name + model_id=ARN", MODEL, {"model_id": args.profile_arn}) == ["1h"])
164
+
165
+ print(f"\n{sum(checks_ok)}/{len(checks_ok)} checks matched the page.")
166
+ return 0 if all(checks_ok) else 1
167
+
168
+
169
+ if __name__ == "__main__":
170
+ sys.exit(main())