cachecanary 0.3.2__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. {cachecanary-0.3.2 → cachecanary-0.4.1}/.github/workflows/ci.yml +24 -1
  2. {cachecanary-0.3.2 → cachecanary-0.4.1}/PKG-INFO +47 -6
  3. {cachecanary-0.3.2 → cachecanary-0.4.1}/README.md +46 -5
  4. {cachecanary-0.3.2 → cachecanary-0.4.1}/action.yml +3 -3
  5. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/capture_langchain_requests.py +2 -1
  6. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/capture_litellm_requests.py +2 -1
  7. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/capture_strands_requests.py +2 -1
  8. cachecanary-0.4.1/scripts/langfuse_capture/README.md +22 -0
  9. cachecanary-0.4.1/scripts/langfuse_capture/fake_bedrock.py +26 -0
  10. cachecanary-0.4.1/scripts/langfuse_capture/send_lc.py +15 -0
  11. cachecanary-0.4.1/scripts/langfuse_capture/send_litellm.py +21 -0
  12. cachecanary-0.4.1/scripts/langfuse_capture/send_strands.py +18 -0
  13. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_langchain_checks.py +10 -9
  14. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_litellm_checks.py +8 -26
  15. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_lookback_boundary.py +5 -1
  16. cachecanary-0.4.1/scripts/live_lookback_positions.py +196 -0
  17. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_strands_checks.py +18 -53
  18. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_test.py +21 -17
  19. cachecanary-0.4.1/scripts/og-cards/og-check.html +9 -0
  20. cachecanary-0.4.1/scripts/og-cards/og-langfuse.html +9 -0
  21. cachecanary-0.4.1/site/check/index.html +196 -0
  22. cachecanary-0.4.1/site/check.js +240 -0
  23. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/index.html +19 -9
  24. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/langchain/index.html +23 -35
  25. cachecanary-0.4.1/site/langfuse/index.html +188 -0
  26. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/litellm/index.html +34 -44
  27. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/llms.txt +9 -7
  28. cachecanary-0.4.1/site/og-check.png +0 -0
  29. cachecanary-0.4.1/site/og-langfuse.png +0 -0
  30. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/sitemap.xml +12 -4
  31. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/strands/index.html +25 -72
  32. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/style.css +40 -0
  33. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/__init__.py +1 -1
  34. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/cli.py +112 -10
  35. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/diff.py +36 -8
  36. cachecanary-0.4.1/src/cachecanary/langfuse.py +404 -0
  37. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/models.py +8 -6
  38. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/request.py +27 -1
  39. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/PKG-INFO +47 -6
  40. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/SOURCES.txt +17 -0
  41. cachecanary-0.4.1/tests/fixtures/langfuse_observations_2026-10.json +487 -0
  42. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_action.py +15 -0
  43. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_cachecanary.py +16 -9
  44. cachecanary-0.4.1/tests/test_check_page.py +203 -0
  45. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_edge_cases.py +192 -29
  46. cachecanary-0.4.1/tests/test_langfuse.py +721 -0
  47. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_site.py +95 -93
  48. {cachecanary-0.3.2 → cachecanary-0.4.1}/.github/workflows/publish.yml +0 -0
  49. {cachecanary-0.3.2 → cachecanary-0.4.1}/LICENSE +0 -0
  50. {cachecanary-0.3.2 → cachecanary-0.4.1}/MANIFEST.in +0 -0
  51. {cachecanary-0.3.2 → cachecanary-0.4.1}/NOTICE +0 -0
  52. {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_bad.json +0 -0
  53. {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_good.json +0 -0
  54. {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_next.json +0 -0
  55. {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_too_short.json +0 -0
  56. {cachecanary-0.3.2 → cachecanary-0.4.1}/pyproject.toml +0 -0
  57. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/_fake_bedrock_stream.py +0 -0
  58. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/check_site.mjs +0 -0
  59. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_logs_test.py +0 -0
  60. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_v03_checks.py +0 -0
  61. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/make_log_fixture.py +0 -0
  62. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og-langchain.html +0 -0
  63. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og-litellm.html +0 -0
  64. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og-strands.html +0 -0
  65. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og.html +0 -0
  66. {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/serve_site.py +0 -0
  67. {cachecanary-0.3.2 → cachecanary-0.4.1}/setup.cfg +0 -0
  68. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/404.html +0 -0
  69. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/_headers +0 -0
  70. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/apple-touch-icon.png +0 -0
  71. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/favicon.ico +0 -0
  72. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/favicon.svg +0 -0
  73. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/icon-192.png +0 -0
  74. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/main.js +0 -0
  75. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og-langchain.png +0 -0
  76. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og-litellm.png +0 -0
  77. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og-strands.png +0 -0
  78. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og.png +0 -0
  79. {cachecanary-0.3.2 → cachecanary-0.4.1}/site/robots.txt +0 -0
  80. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/github.py +0 -0
  81. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/lint.py +0 -0
  82. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/logs.py +0 -0
  83. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/pricing.py +0 -0
  84. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/probe.py +0 -0
  85. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/usage.py +0 -0
  86. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/dependency_links.txt +0 -0
  87. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/entry_points.txt +0 -0
  88. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/requires.txt +0 -0
  89. {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/top_level.txt +0 -0
  90. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/bedrock_invocation_logs_2026-10.json +0 -0
  91. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/date-day1.json +0 -0
  92. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/date-day2.json +0 -0
  93. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/edit-call8.json +0 -0
  94. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/edit-call9.json +0 -0
  95. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/invoke-turn1.json +0 -0
  96. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/invoke-turn2.json +0 -0
  97. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/agent-turn1.json +0 -0
  98. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/agent-turn2.json +0 -0
  99. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/chat-turn6.json +0 -0
  100. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/chat-turn7.json +0 -0
  101. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/with-download.json +0 -0
  102. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/without-download.json +0 -0
  103. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/agent-turn1.json +0 -0
  104. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/agent-turn2.json +0 -0
  105. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/chat-turn20.json +0 -0
  106. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/chat-turn21.json +0 -0
  107. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/ttl-order.json +0 -0
  108. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_github.py +0 -0
  109. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_pricing.py +0 -0
  110. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_readme.py +0 -0
  111. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_scripts_and_formats.py +0 -0
  112. {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_v03_rules.py +0 -0
  113. {cachecanary-0.3.2 → cachecanary-0.4.1}/wrangler.jsonc +0 -0
@@ -51,6 +51,28 @@ jobs:
51
51
  command: diff
52
52
  args: examples/request_bad.json examples/request_next.json
53
53
 
54
+ - name: langfuse on the saved real records (must fail the 80% target)
55
+ id: langfuse_file
56
+ continue-on-error: true
57
+ uses: ./
58
+ with:
59
+ command: langfuse
60
+ args: --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
61
+
62
+ # Proves Langfuse settings given as step env reach the action: the host below is unreachable on purpose,
63
+ # so the step must fail with "could not reach Langfuse at http://127.0.0.1:9".
64
+ - name: langfuse reads keys and host from step env (must fail, unreachable host)
65
+ id: langfuse_env
66
+ continue-on-error: true
67
+ uses: ./
68
+ env:
69
+ LANGFUSE_PUBLIC_KEY: pk-lf-ci
70
+ LANGFUSE_SECRET_KEY: sk-lf-ci
71
+ LANGFUSE_HOST: http://127.0.0.1:9
72
+ with:
73
+ command: langfuse
74
+ args: --since 1h
75
+
54
76
  - name: unknown command (must fail)
55
77
  id: bad_command
56
78
  continue-on-error: true
@@ -63,7 +85,8 @@ jobs:
63
85
  shell: bash
64
86
  run: |
65
87
  set -euo pipefail
66
- for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}"; do
88
+ for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}" \
89
+ "${{ steps.langfuse_file.outcome }}" "${{ steps.langfuse_env.outcome }}"; do
67
90
  if [ "$o" != "failure" ]; then echo "::error::expected failure, got '$o'"; exit 1; fi
68
91
  done
69
92
  echo "All expected failures happened."
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cachecanary
3
- Version: 0.3.2
3
+ Version: 0.4.1
4
4
  Summary: CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock, not when the bill arrives.
5
5
  Author-email: Haris Farooq <harisf98@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -46,11 +46,13 @@ I built CacheCanary after running into these failures while running Claude agent
46
46
 
47
47
  I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
48
48
 
49
- Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
49
+ Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/). If you use Langfuse, the [Langfuse guide](https://cachecanary.com/langfuse/) shows how to read your real hit rate from your traces.
50
50
 
51
51
  What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
52
52
 
53
- CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
53
+ CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs or your Langfuse traces to show hit rates in production.
54
+
55
+ Want a quick number first? Paste three totals from CloudWatch into the [cache checker](https://cachecanary.com/check/). It runs in your browser, nothing to install.
54
56
 
55
57
  ## Install
56
58
 
@@ -90,6 +92,7 @@ The last one is easy to miss. The same prompt caches fine on Sonnet 4.6 (minimum
90
92
  | `diff previous.json next.json` | Explains why the second request missed the first one's cache | the cached part changed |
91
93
  | `probe request.json --model ID [--region R] [--stream]` | Sends the request twice and checks the second one read from cache | nothing was read from cache |
92
94
  | `logs files-or-folders... [--by model\|principal] [--min-hit 0.5] [--price 3]` | Hit rate and input cost per model or IAM principal from Bedrock invocation logs, plus how much the misses cost | a group is under the threshold |
95
+ | `langfuse [--since 7d] [--environment E] [--by model\|name\|environment] [--min-hit 0.5] [--price 3]` | The same, from the generations your Langfuse project already records (no invocation logging needed) | a group is under the threshold |
93
96
 
94
97
  Exit code 2 means it couldn't run at all (bad file, no model access, no AWS credentials). `--json` and `--github` go before the command.
95
98
 
@@ -98,7 +101,7 @@ Exit code 2 means it couldn't run at all (bad file, no model access, no AWS cred
98
101
  - Dates, times, UUIDs or user-specific text before the cache checkpoint.
99
102
  - A cached prefix shorter than the model's minimum (512 to 4,096 tokens depending on the model).
100
103
  - Tools listed in a different order, or a tool definition that changed. Tools come first, so this throws away the whole cache.
101
- - More than 21 new content blocks between the old checkpoint and the new one, which is common after many parallel tool calls. Bedrock only looks back that far: measured live, 21 added blocks still hit and 22 always missed (`scripts/live_lookback_boundary.py`).
104
+ - More than 21 lookback positions between the old checkpoint and the new one, common in long chains of single tool calls or with many separate text blocks. Each text block is one position, a group of tool calls or tool results side by side is one, and thinking blocks don't count. Measured live on October 30 2026: 21 positions still hit and 22 always missed (`scripts/live_lookback_positions.py`). On October 7 2026 Bedrock still counted every tool call and result, so many parallel tool calls lost the cache; by October 28 they no longer did.
102
105
  - Model IDs or application inference profile ARNs that your library may not recognize, so it quietly stops sending cache markers. Legacy models (Sonnet 4, Opus 4 and 4.1, 3.5 Haiku) are checked against Anthropic's documented minimums, since AWS's caching table doesn't list them.
103
106
  - A switch between the Converse and InvokeModel APIs, which builds a different prompt.
104
107
  - More than 4 checkpoints, an unknown cache lifetime (TTL), or a 1-hour checkpoint after a 5-minute one.
@@ -126,6 +129,16 @@ Problems show up as annotations on the pull request, plus a short table in the j
126
129
  with:
127
130
  command: probe
128
131
  args: tests/fixtures/agent_request.json --model us.anthropic.claude-sonnet-4-6 --region us-west-2
132
+
133
+ # Reads last week's hit rate from Langfuse and fails under 80%. Good as a scheduled workflow.
134
+ - uses: Haarris/cachecanary@v0
135
+ env:
136
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
137
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
138
+ LANGFUSE_HOST: https://us.cloud.langfuse.com
139
+ with:
140
+ command: langfuse
141
+ args: --since 7d --environment production --min-hit 0.8
129
142
  ```
130
143
 
131
144
  `args` takes the same arguments as the CLI, separated by spaces. Paths with spaces aren't supported.
@@ -157,20 +170,48 @@ How the dollars work:
157
170
 
158
171
  It reads S3 deliveries, CloudWatch exports and `aws logs filter-log-events` output, gzipped or not, including streamed responses. Bedrock logs InvokeModel calls under an inference profile ARN and Converse calls under the short model ID, so CacheCanary merges the two.
159
172
 
173
+ ## Hit rates from Langfuse
174
+
175
+ If you trace your agents with [Langfuse](https://langfuse.com), you don't need invocation logging (more detail in the [Langfuse guide](https://cachecanary.com/langfuse/)). Langfuse already stores the cache counts for every model call, and `langfuse` reads them through the Langfuse API:
176
+
177
+ ```bash
178
+ export LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-...
179
+ export LANGFUSE_HOST=https://us.cloud.langfuse.com # or https://cloud.langfuse.com (EU), or your own server
180
+ cachecanary langfuse --since 7d --environment production --min-hit 0.8
181
+ ```
182
+
183
+ It uses only token counts, model names and timestamps, and stores nothing. On Langfuse v4 (Cloud and self-hosted) it asks the v2 observations API for just those fields, so prompts and outputs aren't downloaded. Older self-hosted servers only have the v1 API, which sends whole records; CacheCanary ignores everything except the token counts.
184
+
185
+ Depending on the integration, the counts stored in Langfuse don't always add up. Strands before 1.54 sent Bedrock's input count after the cache was already taken out, and Langfuse takes the cache out again, so the input often shows as 0 and the cache looks perfect. LiteLLM's `langfuse` callback leaves cache writes inside the input count as well as reporting them on their own. When a call has a total that was really sent, CacheCanary checks the parts against it, rebuilds the uncached input from the total (Bedrock's total includes the cached tokens), and tells you how many calls it fixed.
186
+
187
+ This is the output on real Langfuse records in this repo. They came from Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain, each making one cache write and one cache read:
188
+
189
+ ```text
190
+ $ cachecanary langfuse --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
191
+ us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0) <-- below threshold
192
+ input cost $0.13 at list price; about $0.07 of it lost to cache misses (target: 80% hit rate)
193
+ List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). Use --price for your own rate.
194
+ Read 10 generations.
195
+ 2 generation(s): Langfuse's input count did not add up with the total that was sent, so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's langfuse callback).
196
+ ```
197
+
198
+ Only model calls (generations) are counted. Agent and chain spans can carry the summed usage of the calls under them, so counting them would count tokens twice. `--file` reads a saved observations API response instead of calling Langfuse.
199
+
160
200
  ## How it's tested
161
201
 
162
- The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)).
202
+ The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 positions hit, 22 miss, re-measured October 30 2026 on Sonnet 4.6 and 5.5 after Bedrock changed how it counts tool calls). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model.
163
203
 
164
204
  To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region <region>`.
165
205
 
166
206
  ## Privacy
167
207
 
168
- Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials.
208
+ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials, and `langfuse` only reads from your own Langfuse project with your own keys and stores nothing.
169
209
 
170
210
  ## Limits
171
211
 
172
212
  - Token counts in `lint` are estimates. On English text Bedrock counted about 11% more than CacheCanary did, so near a model's minimum it warns you and suggests running `probe`.
173
213
  - Bedrock keeps cache counts inside the logged response body, and only bodies up to 100 KB are stored inline. Bigger ones show up as `unparsed` and aren't in the dollar figures.
214
+ - `langfuse` can only check a call's counts when the integration sent a total. Calls with no cache fields at all are counted as uncached and listed in the output, because Langfuse can't tell "caching was off" from "the integration doesn't report cache tokens".
174
215
  - Dollar figures use list prices, which change. Batch, provisioned throughput and priority tiers aren't covered; use `--price` for those.
175
216
  - Bedrock only for now.
176
217
 
@@ -13,11 +13,13 @@ I built CacheCanary after running into these failures while running Claude agent
13
13
 
14
14
  I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
15
15
 
16
- Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
16
+ Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/). If you use Langfuse, the [Langfuse guide](https://cachecanary.com/langfuse/) shows how to read your real hit rate from your traces.
17
17
 
18
18
  What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
19
19
 
20
- CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
20
+ CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs or your Langfuse traces to show hit rates in production.
21
+
22
+ Want a quick number first? Paste three totals from CloudWatch into the [cache checker](https://cachecanary.com/check/). It runs in your browser, nothing to install.
21
23
 
22
24
  ## Install
23
25
 
@@ -57,6 +59,7 @@ The last one is easy to miss. The same prompt caches fine on Sonnet 4.6 (minimum
57
59
  | `diff previous.json next.json` | Explains why the second request missed the first one's cache | the cached part changed |
58
60
  | `probe request.json --model ID [--region R] [--stream]` | Sends the request twice and checks the second one read from cache | nothing was read from cache |
59
61
  | `logs files-or-folders... [--by model\|principal] [--min-hit 0.5] [--price 3]` | Hit rate and input cost per model or IAM principal from Bedrock invocation logs, plus how much the misses cost | a group is under the threshold |
62
+ | `langfuse [--since 7d] [--environment E] [--by model\|name\|environment] [--min-hit 0.5] [--price 3]` | The same, from the generations your Langfuse project already records (no invocation logging needed) | a group is under the threshold |
60
63
 
61
64
  Exit code 2 means it couldn't run at all (bad file, no model access, no AWS credentials). `--json` and `--github` go before the command.
62
65
 
@@ -65,7 +68,7 @@ Exit code 2 means it couldn't run at all (bad file, no model access, no AWS cred
65
68
  - Dates, times, UUIDs or user-specific text before the cache checkpoint.
66
69
  - A cached prefix shorter than the model's minimum (512 to 4,096 tokens depending on the model).
67
70
  - Tools listed in a different order, or a tool definition that changed. Tools come first, so this throws away the whole cache.
68
- - More than 21 new content blocks between the old checkpoint and the new one, which is common after many parallel tool calls. Bedrock only looks back that far: measured live, 21 added blocks still hit and 22 always missed (`scripts/live_lookback_boundary.py`).
71
+ - More than 21 lookback positions between the old checkpoint and the new one, common in long chains of single tool calls or with many separate text blocks. Each text block is one position, a group of tool calls or tool results side by side is one, and thinking blocks don't count. Measured live on October 30 2026: 21 positions still hit and 22 always missed (`scripts/live_lookback_positions.py`). On October 7 2026 Bedrock still counted every tool call and result, so many parallel tool calls lost the cache; by October 28 they no longer did.
69
72
  - Model IDs or application inference profile ARNs that your library may not recognize, so it quietly stops sending cache markers. Legacy models (Sonnet 4, Opus 4 and 4.1, 3.5 Haiku) are checked against Anthropic's documented minimums, since AWS's caching table doesn't list them.
70
73
  - A switch between the Converse and InvokeModel APIs, which builds a different prompt.
71
74
  - More than 4 checkpoints, an unknown cache lifetime (TTL), or a 1-hour checkpoint after a 5-minute one.
@@ -93,6 +96,16 @@ Problems show up as annotations on the pull request, plus a short table in the j
93
96
  with:
94
97
  command: probe
95
98
  args: tests/fixtures/agent_request.json --model us.anthropic.claude-sonnet-4-6 --region us-west-2
99
+
100
+ # Reads last week's hit rate from Langfuse and fails under 80%. Good as a scheduled workflow.
101
+ - uses: Haarris/cachecanary@v0
102
+ env:
103
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
104
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
105
+ LANGFUSE_HOST: https://us.cloud.langfuse.com
106
+ with:
107
+ command: langfuse
108
+ args: --since 7d --environment production --min-hit 0.8
96
109
  ```
97
110
 
98
111
  `args` takes the same arguments as the CLI, separated by spaces. Paths with spaces aren't supported.
@@ -124,20 +137,48 @@ How the dollars work:
124
137
 
125
138
  It reads S3 deliveries, CloudWatch exports and `aws logs filter-log-events` output, gzipped or not, including streamed responses. Bedrock logs InvokeModel calls under an inference profile ARN and Converse calls under the short model ID, so CacheCanary merges the two.
126
139
 
140
+ ## Hit rates from Langfuse
141
+
142
+ If you trace your agents with [Langfuse](https://langfuse.com), you don't need invocation logging (more detail in the [Langfuse guide](https://cachecanary.com/langfuse/)). Langfuse already stores the cache counts for every model call, and `langfuse` reads them through the Langfuse API:
143
+
144
+ ```bash
145
+ export LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-...
146
+ export LANGFUSE_HOST=https://us.cloud.langfuse.com # or https://cloud.langfuse.com (EU), or your own server
147
+ cachecanary langfuse --since 7d --environment production --min-hit 0.8
148
+ ```
149
+
150
+ It uses only token counts, model names and timestamps, and stores nothing. On Langfuse v4 (Cloud and self-hosted) it asks the v2 observations API for just those fields, so prompts and outputs aren't downloaded. Older self-hosted servers only have the v1 API, which sends whole records; CacheCanary ignores everything except the token counts.
151
+
152
+ Depending on the integration, the counts stored in Langfuse don't always add up. Strands before 1.54 sent Bedrock's input count after the cache was already taken out, and Langfuse takes the cache out again, so the input often shows as 0 and the cache looks perfect. LiteLLM's `langfuse` callback leaves cache writes inside the input count as well as reporting them on their own. When a call has a total that was really sent, CacheCanary checks the parts against it, rebuilds the uncached input from the total (Bedrock's total includes the cached tokens), and tells you how many calls it fixed.
153
+
154
+ This is the output on real Langfuse records in this repo. They came from Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain, each making one cache write and one cache read:
155
+
156
+ ```text
157
+ $ cachecanary langfuse --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
158
+ us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0) <-- below threshold
159
+ input cost $0.13 at list price; about $0.07 of it lost to cache misses (target: 80% hit rate)
160
+ List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). Use --price for your own rate.
161
+ Read 10 generations.
162
+ 2 generation(s): Langfuse's input count did not add up with the total that was sent, so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's langfuse callback).
163
+ ```
164
+
165
+ Only model calls (generations) are counted. Agent and chain spans can carry the summed usage of the calls under them, so counting them would count tokens twice. `--file` reads a saved observations API response instead of calling Langfuse.
166
+
127
167
  ## How it's tested
128
168
 
129
- The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)).
169
+ The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 positions hit, 22 miss, re-measured October 30 2026 on Sonnet 4.6 and 5.5 after Bedrock changed how it counts tool calls). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model.
130
170
 
131
171
  To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region <region>`.
132
172
 
133
173
  ## Privacy
134
174
 
135
- Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials.
175
+ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials, and `langfuse` only reads from your own Langfuse project with your own keys and stores nothing.
136
176
 
137
177
  ## Limits
138
178
 
139
179
  - Token counts in `lint` are estimates. On English text Bedrock counted about 11% more than CacheCanary did, so near a model's minimum it warns you and suggests running `probe`.
140
180
  - Bedrock keeps cache counts inside the logged response body, and only bodies up to 100 KB are stored inline. Bigger ones show up as `unparsed` and aren't in the dollar figures.
181
+ - `langfuse` can only check a call's counts when the integration sent a total. Calls with no cache fields at all are counted as uncached and listed in the output, because Langfuse can't tell "caching was off" from "the integration doesn't report cache tokens".
141
182
  - Dollar figures use list prices, which change. Batch, provisioned throughput and priority tiers aren't covered; use `--price` for those.
142
183
  - Bedrock only for now.
143
184
 
@@ -7,7 +7,7 @@ branding:
7
7
 
8
8
  inputs:
9
9
  command:
10
- description: "One of: lint, diff, probe, logs"
10
+ description: "One of: lint, diff, probe, logs, langfuse"
11
11
  required: true
12
12
  args:
13
13
  description: >
@@ -38,8 +38,8 @@ runs:
38
38
  CC_ARGS: ${{ inputs.args }}
39
39
  run: |
40
40
  case "$CC_COMMAND" in
41
- lint|diff|probe|logs) ;;
42
- *) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe or logs)"; exit 2 ;;
41
+ lint|diff|probe|logs|langfuse) ;;
42
+ *) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe, logs or langfuse)"; exit 2 ;;
43
43
  esac
44
44
  read -r -a CC_ARGV <<< "$CC_ARGS"
45
45
  cachecanary --github "$CC_COMMAND" "${CC_ARGV[@]}"
@@ -52,7 +52,8 @@ def main() -> int:
52
52
  def save(name, body):
53
53
  (out / name).write_text(json.dumps(body, indent=2) + "\n")
54
54
 
55
- # Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls.
55
+ # Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now
56
+ # counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept.
56
57
  chat_bedrock = ChatBedrock(model=MODEL, region_name="us-west-2", endpoint_url=fake.url, model_kwargs={"max_tokens": 50})
57
58
  bodies = run(["tools:11", "text"], chat_bedrock, ["Check my orders."], [BedrockPromptCachingMiddleware()])
58
59
  save("invoke-turn1.json", bodies[0])
@@ -83,7 +83,8 @@ def main() -> int:
83
83
  out.mkdir(parents=True, exist_ok=True)
84
84
  last_message = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]
85
85
 
86
- # Finding 2: an agent step with 12 parallel tool calls.
86
+ # Finding 2: an agent step with 12 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now
87
+ # counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept.
87
88
  turn1 = [{"role": "system", "content": SYSTEM},
88
89
  {"role": "user", "content": "Check orders 1 to 12 and tell me which shipped."}]
89
90
  capture(out / "agent-turn1.json", model=SONNET_46, tools=TOOLS, messages=turn1,
@@ -49,7 +49,8 @@ def main() -> int:
49
49
  (out / name).write_text(json.dumps(body, indent=2) + "\n")
50
50
 
51
51
  auto = CacheConfig(strategy="auto")
52
- # Finding 2: one agent step with 11 parallel tool calls.
52
+ # Finding 2: one agent step with 11 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now
53
+ # counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept.
53
54
  bodies = run(["tools:11", "text"], ["Check my orders."], auto)
54
55
  save("agent-turn1.json", bodies[0])
55
56
  save("agent-turn2.json", bodies[1])
@@ -0,0 +1,22 @@
1
+ # What Langfuse stores for cached Bedrock calls
2
+
3
+ These scripts produced `tests/fixtures/langfuse_observations_2026-10.json` and the findings on
4
+ [cachecanary.com/langfuse](https://cachecanary.com/langfuse/) (October 20, 2026). Each library makes two real calls
5
+ to `fake_bedrock.py`, a local stand-in for Bedrock Converse that returns the token counts of a real Bedrock capture
6
+ ([strands-agents/sdk-python#3546](https://github.com/strands-agents/sdk-python/issues/3546)): the first call writes
7
+ 5,848 tokens to the cache, the second reads them, each with 10 new input tokens, 4 output tokens and a total of 5,862.
8
+ The traces go to a real local Langfuse, and the right answer for every pair is a 50% hit rate.
9
+
10
+ Versions used: Langfuse 4.64.0 (official docker compose), strands-agents 1.61.1 and 1.53.0, litellm 1.106.0 with
11
+ langfuse 4.18.0, langchain-aws 1.9.0 with langfuse 5.0.3. No AWS account or credentials are needed.
12
+
13
+ 1. Start Langfuse from its [docker-compose.yml](https://github.com/langfuse/langfuse/blob/main/docker-compose.yml)
14
+ with the web port on 43000 and a project created at start-up (`LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-lf-cctest`,
15
+ `LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-lf-cctest`, plus the other `LANGFUSE_INIT_*` values).
16
+ 2. `python fake_bedrock.py 45999 &`
17
+ 3. One virtual environment per library, then:
18
+ - `python send_strands.py new 45999` (strands-agents[otel] 1.61.1) and `python send_strands.py old 45999` (1.53.0)
19
+ - `python send_litellm.py 45999` and `python send_litellm.py 45999 --otel`
20
+ - `python send_lc.py 45999`
21
+ 4. Read the records back with `/api/public/v2/observations?type=GENERATION&fields=core,basic,model,usage`, or run
22
+ `cachecanary langfuse --host http://127.0.0.1:43000 --since 1h` with the same keys.
@@ -0,0 +1,26 @@
1
+ """Fake Bedrock Runtime Converse endpoint returning the live usage from strands#3546."""
2
+ import json, sys
3
+ from http.server import BaseHTTPRequestHandler, HTTPServer
4
+
5
+ # call 1 writes the cache, call 2 reads it (same numbers as the live capture)
6
+ USAGES = [
7
+ {"inputTokens": 10, "outputTokens": 4, "totalTokens": 5862, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 5848},
8
+ {"inputTokens": 10, "outputTokens": 4, "totalTokens": 5862, "cacheReadInputTokens": 5848, "cacheWriteInputTokens": 0},
9
+ ]
10
+ count = {"n": 0}
11
+
12
+ class H(BaseHTTPRequestHandler):
13
+ def log_message(self, *a): pass
14
+ def do_POST(self):
15
+ self.rfile.read(int(self.headers.get("Content-Length", 0)))
16
+ if not self.path.endswith("/converse"):
17
+ self.send_response(404); self.end_headers(); return
18
+ usage = USAGES[count["n"] % 2]; count["n"] += 1
19
+ body = json.dumps({"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
20
+ "stopReason": "end_turn", "usage": usage, "metrics": {"latencyMs": 12}}).encode()
21
+ self.send_response(200)
22
+ self.send_header("Content-Type", "application/json")
23
+ self.send_header("Content-Length", str(len(body)))
24
+ self.end_headers(); self.wfile.write(body)
25
+
26
+ HTTPServer(("127.0.0.1", int(sys.argv[1])), H).serve_forever()
@@ -0,0 +1,15 @@
1
+ """Two real ChatBedrockConverse calls against the fake Bedrock, traced by Langfuse's LangChain CallbackHandler."""
2
+ import os, sys
3
+ os.environ.update(LANGFUSE_PUBLIC_KEY="pk-lf-cctest", LANGFUSE_SECRET_KEY="sk-lf-cctest", LANGFUSE_BASE_URL="http://127.0.0.1:43000",
4
+ LANGFUSE_HOST="http://127.0.0.1:43000", LANGFUSE_TRACING_ENVIRONMENT="langchain",
5
+ AWS_ACCESS_KEY_ID="x", AWS_SECRET_ACCESS_KEY="x")
6
+ from langchain_aws import ChatBedrockConverse
7
+ from langfuse import get_client
8
+ from langfuse.langchain import CallbackHandler
9
+ llm = ChatBedrockConverse(model="us.anthropic.claude-sonnet-4-6", region_name="us-east-1",
10
+ endpoint_url=f"http://127.0.0.1:{sys.argv[1]}")
11
+ h = CallbackHandler()
12
+ for _ in range(2):
13
+ m = llm.invoke("hi", config={"callbacks": [h]})
14
+ print("langchain usage_metadata:", m.usage_metadata)
15
+ get_client().flush()
@@ -0,0 +1,21 @@
1
+ """Two real LiteLLM Bedrock Converse calls against fake_bedrock.py, logged by LiteLLM's `langfuse` callback
2
+ (or its `langfuse_otel` integration with --otel)."""
3
+ import os
4
+ import sys
5
+ import time
6
+
7
+ port, otel = sys.argv[1], "--otel" in sys.argv
8
+ os.environ.update(LANGFUSE_PUBLIC_KEY="pk-lf-cctest", LANGFUSE_SECRET_KEY="sk-lf-cctest", LANGFUSE_HOST="http://127.0.0.1:43000",
9
+ AWS_ACCESS_KEY_ID="x", AWS_SECRET_ACCESS_KEY="x", AWS_REGION_NAME="us-east-1")
10
+ import litellm # noqa: E402
11
+
12
+ if otel:
13
+ litellm.callbacks = ["langfuse_otel"]
14
+ else:
15
+ litellm.success_callback = ["langfuse"]
16
+ for _ in range(2):
17
+ r = litellm.completion(model="bedrock/converse/us.anthropic.claude-sonnet-4-6",
18
+ messages=[{"role": "user", "content": "hi"}], api_base=f"http://127.0.0.1:{port}",
19
+ metadata={"generation_name": "litellm-otel" if otel else "litellm-legacy"})
20
+ print("litellm usage:", dict(r.usage))
21
+ time.sleep(8) # both integrations send in the background
@@ -0,0 +1,18 @@
1
+ """Two real Strands Agent calls through BedrockModel (non-streaming) against the fake Bedrock,
2
+ traced to local Langfuse over OTLP."""
3
+ import base64, os, sys
4
+ tag = sys.argv[1]
5
+ auth = base64.b64encode(b"pk-lf-cctest:sk-lf-cctest").decode()
6
+ os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = "http://127.0.0.1:43000/api/public/otel/v1/traces"
7
+ os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = f"Authorization=Basic {auth}"
8
+ os.environ["AWS_ACCESS_KEY_ID"] = "x"; os.environ["AWS_SECRET_ACCESS_KEY"] = "x"
9
+ os.environ["AWS_ENDPOINT_URL_BEDROCK_RUNTIME"] = f"http://127.0.0.1:{sys.argv[2]}"
10
+ from strands import Agent
11
+ from strands.models import BedrockModel
12
+ from strands.telemetry import StrandsTelemetry
13
+ tel = StrandsTelemetry().setup_otlp_exporter()
14
+ model = BedrockModel(model_id="us.anthropic.claude-sonnet-4-6", region_name="us-east-1", streaming=False)
15
+ agent = Agent(model=model, name=f"strands-{tag}", trace_attributes={"langfuse.environment": tag})
16
+ agent("hello"); agent("hello again")
17
+ tel.tracer_provider.force_flush()
18
+ print("strands traces sent")
@@ -19,7 +19,6 @@ To include the application inference profile check, create a profile for Sonnet
19
19
 
20
20
  import argparse
21
21
  import json
22
- import os
23
22
  import sys
24
23
  import time
25
24
  import uuid
@@ -128,16 +127,17 @@ def main() -> int:
128
127
  checks += [looked_up[1]["read"] > 1000, named[1]["read"] > 1000]
129
128
 
130
129
  # Agent loops: LangChain's real loop on the stand-in, then the exact requests replayed to Bedrock.
131
- os.environ.setdefault("AWS_ACCESS_KEY_ID", "unused-by-the-stand-in")
132
130
  fake = Fake()
133
131
 
134
132
  def capture(script, *, invoke_route=False, middleware=None, manual_point=False):
135
133
  system, history = tag_prompts()
136
134
  fake.reset(script)
137
135
  if invoke_route:
138
- model = ChatBedrock(model=MODEL, region_name=args.region, endpoint_url=fake.url, model_kwargs={"max_tokens": 50})
136
+ model = ChatBedrock(model=MODEL, region_name=args.region, endpoint_url=fake.url, model_kwargs={"max_tokens": 50},
137
+ aws_access_key_id="unused-by-the-stand-in", aws_secret_access_key="unused-by-the-stand-in")
139
138
  else:
140
- model = ChatBedrockConverse(model=MODEL, region_name=args.region, endpoint_url=fake.url, max_tokens=50)
139
+ model = ChatBedrockConverse(model=MODEL, region_name=args.region, endpoint_url=fake.url, max_tokens=50,
140
+ aws_access_key_id="unused-by-the-stand-in", aws_secret_access_key="unused-by-the-stand-in")
141
141
  if manual_point:
142
142
  system = SystemMessage(content=[{"type": "text", "text": system}, ChatBedrockConverse.create_cache_point()])
143
143
  agent = create_agent(model=model, tools=[lookup_order], system_prompt=system,
@@ -172,14 +172,15 @@ def main() -> int:
172
172
  return kept(rows)
173
173
  return False
174
174
 
175
+ # Since Oct 2026 (by Oct 28) Bedrock counts a group of tool calls (and their results) as one lookback
176
+ # position, so every route keeps the whole conversation after a wide step.
175
177
  checks.append(kept_with_retry("ChatBedrockConverse, 21 parallel calls", ["tools:21", "text"]))
176
- checks.append(not kept(replay("ChatBedrockConverse, 22 parallel calls", capture(["tools:22", "text"]))))
178
+ checks.append(kept_with_retry("ChatBedrockConverse, 40 parallel calls", ["tools:40", "text"]))
177
179
  checks.append(kept_with_retry("ChatBedrock, 10 parallel calls", ["tools:10", "text"], invoke_route=True))
178
- invoke_11 = replay("ChatBedrock, 11 parallel calls", capture(["tools:11", "text"], invoke_route=True))
179
- checks.append(invoke_11[1]["read"] == 0) # nothing read: no separate system prompt marker on this route
180
+ checks.append(kept_with_retry("ChatBedrock, 40 parallel calls", ["tools:40", "text"], invoke_route=True))
180
181
  checks.append(kept_with_retry("middleware only, 11 parallel calls", ["tools:11", "text"]))
181
- checks.append(not kept(replay("manual system cache point + middleware, 11 parallel calls",
182
- capture(["tools:11", "text"], manual_point=True))))
182
+ checks.append(kept_with_retry("manual system cache point + middleware, 40 parallel calls", ["tools:40", "text"],
183
+ manual_point=True))
183
184
 
184
185
  # Context editing: stock clearing vs clearing in batches vs none, 13 calls with long tool results.
185
186
  tool_output["repeat"] = 40
@@ -25,28 +25,10 @@ import uuid
25
25
  MODEL = "bedrock/converse/us.anthropic.claude-sonnet-4-6"
26
26
  TOOLS = [{"type": "function", "function": {"name": "lookup_order", "description": "Look up an order.",
27
27
  "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}}]
28
+ # The setup on the page (tests/test_site.py checks they match).
28
29
  SYSTEM_AND_LAST = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]
29
30
 
30
31
 
31
- # Same helper as on the page (tests/test_site.py checks they match).
32
- def cache_points(messages):
33
- """Bedrock cache points for this call (Bedrock allows 4)."""
34
- # The system prompt and the end of the conversation.
35
- points = [{"location": "message", "role": "system"},
36
- {"location": "message", "index": -1}]
37
- users = [i for i, m in enumerate(messages) if m["role"] == "user"]
38
- rounds = [i for i, m in enumerate(messages)
39
- if m["role"] == "assistant" and m.get("tool_calls")]
40
- # The newest user message.
41
- if users:
42
- points.append({"location": "message", "index": users[-1]})
43
- # The first tool result of the newest round of tool calls.
44
- if rounds and (not users or rounds[-1] > users[-1]) \
45
- and rounds[-1] + 1 < len(messages):
46
- points.append({"location": "message", "index": rounds[-1] + 1})
47
- return points
48
-
49
-
50
32
  def lifetime_check() -> int:
51
33
  """One call asking for a 1-hour cache; prints the lifetime Bedrock reports it used."""
52
34
  model, extra = sys.argv[2], json.loads(sys.argv[3])
@@ -121,27 +103,27 @@ def main() -> int:
121
103
  checks.append((label, call(f"{label}, turn 10", chat + [{"role": "user", "content": "Question 9?"}], points)))
122
104
  checks_ok = [checks[0][1]["uncached"] > 5000, checks[1][1]["uncached"] < 20]
123
105
 
124
- # Finding 2: 4 vs 12 parallel tool calls with points on the system prompt and the last message.
106
+ # Finding 2: 4 vs 12 parallel tool calls with points on the system prompt and the last message. Since late
107
+ # Oct 2026 Bedrock counts a group of tool calls (and their results) as one lookback position, so both keep it.
125
108
  start = fresh()
126
- call("agent start", start, SYSTEM_AND_LAST, TOOLS)
109
+ first = call("agent start", start, SYSTEM_AND_LAST, TOOLS)
127
110
  twelve = call("12 parallel calls", start + tool_round(1, 12), SYSTEM_AND_LAST, TOOLS)
128
111
  four = call("4 parallel calls", start + tool_round(1, 4), SYSTEM_AND_LAST, TOOLS)
129
- checks_ok += [twelve["read"] < four["read"] / 2, four["write"] < 1000]
112
+ checks_ok += [twelve["read"] >= first["prompt"] - 5, four["read"] >= first["prompt"] - 5, four["write"] < 1000]
130
113
 
131
- # The helper through a 7-step session, then a 21-call round.
114
+ # The page's setup through an agent session with wide rounds: every step reads the whole previous request.
132
115
  steps = [fresh()]
133
116
  for r, n in ((1, 12), (2, 1), (3, 20)):
134
117
  steps.append(steps[-1] + tool_round(r, n))
135
118
  steps.append(steps[-1] + [{"role": "assistant", "content": "All checked."}, {"role": "user", "content": "Refund order 3-4."}])
136
119
  steps.append(steps[-1] + tool_round(4, 3))
120
+ steps.append(steps[-1] + tool_round(5, 40))
137
121
  previous = None
138
122
  for i, messages in enumerate(steps):
139
- row = call(f"helper step {i}", messages, cache_points(messages), TOOLS)
123
+ row = call(f"session step {i}", messages, SYSTEM_AND_LAST, TOOLS)
140
124
  if previous is not None:
141
125
  checks_ok.append(row["read"] >= previous["prompt"] - 5) # read almost the whole previous request
142
126
  previous = row
143
- wide = call("helper, 21 parallel calls", steps[-1] + tool_round(5, 21), cache_points(steps[-1] + tool_round(5, 21)), TOOLS)
144
- checks_ok.append(wide["read"] < previous["prompt"] - 5)
145
127
 
146
128
  # Finding 4: the 1-hour lifetime. The price list is read at import, so each case runs in its own process.
147
129
  def lifetime(case, model, extra=None, built_in_prices=False):
@@ -1,4 +1,8 @@
1
- """Measure where Bedrock's ~20-block cache lookback actually ends.
1
+ """Measure where Bedrock's ~20-block cache lookback actually ends (the first measurement, early Oct 2026).
2
+
3
+ Historical: when this ran, Bedrock counted every block, so 22+ blocks of parallel tool calls missed.
4
+ By Oct 28 2026 a run of tool calls (or tool results) counts as one position and these shapes all
5
+ hit. scripts/live_lookback_positions.py is the current check.
2
6
 
3
7
  Request A: tools + a short uncached system prompt + one long user turn ending in a cache point.
4
8
  Request B: the same history with that cache point removed, then an assistant turn with N parallel