cachecanary 0.3.2__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cachecanary-0.3.2 → cachecanary-0.4.1}/.github/workflows/ci.yml +24 -1
- {cachecanary-0.3.2 → cachecanary-0.4.1}/PKG-INFO +47 -6
- {cachecanary-0.3.2 → cachecanary-0.4.1}/README.md +46 -5
- {cachecanary-0.3.2 → cachecanary-0.4.1}/action.yml +3 -3
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/capture_langchain_requests.py +2 -1
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/capture_litellm_requests.py +2 -1
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/capture_strands_requests.py +2 -1
- cachecanary-0.4.1/scripts/langfuse_capture/README.md +22 -0
- cachecanary-0.4.1/scripts/langfuse_capture/fake_bedrock.py +26 -0
- cachecanary-0.4.1/scripts/langfuse_capture/send_lc.py +15 -0
- cachecanary-0.4.1/scripts/langfuse_capture/send_litellm.py +21 -0
- cachecanary-0.4.1/scripts/langfuse_capture/send_strands.py +18 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_langchain_checks.py +10 -9
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_litellm_checks.py +8 -26
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_lookback_boundary.py +5 -1
- cachecanary-0.4.1/scripts/live_lookback_positions.py +196 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_strands_checks.py +18 -53
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_test.py +21 -17
- cachecanary-0.4.1/scripts/og-cards/og-check.html +9 -0
- cachecanary-0.4.1/scripts/og-cards/og-langfuse.html +9 -0
- cachecanary-0.4.1/site/check/index.html +196 -0
- cachecanary-0.4.1/site/check.js +240 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/index.html +19 -9
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/langchain/index.html +23 -35
- cachecanary-0.4.1/site/langfuse/index.html +188 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/litellm/index.html +34 -44
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/llms.txt +9 -7
- cachecanary-0.4.1/site/og-check.png +0 -0
- cachecanary-0.4.1/site/og-langfuse.png +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/sitemap.xml +12 -4
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/strands/index.html +25 -72
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/style.css +40 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/__init__.py +1 -1
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/cli.py +112 -10
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/diff.py +36 -8
- cachecanary-0.4.1/src/cachecanary/langfuse.py +404 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/models.py +8 -6
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/request.py +27 -1
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/PKG-INFO +47 -6
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/SOURCES.txt +17 -0
- cachecanary-0.4.1/tests/fixtures/langfuse_observations_2026-10.json +487 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_action.py +15 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_cachecanary.py +16 -9
- cachecanary-0.4.1/tests/test_check_page.py +203 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_edge_cases.py +192 -29
- cachecanary-0.4.1/tests/test_langfuse.py +721 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_site.py +95 -93
- {cachecanary-0.3.2 → cachecanary-0.4.1}/.github/workflows/publish.yml +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/LICENSE +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/MANIFEST.in +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/NOTICE +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_bad.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_good.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_next.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/examples/request_too_short.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/pyproject.toml +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/_fake_bedrock_stream.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/check_site.mjs +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_logs_test.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/live_v03_checks.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/make_log_fixture.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og-langchain.html +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og-litellm.html +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og-strands.html +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/og-cards/og.html +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/scripts/serve_site.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/setup.cfg +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/404.html +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/_headers +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/apple-touch-icon.png +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/favicon.ico +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/favicon.svg +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/icon-192.png +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/main.js +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og-langchain.png +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og-litellm.png +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og-strands.png +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/og.png +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/site/robots.txt +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/github.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/lint.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/logs.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/pricing.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/probe.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary/usage.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/dependency_links.txt +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/entry_points.txt +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/requires.txt +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/src/cachecanary.egg-info/top_level.txt +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/bedrock_invocation_logs_2026-10.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/date-day1.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/date-day2.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/edit-call8.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/edit-call9.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/invoke-turn1.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/langchain/invoke-turn2.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/agent-turn1.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/agent-turn2.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/chat-turn6.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/chat-turn7.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/with-download.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/litellm/without-download.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/agent-turn1.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/agent-turn2.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/chat-turn20.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/chat-turn21.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/fixtures/strands/ttl-order.json +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_github.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_pricing.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_readme.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_scripts_and_formats.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/tests/test_v03_rules.py +0 -0
- {cachecanary-0.3.2 → cachecanary-0.4.1}/wrangler.jsonc +0 -0
|
@@ -51,6 +51,28 @@ jobs:
|
|
|
51
51
|
command: diff
|
|
52
52
|
args: examples/request_bad.json examples/request_next.json
|
|
53
53
|
|
|
54
|
+
- name: langfuse on the saved real records (must fail the 80% target)
|
|
55
|
+
id: langfuse_file
|
|
56
|
+
continue-on-error: true
|
|
57
|
+
uses: ./
|
|
58
|
+
with:
|
|
59
|
+
command: langfuse
|
|
60
|
+
args: --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
|
|
61
|
+
|
|
62
|
+
# Proves Langfuse settings given as step env reach the action: the host below is unreachable on purpose,
|
|
63
|
+
# so the step must fail with "could not reach Langfuse at http://127.0.0.1:9".
|
|
64
|
+
- name: langfuse reads keys and host from step env (must fail, unreachable host)
|
|
65
|
+
id: langfuse_env
|
|
66
|
+
continue-on-error: true
|
|
67
|
+
uses: ./
|
|
68
|
+
env:
|
|
69
|
+
LANGFUSE_PUBLIC_KEY: pk-lf-ci
|
|
70
|
+
LANGFUSE_SECRET_KEY: sk-lf-ci
|
|
71
|
+
LANGFUSE_HOST: http://127.0.0.1:9
|
|
72
|
+
with:
|
|
73
|
+
command: langfuse
|
|
74
|
+
args: --since 1h
|
|
75
|
+
|
|
54
76
|
- name: unknown command (must fail)
|
|
55
77
|
id: bad_command
|
|
56
78
|
continue-on-error: true
|
|
@@ -63,7 +85,8 @@ jobs:
|
|
|
63
85
|
shell: bash
|
|
64
86
|
run: |
|
|
65
87
|
set -euo pipefail
|
|
66
|
-
for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}"
|
|
88
|
+
for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}" \
|
|
89
|
+
"${{ steps.langfuse_file.outcome }}" "${{ steps.langfuse_env.outcome }}"; do
|
|
67
90
|
if [ "$o" != "failure" ]; then echo "::error::expected failure, got '$o'"; exit 1; fi
|
|
68
91
|
done
|
|
69
92
|
echo "All expected failures happened."
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cachecanary
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock, not when the bill arrives.
|
|
5
5
|
Author-email: Haris Farooq <harisf98@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -46,11 +46,13 @@ I built CacheCanary after running into these failures while running Claude agent
|
|
|
46
46
|
|
|
47
47
|
I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
|
|
48
48
|
|
|
49
|
-
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
|
|
49
|
+
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/). If you use Langfuse, the [Langfuse guide](https://cachecanary.com/langfuse/) shows how to read your real hit rate from your traces.
|
|
50
50
|
|
|
51
51
|
What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
|
|
52
52
|
|
|
53
|
-
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
|
|
53
|
+
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs or your Langfuse traces to show hit rates in production.
|
|
54
|
+
|
|
55
|
+
Want a quick number first? Paste three totals from CloudWatch into the [cache checker](https://cachecanary.com/check/). It runs in your browser, nothing to install.
|
|
54
56
|
|
|
55
57
|
## Install
|
|
56
58
|
|
|
@@ -90,6 +92,7 @@ The last one is easy to miss. The same prompt caches fine on Sonnet 4.6 (minimum
|
|
|
90
92
|
| `diff previous.json next.json` | Explains why the second request missed the first one's cache | the cached part changed |
|
|
91
93
|
| `probe request.json --model ID [--region R] [--stream]` | Sends the request twice and checks the second one read from cache | nothing was read from cache |
|
|
92
94
|
| `logs files-or-folders... [--by model\|principal] [--min-hit 0.5] [--price 3]` | Hit rate and input cost per model or IAM principal from Bedrock invocation logs, plus how much the misses cost | a group is under the threshold |
|
|
95
|
+
| `langfuse [--since 7d] [--environment E] [--by model\|name\|environment] [--min-hit 0.5] [--price 3]` | The same, from the generations your Langfuse project already records (no invocation logging needed) | a group is under the threshold |
|
|
93
96
|
|
|
94
97
|
Exit code 2 means it couldn't run at all (bad file, no model access, no AWS credentials). `--json` and `--github` go before the command.
|
|
95
98
|
|
|
@@ -98,7 +101,7 @@ Exit code 2 means it couldn't run at all (bad file, no model access, no AWS cred
|
|
|
98
101
|
- Dates, times, UUIDs or user-specific text before the cache checkpoint.
|
|
99
102
|
- A cached prefix shorter than the model's minimum (512 to 4,096 tokens depending on the model).
|
|
100
103
|
- Tools listed in a different order, or a tool definition that changed. Tools come first, so this throws away the whole cache.
|
|
101
|
-
- More than 21
|
|
104
|
+
- More than 21 lookback positions between the old checkpoint and the new one, common in long chains of single tool calls or with many separate text blocks. Each text block is one position, a group of tool calls or tool results side by side is one, and thinking blocks don't count. Measured live on October 30 2026: 21 positions still hit and 22 always missed (`scripts/live_lookback_positions.py`). On October 7 2026 Bedrock still counted every tool call and result, so many parallel tool calls lost the cache; by October 28 they no longer did.
|
|
102
105
|
- Model IDs or application inference profile ARNs that your library may not recognize, so it quietly stops sending cache markers. Legacy models (Sonnet 4, Opus 4 and 4.1, 3.5 Haiku) are checked against Anthropic's documented minimums, since AWS's caching table doesn't list them.
|
|
103
106
|
- A switch between the Converse and InvokeModel APIs, which builds a different prompt.
|
|
104
107
|
- More than 4 checkpoints, an unknown cache lifetime (TTL), or a 1-hour checkpoint after a 5-minute one.
|
|
@@ -126,6 +129,16 @@ Problems show up as annotations on the pull request, plus a short table in the j
|
|
|
126
129
|
with:
|
|
127
130
|
command: probe
|
|
128
131
|
args: tests/fixtures/agent_request.json --model us.anthropic.claude-sonnet-4-6 --region us-west-2
|
|
132
|
+
|
|
133
|
+
# Reads last week's hit rate from Langfuse and fails under 80%. Good as a scheduled workflow.
|
|
134
|
+
- uses: Haarris/cachecanary@v0
|
|
135
|
+
env:
|
|
136
|
+
LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
|
|
137
|
+
LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
|
|
138
|
+
LANGFUSE_HOST: https://us.cloud.langfuse.com
|
|
139
|
+
with:
|
|
140
|
+
command: langfuse
|
|
141
|
+
args: --since 7d --environment production --min-hit 0.8
|
|
129
142
|
```
|
|
130
143
|
|
|
131
144
|
`args` takes the same arguments as the CLI, separated by spaces. Paths with spaces aren't supported.
|
|
@@ -157,20 +170,48 @@ How the dollars work:
|
|
|
157
170
|
|
|
158
171
|
It reads S3 deliveries, CloudWatch exports and `aws logs filter-log-events` output, gzipped or not, including streamed responses. Bedrock logs InvokeModel calls under an inference profile ARN and Converse calls under the short model ID, so CacheCanary merges the two.
|
|
159
172
|
|
|
173
|
+
## Hit rates from Langfuse
|
|
174
|
+
|
|
175
|
+
If you trace your agents with [Langfuse](https://langfuse.com), you don't need invocation logging (more detail in the [Langfuse guide](https://cachecanary.com/langfuse/)). Langfuse already stores the cache counts for every model call, and `langfuse` reads them through the Langfuse API:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
export LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-...
|
|
179
|
+
export LANGFUSE_HOST=https://us.cloud.langfuse.com # or https://cloud.langfuse.com (EU), or your own server
|
|
180
|
+
cachecanary langfuse --since 7d --environment production --min-hit 0.8
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
It uses only token counts, model names and timestamps, and stores nothing. On Langfuse v4 (Cloud and self-hosted) it asks the v2 observations API for just those fields, so prompts and outputs aren't downloaded. Older self-hosted servers only have the v1 API, which sends whole records; CacheCanary ignores everything except the token counts.
|
|
184
|
+
|
|
185
|
+
Depending on the integration, the counts stored in Langfuse don't always add up. Strands before 1.54 sent Bedrock's input count after the cache was already taken out, and Langfuse takes the cache out again, so the input often shows as 0 and the cache looks perfect. LiteLLM's `langfuse` callback leaves cache writes inside the input count as well as reporting them on their own. When a call has a total that was really sent, CacheCanary checks the parts against it, rebuilds the uncached input from the total (Bedrock's total includes the cached tokens), and tells you how many calls it fixed.
|
|
186
|
+
|
|
187
|
+
This is the output on real Langfuse records in this repo. They came from Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain, each making one cache write and one cache read:
|
|
188
|
+
|
|
189
|
+
```text
|
|
190
|
+
$ cachecanary langfuse --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
|
|
191
|
+
us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0) <-- below threshold
|
|
192
|
+
input cost $0.13 at list price; about $0.07 of it lost to cache misses (target: 80% hit rate)
|
|
193
|
+
List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). Use --price for your own rate.
|
|
194
|
+
Read 10 generations.
|
|
195
|
+
2 generation(s): Langfuse's input count did not add up with the total that was sent, so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's langfuse callback).
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Only model calls (generations) are counted. Agent and chain spans can carry the summed usage of the calls under them, so counting them would count tokens twice. `--file` reads a saved observations API response instead of calling Langfuse.
|
|
199
|
+
|
|
160
200
|
## How it's tested
|
|
161
201
|
|
|
162
|
-
The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21
|
|
202
|
+
The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 positions hit, 22 miss, re-measured October 30 2026 on Sonnet 4.6 and 5.5 after Bedrock changed how it counts tool calls). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model.
|
|
163
203
|
|
|
164
204
|
To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region <region>`.
|
|
165
205
|
|
|
166
206
|
## Privacy
|
|
167
207
|
|
|
168
|
-
Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials.
|
|
208
|
+
Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials, and `langfuse` only reads from your own Langfuse project with your own keys and stores nothing.
|
|
169
209
|
|
|
170
210
|
## Limits
|
|
171
211
|
|
|
172
212
|
- Token counts in `lint` are estimates. On English text Bedrock counted about 11% more than CacheCanary did, so near a model's minimum it warns you and suggests running `probe`.
|
|
173
213
|
- Bedrock keeps cache counts inside the logged response body, and only bodies up to 100 KB are stored inline. Bigger ones show up as `unparsed` and aren't in the dollar figures.
|
|
214
|
+
- `langfuse` can only check a call's counts when the integration sent a total. Calls with no cache fields at all are counted as uncached and listed in the output, because Langfuse can't tell "caching was off" from "the integration doesn't report cache tokens".
|
|
174
215
|
- Dollar figures use list prices, which change. Batch, provisioned throughput and priority tiers aren't covered; use `--price` for those.
|
|
175
216
|
- Bedrock only for now.
|
|
176
217
|
|
|
@@ -13,11 +13,13 @@ I built CacheCanary after running into these failures while running Claude agent
|
|
|
13
13
|
|
|
14
14
|
I wrote up the five ways I've seen caching break on Bedrock, with the bug reports behind each one and the lookback limit I measured: [Five ways Claude prompt caching quietly breaks on Amazon Bedrock](https://harisfarooq.substack.com/p/five-ways-claude-prompt-caching-quietly).
|
|
15
15
|
|
|
16
|
-
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/).
|
|
16
|
+
Website: [cachecanary.com](https://cachecanary.com). It has guides that show exactly what a framework sends to Bedrock, where the cache is lost, and a tested setup that keeps it: [LiteLLM](https://cachecanary.com/litellm/), [Strands Agents](https://cachecanary.com/strands/) and [LangChain](https://cachecanary.com/langchain/). If you use Langfuse, the [Langfuse guide](https://cachecanary.com/langfuse/) shows how to read your real hit rate from your traces.
|
|
17
17
|
|
|
18
18
|
What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
|
|
19
19
|
|
|
20
|
-
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
|
|
20
|
+
CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs or your Langfuse traces to show hit rates in production.
|
|
21
|
+
|
|
22
|
+
Want a quick number first? Paste three totals from CloudWatch into the [cache checker](https://cachecanary.com/check/). It runs in your browser, nothing to install.
|
|
21
23
|
|
|
22
24
|
## Install
|
|
23
25
|
|
|
@@ -57,6 +59,7 @@ The last one is easy to miss. The same prompt caches fine on Sonnet 4.6 (minimum
|
|
|
57
59
|
| `diff previous.json next.json` | Explains why the second request missed the first one's cache | the cached part changed |
|
|
58
60
|
| `probe request.json --model ID [--region R] [--stream]` | Sends the request twice and checks the second one read from cache | nothing was read from cache |
|
|
59
61
|
| `logs files-or-folders... [--by model\|principal] [--min-hit 0.5] [--price 3]` | Hit rate and input cost per model or IAM principal from Bedrock invocation logs, plus how much the misses cost | a group is under the threshold |
|
|
62
|
+
| `langfuse [--since 7d] [--environment E] [--by model\|name\|environment] [--min-hit 0.5] [--price 3]` | The same, from the generations your Langfuse project already records (no invocation logging needed) | a group is under the threshold |
|
|
60
63
|
|
|
61
64
|
Exit code 2 means it couldn't run at all (bad file, no model access, no AWS credentials). `--json` and `--github` go before the command.
|
|
62
65
|
|
|
@@ -65,7 +68,7 @@ Exit code 2 means it couldn't run at all (bad file, no model access, no AWS cred
|
|
|
65
68
|
- Dates, times, UUIDs or user-specific text before the cache checkpoint.
|
|
66
69
|
- A cached prefix shorter than the model's minimum (512 to 4,096 tokens depending on the model).
|
|
67
70
|
- Tools listed in a different order, or a tool definition that changed. Tools come first, so this throws away the whole cache.
|
|
68
|
-
- More than 21
|
|
71
|
+
- More than 21 lookback positions between the old checkpoint and the new one, common in long chains of single tool calls or with many separate text blocks. Each text block is one position, a group of tool calls or tool results side by side is one, and thinking blocks don't count. Measured live on October 30 2026: 21 positions still hit and 22 always missed (`scripts/live_lookback_positions.py`). On October 7 2026 Bedrock still counted every tool call and result, so many parallel tool calls lost the cache; by October 28 they no longer did.
|
|
69
72
|
- Model IDs or application inference profile ARNs that your library may not recognize, so it quietly stops sending cache markers. Legacy models (Sonnet 4, Opus 4 and 4.1, 3.5 Haiku) are checked against Anthropic's documented minimums, since AWS's caching table doesn't list them.
|
|
70
73
|
- A switch between the Converse and InvokeModel APIs, which builds a different prompt.
|
|
71
74
|
- More than 4 checkpoints, an unknown cache lifetime (TTL), or a 1-hour checkpoint after a 5-minute one.
|
|
@@ -93,6 +96,16 @@ Problems show up as annotations on the pull request, plus a short table in the j
|
|
|
93
96
|
with:
|
|
94
97
|
command: probe
|
|
95
98
|
args: tests/fixtures/agent_request.json --model us.anthropic.claude-sonnet-4-6 --region us-west-2
|
|
99
|
+
|
|
100
|
+
# Reads last week's hit rate from Langfuse and fails under 80%. Good as a scheduled workflow.
|
|
101
|
+
- uses: Haarris/cachecanary@v0
|
|
102
|
+
env:
|
|
103
|
+
LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
|
|
104
|
+
LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
|
|
105
|
+
LANGFUSE_HOST: https://us.cloud.langfuse.com
|
|
106
|
+
with:
|
|
107
|
+
command: langfuse
|
|
108
|
+
args: --since 7d --environment production --min-hit 0.8
|
|
96
109
|
```
|
|
97
110
|
|
|
98
111
|
`args` takes the same arguments as the CLI, separated by spaces. Paths with spaces aren't supported.
|
|
@@ -124,20 +137,48 @@ How the dollars work:
|
|
|
124
137
|
|
|
125
138
|
It reads S3 deliveries, CloudWatch exports and `aws logs filter-log-events` output, gzipped or not, including streamed responses. Bedrock logs InvokeModel calls under an inference profile ARN and Converse calls under the short model ID, so CacheCanary merges the two.
|
|
126
139
|
|
|
140
|
+
## Hit rates from Langfuse
|
|
141
|
+
|
|
142
|
+
If you trace your agents with [Langfuse](https://langfuse.com), you don't need invocation logging (more detail in the [Langfuse guide](https://cachecanary.com/langfuse/)). Langfuse already stores the cache counts for every model call, and `langfuse` reads them through the Langfuse API:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
export LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-...
|
|
146
|
+
export LANGFUSE_HOST=https://us.cloud.langfuse.com # or https://cloud.langfuse.com (EU), or your own server
|
|
147
|
+
cachecanary langfuse --since 7d --environment production --min-hit 0.8
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
It uses only token counts, model names and timestamps, and stores nothing. On Langfuse v4 (Cloud and self-hosted) it asks the v2 observations API for just those fields, so prompts and outputs aren't downloaded. Older self-hosted servers only have the v1 API, which sends whole records; CacheCanary ignores everything except the token counts.
|
|
151
|
+
|
|
152
|
+
Depending on the integration, the counts stored in Langfuse don't always add up. Strands before 1.54 sent Bedrock's input count after the cache was already taken out, and Langfuse takes the cache out again, so the input often shows as 0 and the cache looks perfect. LiteLLM's `langfuse` callback leaves cache writes inside the input count as well as reporting them on their own. When a call has a total that was really sent, CacheCanary checks the parts against it, rebuilds the uncached input from the total (Bedrock's total includes the cached tokens), and tells you how many calls it fixed.
|
|
153
|
+
|
|
154
|
+
This is the output on real Langfuse records in this repo. They came from Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain, each making one cache write and one cache read:
|
|
155
|
+
|
|
156
|
+
```text
|
|
157
|
+
$ cachecanary langfuse --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
|
|
158
|
+
us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0) <-- below threshold
|
|
159
|
+
input cost $0.13 at list price; about $0.07 of it lost to cache misses (target: 80% hit rate)
|
|
160
|
+
List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). Use --price for your own rate.
|
|
161
|
+
Read 10 generations.
|
|
162
|
+
2 generation(s): Langfuse's input count did not add up with the total that was sent, so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's langfuse callback).
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Only model calls (generations) are counted. Agent and chain spans can carry the summed usage of the calls under them, so counting them would count tokens twice. `--file` reads a saved observations API response instead of calling Langfuse.
|
|
166
|
+
|
|
127
167
|
## How it's tested
|
|
128
168
|
|
|
129
|
-
The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21
|
|
169
|
+
The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 positions hit, 22 miss, re-measured October 30 2026 on Sonnet 4.6 and 5.5 after Bedrock changed how it counts tool calls). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model.
|
|
130
170
|
|
|
131
171
|
To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region <region>`.
|
|
132
172
|
|
|
133
173
|
## Privacy
|
|
134
174
|
|
|
135
|
-
Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials.
|
|
175
|
+
Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials, and `langfuse` only reads from your own Langfuse project with your own keys and stores nothing.
|
|
136
176
|
|
|
137
177
|
## Limits
|
|
138
178
|
|
|
139
179
|
- Token counts in `lint` are estimates. On English text Bedrock counted about 11% more than CacheCanary did, so near a model's minimum it warns you and suggests running `probe`.
|
|
140
180
|
- Bedrock keeps cache counts inside the logged response body, and only bodies up to 100 KB are stored inline. Bigger ones show up as `unparsed` and aren't in the dollar figures.
|
|
181
|
+
- `langfuse` can only check a call's counts when the integration sent a total. Calls with no cache fields at all are counted as uncached and listed in the output, because Langfuse can't tell "caching was off" from "the integration doesn't report cache tokens".
|
|
141
182
|
- Dollar figures use list prices, which change. Batch, provisioned throughput and priority tiers aren't covered; use `--price` for those.
|
|
142
183
|
- Bedrock only for now.
|
|
143
184
|
|
|
@@ -7,7 +7,7 @@ branding:
|
|
|
7
7
|
|
|
8
8
|
inputs:
|
|
9
9
|
command:
|
|
10
|
-
description: "One of: lint, diff, probe, logs"
|
|
10
|
+
description: "One of: lint, diff, probe, logs, langfuse"
|
|
11
11
|
required: true
|
|
12
12
|
args:
|
|
13
13
|
description: >
|
|
@@ -38,8 +38,8 @@ runs:
|
|
|
38
38
|
CC_ARGS: ${{ inputs.args }}
|
|
39
39
|
run: |
|
|
40
40
|
case "$CC_COMMAND" in
|
|
41
|
-
lint|diff|probe|logs) ;;
|
|
42
|
-
*) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe or
|
|
41
|
+
lint|diff|probe|logs|langfuse) ;;
|
|
42
|
+
*) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe, logs or langfuse)"; exit 2 ;;
|
|
43
43
|
esac
|
|
44
44
|
read -r -a CC_ARGV <<< "$CC_ARGS"
|
|
45
45
|
cachecanary --github "$CC_COMMAND" "${CC_ARGV[@]}"
|
|
@@ -52,7 +52,8 @@ def main() -> int:
|
|
|
52
52
|
def save(name, body):
|
|
53
53
|
(out / name).write_text(json.dumps(body, indent=2) + "\n")
|
|
54
54
|
|
|
55
|
-
# Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls.
|
|
55
|
+
# Finding 2: ChatBedrock (InvokeModel) with the caching middleware, 11 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now
|
|
56
|
+
# counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept.
|
|
56
57
|
chat_bedrock = ChatBedrock(model=MODEL, region_name="us-west-2", endpoint_url=fake.url, model_kwargs={"max_tokens": 50})
|
|
57
58
|
bodies = run(["tools:11", "text"], chat_bedrock, ["Check my orders."], [BedrockPromptCachingMiddleware()])
|
|
58
59
|
save("invoke-turn1.json", bodies[0])
|
|
@@ -83,7 +83,8 @@ def main() -> int:
|
|
|
83
83
|
out.mkdir(parents=True, exist_ok=True)
|
|
84
84
|
last_message = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]
|
|
85
85
|
|
|
86
|
-
# Finding 2: an agent step with 12 parallel tool calls.
|
|
86
|
+
# Finding 2: an agent step with 12 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now
|
|
87
|
+
# counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept.
|
|
87
88
|
turn1 = [{"role": "system", "content": SYSTEM},
|
|
88
89
|
{"role": "user", "content": "Check orders 1 to 12 and tell me which shipped."}]
|
|
89
90
|
capture(out / "agent-turn1.json", model=SONNET_46, tools=TOOLS, messages=turn1,
|
|
@@ -49,7 +49,8 @@ def main() -> int:
|
|
|
49
49
|
(out / name).write_text(json.dumps(body, indent=2) + "\n")
|
|
50
50
|
|
|
51
51
|
auto = CacheConfig(strategy="auto")
|
|
52
|
-
# Finding 2: one agent step with 11 parallel tool calls.
|
|
52
|
+
# Finding 2: one agent step with 11 parallel tool calls. On Oct 7 2026 this lost the cache; Bedrock now
|
|
53
|
+
# counts a group of tool calls as one lookback position, so cachecanary diff reports it as kept.
|
|
53
54
|
bodies = run(["tools:11", "text"], ["Check my orders."], auto)
|
|
54
55
|
save("agent-turn1.json", bodies[0])
|
|
55
56
|
save("agent-turn2.json", bodies[1])
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# What Langfuse stores for cached Bedrock calls
|
|
2
|
+
|
|
3
|
+
These scripts produced `tests/fixtures/langfuse_observations_2026-10.json` and the findings on
|
|
4
|
+
[cachecanary.com/langfuse](https://cachecanary.com/langfuse/) (October 20, 2026). Each library makes two real calls
|
|
5
|
+
to `fake_bedrock.py`, a local stand-in for Bedrock Converse that returns the token counts of a real Bedrock capture
|
|
6
|
+
([strands-agents/sdk-python#3546](https://github.com/strands-agents/sdk-python/issues/3546)): the first call writes
|
|
7
|
+
5,848 tokens to the cache, the second reads them, each with 10 new input tokens, 4 output tokens and a total of 5,862.
|
|
8
|
+
The traces go to a real local Langfuse, and the right answer for every pair is a 50% hit rate.
|
|
9
|
+
|
|
10
|
+
Versions used: Langfuse 4.64.0 (official docker compose), strands-agents 1.61.1 and 1.53.0, litellm 1.106.0 with
|
|
11
|
+
langfuse 4.18.0, langchain-aws 1.9.0 with langfuse 5.0.3. No AWS account or credentials are needed.
|
|
12
|
+
|
|
13
|
+
1. Start Langfuse from its [docker-compose.yml](https://github.com/langfuse/langfuse/blob/main/docker-compose.yml)
|
|
14
|
+
with the web port on 43000 and a project created at start-up (`LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-lf-cctest`,
|
|
15
|
+
`LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-lf-cctest`, plus the other `LANGFUSE_INIT_*` values).
|
|
16
|
+
2. `python fake_bedrock.py 45999 &`
|
|
17
|
+
3. One virtual environment per library, then:
|
|
18
|
+
- `python send_strands.py new 45999` (strands-agents[otel] 1.61.1) and `python send_strands.py old 45999` (1.53.0)
|
|
19
|
+
- `python send_litellm.py 45999` and `python send_litellm.py 45999 --otel`
|
|
20
|
+
- `python send_lc.py 45999`
|
|
21
|
+
4. Read the records back with `/api/public/v2/observations?type=GENERATION&fields=core,basic,model,usage`, or run
|
|
22
|
+
`cachecanary langfuse --host http://127.0.0.1:43000 --since 1h` with the same keys.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Fake Bedrock Runtime Converse endpoint returning the live usage from strands#3546."""
|
|
2
|
+
import json, sys
|
|
3
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
4
|
+
|
|
5
|
+
# call 1 writes the cache, call 2 reads it (same numbers as the live capture)
|
|
6
|
+
USAGES = [
|
|
7
|
+
{"inputTokens": 10, "outputTokens": 4, "totalTokens": 5862, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 5848},
|
|
8
|
+
{"inputTokens": 10, "outputTokens": 4, "totalTokens": 5862, "cacheReadInputTokens": 5848, "cacheWriteInputTokens": 0},
|
|
9
|
+
]
|
|
10
|
+
count = {"n": 0}
|
|
11
|
+
|
|
12
|
+
class H(BaseHTTPRequestHandler):
|
|
13
|
+
def log_message(self, *a): pass
|
|
14
|
+
def do_POST(self):
|
|
15
|
+
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
|
16
|
+
if not self.path.endswith("/converse"):
|
|
17
|
+
self.send_response(404); self.end_headers(); return
|
|
18
|
+
usage = USAGES[count["n"] % 2]; count["n"] += 1
|
|
19
|
+
body = json.dumps({"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
|
|
20
|
+
"stopReason": "end_turn", "usage": usage, "metrics": {"latencyMs": 12}}).encode()
|
|
21
|
+
self.send_response(200)
|
|
22
|
+
self.send_header("Content-Type", "application/json")
|
|
23
|
+
self.send_header("Content-Length", str(len(body)))
|
|
24
|
+
self.end_headers(); self.wfile.write(body)
|
|
25
|
+
|
|
26
|
+
HTTPServer(("127.0.0.1", int(sys.argv[1])), H).serve_forever()
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Two real ChatBedrockConverse calls against the fake Bedrock, traced by Langfuse's LangChain CallbackHandler."""
|
|
2
|
+
import os, sys
|
|
3
|
+
os.environ.update(LANGFUSE_PUBLIC_KEY="pk-lf-cctest", LANGFUSE_SECRET_KEY="sk-lf-cctest", LANGFUSE_BASE_URL="http://127.0.0.1:43000",
|
|
4
|
+
LANGFUSE_HOST="http://127.0.0.1:43000", LANGFUSE_TRACING_ENVIRONMENT="langchain",
|
|
5
|
+
AWS_ACCESS_KEY_ID="x", AWS_SECRET_ACCESS_KEY="x")
|
|
6
|
+
from langchain_aws import ChatBedrockConverse
|
|
7
|
+
from langfuse import get_client
|
|
8
|
+
from langfuse.langchain import CallbackHandler
|
|
9
|
+
llm = ChatBedrockConverse(model="us.anthropic.claude-sonnet-4-6", region_name="us-east-1",
|
|
10
|
+
endpoint_url=f"http://127.0.0.1:{sys.argv[1]}")
|
|
11
|
+
h = CallbackHandler()
|
|
12
|
+
for _ in range(2):
|
|
13
|
+
m = llm.invoke("hi", config={"callbacks": [h]})
|
|
14
|
+
print("langchain usage_metadata:", m.usage_metadata)
|
|
15
|
+
get_client().flush()
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Two real LiteLLM Bedrock Converse calls against fake_bedrock.py, logged by LiteLLM's `langfuse` callback
|
|
2
|
+
(or its `langfuse_otel` integration with --otel)."""
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
port, otel = sys.argv[1], "--otel" in sys.argv
|
|
8
|
+
os.environ.update(LANGFUSE_PUBLIC_KEY="pk-lf-cctest", LANGFUSE_SECRET_KEY="sk-lf-cctest", LANGFUSE_HOST="http://127.0.0.1:43000",
|
|
9
|
+
AWS_ACCESS_KEY_ID="x", AWS_SECRET_ACCESS_KEY="x", AWS_REGION_NAME="us-east-1")
|
|
10
|
+
import litellm # noqa: E402
|
|
11
|
+
|
|
12
|
+
if otel:
|
|
13
|
+
litellm.callbacks = ["langfuse_otel"]
|
|
14
|
+
else:
|
|
15
|
+
litellm.success_callback = ["langfuse"]
|
|
16
|
+
for _ in range(2):
|
|
17
|
+
r = litellm.completion(model="bedrock/converse/us.anthropic.claude-sonnet-4-6",
|
|
18
|
+
messages=[{"role": "user", "content": "hi"}], api_base=f"http://127.0.0.1:{port}",
|
|
19
|
+
metadata={"generation_name": "litellm-otel" if otel else "litellm-legacy"})
|
|
20
|
+
print("litellm usage:", dict(r.usage))
|
|
21
|
+
time.sleep(8) # both integrations send in the background
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Two real Strands Agent calls through BedrockModel (non-streaming) against the fake Bedrock,
|
|
2
|
+
traced to local Langfuse over OTLP."""
|
|
3
|
+
import base64, os, sys
|
|
4
|
+
tag = sys.argv[1]
|
|
5
|
+
auth = base64.b64encode(b"pk-lf-cctest:sk-lf-cctest").decode()
|
|
6
|
+
os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = "http://127.0.0.1:43000/api/public/otel/v1/traces"
|
|
7
|
+
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = f"Authorization=Basic {auth}"
|
|
8
|
+
os.environ["AWS_ACCESS_KEY_ID"] = "x"; os.environ["AWS_SECRET_ACCESS_KEY"] = "x"
|
|
9
|
+
os.environ["AWS_ENDPOINT_URL_BEDROCK_RUNTIME"] = f"http://127.0.0.1:{sys.argv[2]}"
|
|
10
|
+
from strands import Agent
|
|
11
|
+
from strands.models import BedrockModel
|
|
12
|
+
from strands.telemetry import StrandsTelemetry
|
|
13
|
+
tel = StrandsTelemetry().setup_otlp_exporter()
|
|
14
|
+
model = BedrockModel(model_id="us.anthropic.claude-sonnet-4-6", region_name="us-east-1", streaming=False)
|
|
15
|
+
agent = Agent(model=model, name=f"strands-{tag}", trace_attributes={"langfuse.environment": tag})
|
|
16
|
+
agent("hello"); agent("hello again")
|
|
17
|
+
tel.tracer_provider.force_flush()
|
|
18
|
+
print("strands traces sent")
|
|
@@ -19,7 +19,6 @@ To include the application inference profile check, create a profile for Sonnet
|
|
|
19
19
|
|
|
20
20
|
import argparse
|
|
21
21
|
import json
|
|
22
|
-
import os
|
|
23
22
|
import sys
|
|
24
23
|
import time
|
|
25
24
|
import uuid
|
|
@@ -128,16 +127,17 @@ def main() -> int:
|
|
|
128
127
|
checks += [looked_up[1]["read"] > 1000, named[1]["read"] > 1000]
|
|
129
128
|
|
|
130
129
|
# Agent loops: LangChain's real loop on the stand-in, then the exact requests replayed to Bedrock.
|
|
131
|
-
os.environ.setdefault("AWS_ACCESS_KEY_ID", "unused-by-the-stand-in")
|
|
132
130
|
fake = Fake()
|
|
133
131
|
|
|
134
132
|
def capture(script, *, invoke_route=False, middleware=None, manual_point=False):
|
|
135
133
|
system, history = tag_prompts()
|
|
136
134
|
fake.reset(script)
|
|
137
135
|
if invoke_route:
|
|
138
|
-
model = ChatBedrock(model=MODEL, region_name=args.region, endpoint_url=fake.url, model_kwargs={"max_tokens": 50}
|
|
136
|
+
model = ChatBedrock(model=MODEL, region_name=args.region, endpoint_url=fake.url, model_kwargs={"max_tokens": 50},
|
|
137
|
+
aws_access_key_id="unused-by-the-stand-in", aws_secret_access_key="unused-by-the-stand-in")
|
|
139
138
|
else:
|
|
140
|
-
model = ChatBedrockConverse(model=MODEL, region_name=args.region, endpoint_url=fake.url, max_tokens=50
|
|
139
|
+
model = ChatBedrockConverse(model=MODEL, region_name=args.region, endpoint_url=fake.url, max_tokens=50,
|
|
140
|
+
aws_access_key_id="unused-by-the-stand-in", aws_secret_access_key="unused-by-the-stand-in")
|
|
141
141
|
if manual_point:
|
|
142
142
|
system = SystemMessage(content=[{"type": "text", "text": system}, ChatBedrockConverse.create_cache_point()])
|
|
143
143
|
agent = create_agent(model=model, tools=[lookup_order], system_prompt=system,
|
|
@@ -172,14 +172,15 @@ def main() -> int:
|
|
|
172
172
|
return kept(rows)
|
|
173
173
|
return False
|
|
174
174
|
|
|
175
|
+
# Since Oct 2026 (by Oct 28) Bedrock counts a group of tool calls (and their results) as one lookback
|
|
176
|
+
# position, so every route keeps the whole conversation after a wide step.
|
|
175
177
|
checks.append(kept_with_retry("ChatBedrockConverse, 21 parallel calls", ["tools:21", "text"]))
|
|
176
|
-
checks.append(
|
|
178
|
+
checks.append(kept_with_retry("ChatBedrockConverse, 40 parallel calls", ["tools:40", "text"]))
|
|
177
179
|
checks.append(kept_with_retry("ChatBedrock, 10 parallel calls", ["tools:10", "text"], invoke_route=True))
|
|
178
|
-
|
|
179
|
-
checks.append(invoke_11[1]["read"] == 0) # nothing read: no separate system prompt marker on this route
|
|
180
|
+
checks.append(kept_with_retry("ChatBedrock, 40 parallel calls", ["tools:40", "text"], invoke_route=True))
|
|
180
181
|
checks.append(kept_with_retry("middleware only, 11 parallel calls", ["tools:11", "text"]))
|
|
181
|
-
checks.append(
|
|
182
|
-
|
|
182
|
+
checks.append(kept_with_retry("manual system cache point + middleware, 40 parallel calls", ["tools:40", "text"],
|
|
183
|
+
manual_point=True))
|
|
183
184
|
|
|
184
185
|
# Context editing: stock clearing vs clearing in batches vs none, 13 calls with long tool results.
|
|
185
186
|
tool_output["repeat"] = 40
|
|
@@ -25,28 +25,10 @@ import uuid
|
|
|
25
25
|
MODEL = "bedrock/converse/us.anthropic.claude-sonnet-4-6"
|
|
26
26
|
TOOLS = [{"type": "function", "function": {"name": "lookup_order", "description": "Look up an order.",
|
|
27
27
|
"parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}}]
|
|
28
|
+
# The setup on the page (tests/test_site.py checks they match).
|
|
28
29
|
SYSTEM_AND_LAST = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
# Same helper as on the page (tests/test_site.py checks they match).
|
|
32
|
-
def cache_points(messages):
|
|
33
|
-
"""Bedrock cache points for this call (Bedrock allows 4)."""
|
|
34
|
-
# The system prompt and the end of the conversation.
|
|
35
|
-
points = [{"location": "message", "role": "system"},
|
|
36
|
-
{"location": "message", "index": -1}]
|
|
37
|
-
users = [i for i, m in enumerate(messages) if m["role"] == "user"]
|
|
38
|
-
rounds = [i for i, m in enumerate(messages)
|
|
39
|
-
if m["role"] == "assistant" and m.get("tool_calls")]
|
|
40
|
-
# The newest user message.
|
|
41
|
-
if users:
|
|
42
|
-
points.append({"location": "message", "index": users[-1]})
|
|
43
|
-
# The first tool result of the newest round of tool calls.
|
|
44
|
-
if rounds and (not users or rounds[-1] > users[-1]) \
|
|
45
|
-
and rounds[-1] + 1 < len(messages):
|
|
46
|
-
points.append({"location": "message", "index": rounds[-1] + 1})
|
|
47
|
-
return points
|
|
48
|
-
|
|
49
|
-
|
|
50
32
|
def lifetime_check() -> int:
|
|
51
33
|
"""One call asking for a 1-hour cache; prints the lifetime Bedrock reports it used."""
|
|
52
34
|
model, extra = sys.argv[2], json.loads(sys.argv[3])
|
|
@@ -121,27 +103,27 @@ def main() -> int:
|
|
|
121
103
|
checks.append((label, call(f"{label}, turn 10", chat + [{"role": "user", "content": "Question 9?"}], points)))
|
|
122
104
|
checks_ok = [checks[0][1]["uncached"] > 5000, checks[1][1]["uncached"] < 20]
|
|
123
105
|
|
|
124
|
-
# Finding 2: 4 vs 12 parallel tool calls with points on the system prompt and the last message.
|
|
106
|
+
# Finding 2: 4 vs 12 parallel tool calls with points on the system prompt and the last message. Since late
|
|
107
|
+
# Oct 2026 Bedrock counts a group of tool calls (and their results) as one lookback position, so both keep it.
|
|
125
108
|
start = fresh()
|
|
126
|
-
call("agent start", start, SYSTEM_AND_LAST, TOOLS)
|
|
109
|
+
first = call("agent start", start, SYSTEM_AND_LAST, TOOLS)
|
|
127
110
|
twelve = call("12 parallel calls", start + tool_round(1, 12), SYSTEM_AND_LAST, TOOLS)
|
|
128
111
|
four = call("4 parallel calls", start + tool_round(1, 4), SYSTEM_AND_LAST, TOOLS)
|
|
129
|
-
checks_ok += [twelve["read"]
|
|
112
|
+
checks_ok += [twelve["read"] >= first["prompt"] - 5, four["read"] >= first["prompt"] - 5, four["write"] < 1000]
|
|
130
113
|
|
|
131
|
-
# The
|
|
114
|
+
# The page's setup through an agent session with wide rounds: every step reads the whole previous request.
|
|
132
115
|
steps = [fresh()]
|
|
133
116
|
for r, n in ((1, 12), (2, 1), (3, 20)):
|
|
134
117
|
steps.append(steps[-1] + tool_round(r, n))
|
|
135
118
|
steps.append(steps[-1] + [{"role": "assistant", "content": "All checked."}, {"role": "user", "content": "Refund order 3-4."}])
|
|
136
119
|
steps.append(steps[-1] + tool_round(4, 3))
|
|
120
|
+
steps.append(steps[-1] + tool_round(5, 40))
|
|
137
121
|
previous = None
|
|
138
122
|
for i, messages in enumerate(steps):
|
|
139
|
-
row = call(f"
|
|
123
|
+
row = call(f"session step {i}", messages, SYSTEM_AND_LAST, TOOLS)
|
|
140
124
|
if previous is not None:
|
|
141
125
|
checks_ok.append(row["read"] >= previous["prompt"] - 5) # read almost the whole previous request
|
|
142
126
|
previous = row
|
|
143
|
-
wide = call("helper, 21 parallel calls", steps[-1] + tool_round(5, 21), cache_points(steps[-1] + tool_round(5, 21)), TOOLS)
|
|
144
|
-
checks_ok.append(wide["read"] < previous["prompt"] - 5)
|
|
145
127
|
|
|
146
128
|
# Finding 4: the 1-hour lifetime. The price list is read at import, so each case runs in its own process.
|
|
147
129
|
def lifetime(case, model, extra=None, built_in_prices=False):
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
"""Measure where Bedrock's ~20-block cache lookback actually ends.
|
|
1
|
+
"""Measure where Bedrock's ~20-block cache lookback actually ends (the first measurement, early Oct 2026).
|
|
2
|
+
|
|
3
|
+
Historical: when this ran, Bedrock counted every block, so 22+ blocks of parallel tool calls missed.
|
|
4
|
+
By Oct 28 2026 a run of tool calls (or tool results) counts as one position and these shapes all
|
|
5
|
+
hit. scripts/live_lookback_positions.py is the current check.
|
|
2
6
|
|
|
3
7
|
Request A: tools + a short uncached system prompt + one long user turn ending in a cache point.
|
|
4
8
|
Request B: the same history with that cache point removed, then an assistant turn with N parallel
|