cachecanary 0.3.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. {cachecanary-0.3.2 → cachecanary-0.4.0}/.github/workflows/ci.yml +24 -1
  2. {cachecanary-0.3.2 → cachecanary-0.4.0}/PKG-INFO +43 -4
  3. {cachecanary-0.3.2 → cachecanary-0.4.0}/README.md +42 -3
  4. {cachecanary-0.3.2 → cachecanary-0.4.0}/action.yml +3 -3
  5. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/index.html +2 -1
  6. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/__init__.py +1 -1
  7. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/cli.py +112 -10
  8. cachecanary-0.4.0/src/cachecanary/langfuse.py +403 -0
  9. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary.egg-info/PKG-INFO +43 -4
  10. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary.egg-info/SOURCES.txt +3 -0
  11. cachecanary-0.4.0/tests/fixtures/langfuse_observations_2026-10.json +487 -0
  12. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_action.py +15 -0
  13. cachecanary-0.4.0/tests/test_langfuse.py +721 -0
  14. {cachecanary-0.3.2 → cachecanary-0.4.0}/.github/workflows/publish.yml +0 -0
  15. {cachecanary-0.3.2 → cachecanary-0.4.0}/LICENSE +0 -0
  16. {cachecanary-0.3.2 → cachecanary-0.4.0}/MANIFEST.in +0 -0
  17. {cachecanary-0.3.2 → cachecanary-0.4.0}/NOTICE +0 -0
  18. {cachecanary-0.3.2 → cachecanary-0.4.0}/examples/request_bad.json +0 -0
  19. {cachecanary-0.3.2 → cachecanary-0.4.0}/examples/request_good.json +0 -0
  20. {cachecanary-0.3.2 → cachecanary-0.4.0}/examples/request_next.json +0 -0
  21. {cachecanary-0.3.2 → cachecanary-0.4.0}/examples/request_too_short.json +0 -0
  22. {cachecanary-0.3.2 → cachecanary-0.4.0}/pyproject.toml +0 -0
  23. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/_fake_bedrock_stream.py +0 -0
  24. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/capture_langchain_requests.py +0 -0
  25. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/capture_litellm_requests.py +0 -0
  26. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/capture_strands_requests.py +0 -0
  27. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/check_site.mjs +0 -0
  28. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/live_langchain_checks.py +0 -0
  29. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/live_litellm_checks.py +0 -0
  30. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/live_logs_test.py +0 -0
  31. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/live_lookback_boundary.py +0 -0
  32. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/live_strands_checks.py +0 -0
  33. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/live_test.py +0 -0
  34. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/live_v03_checks.py +0 -0
  35. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/make_log_fixture.py +0 -0
  36. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/og-cards/og-langchain.html +0 -0
  37. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/og-cards/og-litellm.html +0 -0
  38. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/og-cards/og-strands.html +0 -0
  39. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/og-cards/og.html +0 -0
  40. {cachecanary-0.3.2 → cachecanary-0.4.0}/scripts/serve_site.py +0 -0
  41. {cachecanary-0.3.2 → cachecanary-0.4.0}/setup.cfg +0 -0
  42. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/404.html +0 -0
  43. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/_headers +0 -0
  44. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/apple-touch-icon.png +0 -0
  45. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/favicon.ico +0 -0
  46. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/favicon.svg +0 -0
  47. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/icon-192.png +0 -0
  48. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/langchain/index.html +0 -0
  49. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/litellm/index.html +0 -0
  50. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/llms.txt +0 -0
  51. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/main.js +0 -0
  52. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/og-langchain.png +0 -0
  53. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/og-litellm.png +0 -0
  54. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/og-strands.png +0 -0
  55. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/og.png +0 -0
  56. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/robots.txt +0 -0
  57. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/sitemap.xml +0 -0
  58. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/strands/index.html +0 -0
  59. {cachecanary-0.3.2 → cachecanary-0.4.0}/site/style.css +0 -0
  60. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/diff.py +0 -0
  61. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/github.py +0 -0
  62. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/lint.py +0 -0
  63. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/logs.py +0 -0
  64. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/models.py +0 -0
  65. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/pricing.py +0 -0
  66. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/probe.py +0 -0
  67. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/request.py +0 -0
  68. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary/usage.py +0 -0
  69. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary.egg-info/dependency_links.txt +0 -0
  70. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary.egg-info/entry_points.txt +0 -0
  71. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary.egg-info/requires.txt +0 -0
  72. {cachecanary-0.3.2 → cachecanary-0.4.0}/src/cachecanary.egg-info/top_level.txt +0 -0
  73. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/bedrock_invocation_logs_2026-10.json +0 -0
  74. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/langchain/date-day1.json +0 -0
  75. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/langchain/date-day2.json +0 -0
  76. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/langchain/edit-call8.json +0 -0
  77. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/langchain/edit-call9.json +0 -0
  78. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/langchain/invoke-turn1.json +0 -0
  79. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/langchain/invoke-turn2.json +0 -0
  80. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/litellm/agent-turn1.json +0 -0
  81. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/litellm/agent-turn2.json +0 -0
  82. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/litellm/chat-turn6.json +0 -0
  83. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/litellm/chat-turn7.json +0 -0
  84. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/litellm/with-download.json +0 -0
  85. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/litellm/without-download.json +0 -0
  86. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/strands/agent-turn1.json +0 -0
  87. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/strands/agent-turn2.json +0 -0
  88. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/strands/chat-turn20.json +0 -0
  89. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/strands/chat-turn21.json +0 -0
  90. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/fixtures/strands/ttl-order.json +0 -0
  91. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_cachecanary.py +0 -0
  92. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_edge_cases.py +0 -0
  93. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_github.py +0 -0
  94. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_pricing.py +0 -0
  95. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_readme.py +0 -0
  96. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_scripts_and_formats.py +0 -0
  97. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_site.py +0 -0
  98. {cachecanary-0.3.2 → cachecanary-0.4.0}/tests/test_v03_rules.py +0 -0
  99. {cachecanary-0.3.2 → cachecanary-0.4.0}/wrangler.jsonc +0 -0
@@ -51,6 +51,28 @@ jobs:
51
51
  command: diff
52
52
  args: examples/request_bad.json examples/request_next.json
53
53
 
54
+ - name: langfuse on the saved real records (must fail the 80% target)
55
+ id: langfuse_file
56
+ continue-on-error: true
57
+ uses: ./
58
+ with:
59
+ command: langfuse
60
+ args: --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
61
+
62
+ # Proves Langfuse settings given as step env reach the action: the host below is unreachable on purpose,
63
+ # so the step must fail with "could not reach Langfuse at http://127.0.0.1:9".
64
+ - name: langfuse reads keys and host from step env (must fail, unreachable host)
65
+ id: langfuse_env
66
+ continue-on-error: true
67
+ uses: ./
68
+ env:
69
+ LANGFUSE_PUBLIC_KEY: pk-lf-ci
70
+ LANGFUSE_SECRET_KEY: sk-lf-ci
71
+ LANGFUSE_HOST: http://127.0.0.1:9
72
+ with:
73
+ command: langfuse
74
+ args: --since 1h
75
+
54
76
  - name: unknown command (must fail)
55
77
  id: bad_command
56
78
  continue-on-error: true
@@ -63,7 +85,8 @@ jobs:
63
85
  shell: bash
64
86
  run: |
65
87
  set -euo pipefail
66
- for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}"; do
88
+ for o in "${{ steps.lint_short.outcome }}" "${{ steps.diff_changed.outcome }}" "${{ steps.bad_command.outcome }}" \
89
+ "${{ steps.langfuse_file.outcome }}" "${{ steps.langfuse_env.outcome }}"; do
67
90
  if [ "$o" != "failure" ]; then echo "::error::expected failure, got '$o'"; exit 1; fi
68
91
  done
69
92
  echo "All expected failures happened."
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cachecanary
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock, not when the bill arrives.
5
5
  Author-email: Haris Farooq <harisf98@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -50,7 +50,7 @@ Website: [cachecanary.com](https://cachecanary.com). It has guides that show exa
50
50
 
51
51
  What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
52
52
 
53
- CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
53
+ CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs or your Langfuse traces to show hit rates in production.
54
54
 
55
55
  ## Install
56
56
 
@@ -90,6 +90,7 @@ The last one is easy to miss. The same prompt caches fine on Sonnet 4.6 (minimum
90
90
  | `diff previous.json next.json` | Explains why the second request missed the first one's cache | the cached part changed |
91
91
  | `probe request.json --model ID [--region R] [--stream]` | Sends the request twice and checks the second one read from cache | nothing was read from cache |
92
92
  | `logs files-or-folders... [--by model\|principal] [--min-hit 0.5] [--price 3]` | Hit rate and input cost per model or IAM principal from Bedrock invocation logs, plus how much the misses cost | a group is under the threshold |
93
+ | `langfuse [--since 7d] [--environment E] [--by model\|name\|environment] [--min-hit 0.5] [--price 3]` | The same, from the generations your Langfuse project already records (no invocation logging needed) | a group is under the threshold |
93
94
 
94
95
  Exit code 2 means it couldn't run at all (bad file, no model access, no AWS credentials). `--json` and `--github` go before the command.
95
96
 
@@ -126,6 +127,16 @@ Problems show up as annotations on the pull request, plus a short table in the j
126
127
  with:
127
128
  command: probe
128
129
  args: tests/fixtures/agent_request.json --model us.anthropic.claude-sonnet-4-6 --region us-west-2
130
+
131
+ # Reads last week's hit rate from Langfuse and fails under 80%. Good as a scheduled workflow.
132
+ - uses: Haarris/cachecanary@v0
133
+ env:
134
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
135
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
136
+ LANGFUSE_HOST: https://us.cloud.langfuse.com
137
+ with:
138
+ command: langfuse
139
+ args: --since 7d --environment production --min-hit 0.8
129
140
  ```
130
141
 
131
142
  `args` takes the same arguments as the CLI, separated by spaces. Paths with spaces aren't supported.
@@ -157,20 +168,48 @@ How the dollars work:
157
168
 
158
169
  It reads S3 deliveries, CloudWatch exports and `aws logs filter-log-events` output, gzipped or not, including streamed responses. Bedrock logs InvokeModel calls under an inference profile ARN and Converse calls under the short model ID, so CacheCanary merges the two.
159
170
 
171
+ ## Hit rates from Langfuse
172
+
173
+ If you trace your agents with [Langfuse](https://langfuse.com), you don't need invocation logging. Langfuse already stores the cache counts for every model call, and `langfuse` reads them through the Langfuse API:
174
+
175
+ ```bash
176
+ export LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-...
177
+ export LANGFUSE_HOST=https://us.cloud.langfuse.com # or https://cloud.langfuse.com (EU), or your own server
178
+ cachecanary langfuse --since 7d --environment production --min-hit 0.8
179
+ ```
180
+
181
+ It asks Langfuse only for token counts, model names and timestamps, never prompts or outputs. It works with Langfuse Cloud and self-hosted v3 and v4 (on v4 it uses the faster v2 observations API).
182
+
183
+ Langfuse stores the counts the way each integration sends them, and some don't add up. Strands before 1.54 sent Bedrock's input count after the cache was already taken out, and Langfuse takes the cache out again, so the input often shows as 0 and the cache looks perfect. LiteLLM's `langfuse` callback leaves cache writes inside the input count as well as reporting them on their own. When a call has a total that was really sent, CacheCanary checks the parts against it, rebuilds the uncached input from the total (Bedrock's total includes the cached tokens), and tells you how many calls it fixed.
184
+
185
+ This is the output on real Langfuse records in this repo. They came from Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain, each making one cache write and one cache read:
186
+
187
+ ```text
188
+ $ cachecanary langfuse --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
189
+ us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0) <-- below threshold
190
+ input cost $0.13 at list price; about $0.07 of it lost to cache misses (target: 80% hit rate)
191
+ List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). Use --price for your own rate.
192
+ Read 10 generations.
193
+ 2 generation(s): Langfuse's input count did not add up with the total that was sent, so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's langfuse callback).
194
+ ```
195
+
196
+ Only model calls (generations) are counted. Agent and chain spans can carry the summed usage of the calls under them, so counting them would count tokens twice. `--file` reads a saved observations API response instead of calling Langfuse.
197
+
160
198
  ## How it's tested
161
199
 
162
- The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)).
200
+ The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model.
163
201
 
164
202
  To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region <region>`.
165
203
 
166
204
  ## Privacy
167
205
 
168
- Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials.
206
+ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials, and `langfuse` only reads token counts from your own Langfuse project with your own keys.
169
207
 
170
208
  ## Limits
171
209
 
172
210
  - Token counts in `lint` are estimates. On English text Bedrock counted about 11% more than CacheCanary did, so near a model's minimum it warns you and suggests running `probe`.
173
211
  - Bedrock keeps cache counts inside the logged response body, and only bodies up to 100 KB are stored inline. Bigger ones show up as `unparsed` and aren't in the dollar figures.
212
+ - `langfuse` can only check a call's counts when the integration sent a total. Calls with no cache fields at all are counted as uncached and listed in the output, because Langfuse can't tell "caching was off" from "the integration doesn't report cache tokens".
174
213
  - Dollar figures use list prices, which change. Batch, provisioned throughput and priority tiers aren't covered; use `--price` for those.
175
214
  - Bedrock only for now.
176
215
 
@@ -17,7 +17,7 @@ Website: [cachecanary.com](https://cachecanary.com). It has guides that show exa
17
17
 
18
18
  What that costs: the cached part of each request becomes about 10x more expensive, because cache reads are billed at a tenth of the input price. For one agent with a 10,000-token prompt and 100,000 requests a month, that's roughly $2,700 a month extra at Anthropic's list price for Claude Sonnet 4.6 ($3 per million input tokens; Bedrock prices vary by region).
19
19
 
20
- CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs to show hit rates in production.
20
+ CacheCanary is a small CLI and GitHub Action for this. It checks your requests in CI, tells you why a request missed the cache, and reads your Bedrock logs or your Langfuse traces to show hit rates in production.
21
21
 
22
22
  ## Install
23
23
 
@@ -57,6 +57,7 @@ The last one is easy to miss. The same prompt caches fine on Sonnet 4.6 (minimum
57
57
  | `diff previous.json next.json` | Explains why the second request missed the first one's cache | the cached part changed |
58
58
  | `probe request.json --model ID [--region R] [--stream]` | Sends the request twice and checks the second one read from cache | nothing was read from cache |
59
59
  | `logs files-or-folders... [--by model\|principal] [--min-hit 0.5] [--price 3]` | Hit rate and input cost per model or IAM principal from Bedrock invocation logs, plus how much the misses cost | a group is under the threshold |
60
+ | `langfuse [--since 7d] [--environment E] [--by model\|name\|environment] [--min-hit 0.5] [--price 3]` | The same, from the generations your Langfuse project already records (no invocation logging needed) | a group is under the threshold |
60
61
 
61
62
  Exit code 2 means it couldn't run at all (bad file, no model access, no AWS credentials). `--json` and `--github` go before the command.
62
63
 
@@ -93,6 +94,16 @@ Problems show up as annotations on the pull request, plus a short table in the j
93
94
  with:
94
95
  command: probe
95
96
  args: tests/fixtures/agent_request.json --model us.anthropic.claude-sonnet-4-6 --region us-west-2
97
+
98
+ # Reads last week's hit rate from Langfuse and fails under 80%. Good as a scheduled workflow.
99
+ - uses: Haarris/cachecanary@v0
100
+ env:
101
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
102
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
103
+ LANGFUSE_HOST: https://us.cloud.langfuse.com
104
+ with:
105
+ command: langfuse
106
+ args: --since 7d --environment production --min-hit 0.8
96
107
  ```
97
108
 
98
109
  `args` takes the same arguments as the CLI, separated by spaces. Paths with spaces aren't supported.
@@ -124,20 +135,48 @@ How the dollars work:
124
135
 
125
136
  It reads S3 deliveries, CloudWatch exports and `aws logs filter-log-events` output, gzipped or not, including streamed responses. Bedrock logs InvokeModel calls under an inference profile ARN and Converse calls under the short model ID, so CacheCanary merges the two.
126
137
 
138
+ ## Hit rates from Langfuse
139
+
140
+ If you trace your agents with [Langfuse](https://langfuse.com), you don't need invocation logging. Langfuse already stores the cache counts for every model call, and `langfuse` reads them through the Langfuse API:
141
+
142
+ ```bash
143
+ export LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-...
144
+ export LANGFUSE_HOST=https://us.cloud.langfuse.com # or https://cloud.langfuse.com (EU), or your own server
145
+ cachecanary langfuse --since 7d --environment production --min-hit 0.8
146
+ ```
147
+
148
+ It asks Langfuse only for token counts, model names and timestamps, never prompts or outputs. It works with Langfuse Cloud and self-hosted v3 and v4 (on v4 it uses the faster v2 observations API).
149
+
150
+ Langfuse stores the counts the way each integration sends them, and some don't add up. Strands before 1.54 sent Bedrock's input count after the cache was already taken out, and Langfuse takes the cache out again, so the input often shows as 0 and the cache looks perfect. LiteLLM's `langfuse` callback leaves cache writes inside the input count as well as reporting them on their own. When a call has a total that was really sent, CacheCanary checks the parts against it, rebuilds the uncached input from the total (Bedrock's total includes the cached tokens), and tells you how many calls it fixed.
151
+
152
+ This is the output on real Langfuse records in this repo. They came from Strands 1.61 and 1.53, LiteLLM (both Langfuse integrations) and LangChain, each making one cache write and one cache read:
153
+
154
+ ```text
155
+ $ cachecanary langfuse --file tests/fixtures/langfuse_observations_2026-10.json --min-hit 0.8
156
+ us.anthropic.claude-sonnet-4-6: hit 50% over 10 calls (read 29240, write 29240, uncached 100, no usage 0) <-- below threshold
157
+ input cost $0.13 at list price; about $0.07 of it lost to cache misses (target: 80% hit rate)
158
+ List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). Use --price for your own rate.
159
+ Read 10 generations.
160
+ 2 generation(s): Langfuse's input count did not add up with the total that was sent, so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's langfuse callback).
161
+ ```
162
+
163
+ Only model calls (generations) are counted. Agent and chain spans can carry the summed usage of the calls under them, so counting them would count tokens twice. `--file` reads a saved observations API response instead of calling Langfuse.
164
+
127
165
  ## How it's tested
128
166
 
129
- The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)).
167
+ The checks follow AWS's documented prompt-caching rules. The core behaviour was verified against live Amazon Bedrock: Converse and InvokeModel, streaming, both cache lifetimes (5 minutes and 1 hour), model minimums, prompt changes and the exact lookback limit (21 added blocks hit, 22 miss). Every check added in 0.3 was measured live first, and `scripts/live_v03_checks.py` re-runs those cases against Bedrock and compares CacheCanary's verdict with what Bedrock did (15 of 15 agreed in October 2026). Every check also has unit tests. The log reader is tested against real Bedrock invocation logs (redacted copies are in [tests/fixtures](https://github.com/Haarris/cachecanary/tree/main/tests/fixtures)). The Langfuse reader is tested against records that Strands, LiteLLM and LangChain wrote into a real Langfuse 4.64 server, and against a production Langfuse Cloud project, where its numbers matched a separate recount for every model.
130
168
 
131
169
  To run the live checks yourself, use an AWS Region without production traffic: `python scripts/live_test.py --region <region>`.
132
170
 
133
171
  ## Privacy
134
172
 
135
- Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials.
173
+ Everything runs on your machine or CI runner. Nothing is sent anywhere. `probe` only calls your own Bedrock endpoint with your own credentials, and `langfuse` only reads token counts from your own Langfuse project with your own keys.
136
174
 
137
175
  ## Limits
138
176
 
139
177
  - Token counts in `lint` are estimates. On English text Bedrock counted about 11% more than CacheCanary did, so near a model's minimum it warns you and suggests running `probe`.
140
178
  - Bedrock keeps cache counts inside the logged response body, and only bodies up to 100 KB are stored inline. Bigger ones show up as `unparsed` and aren't in the dollar figures.
179
+ - `langfuse` can only check a call's counts when the integration sent a total. Calls with no cache fields at all are counted as uncached and listed in the output, because Langfuse can't tell "caching was off" from "the integration doesn't report cache tokens".
141
180
  - Dollar figures use list prices, which change. Batch, provisioned throughput and priority tiers aren't covered; use `--price` for those.
142
181
  - Bedrock only for now.
143
182
 
@@ -7,7 +7,7 @@ branding:
7
7
 
8
8
  inputs:
9
9
  command:
10
- description: "One of: lint, diff, probe, logs"
10
+ description: "One of: lint, diff, probe, logs, langfuse"
11
11
  required: true
12
12
  args:
13
13
  description: >
@@ -38,8 +38,8 @@ runs:
38
38
  CC_ARGS: ${{ inputs.args }}
39
39
  run: |
40
40
  case "$CC_COMMAND" in
41
- lint|diff|probe|logs) ;;
42
- *) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe or logs)"; exit 2 ;;
41
+ lint|diff|probe|logs|langfuse) ;;
42
+ *) echo "::error title=CacheCanary::unknown command '$CC_COMMAND' (use lint, diff, probe, logs or langfuse)"; exit 2 ;;
43
43
  esac
44
44
  read -r -a CC_ARGV <<< "$CC_ARGS"
45
45
  cachecanary --github "$CC_COMMAND" "${CC_ARGV[@]}"
@@ -31,7 +31,7 @@
31
31
  "url": "https://cachecanary.com/",
32
32
  "applicationCategory": "DeveloperApplication",
33
33
  "operatingSystem": "macOS, Linux, Windows",
34
- "softwareVersion": "0.3.2",
34
+ "softwareVersion": "0.4.0",
35
35
  "downloadUrl": "https://pypi.org/project/cachecanary/",
36
36
  "license": "https://www.apache.org/licenses/LICENSE-2.0",
37
37
  "offers": {
@@ -101,6 +101,7 @@
101
101
  </div>
102
102
  <p class="note under-actions">Free and open source (Apache 2.0). The tool runs on your machine and sends nothing anywhere.</p>
103
103
  <p class="note">Framework guides: <a href="/litellm/">LiteLLM</a> · <a href="/strands/">Strands Agents</a> · <a href="/langchain/">LangChain</a></p>
104
+ <p class="note">Also available as a <a href="https://gitbot.codeongrass.com/bots/cachecanary/">GitBot bot</a> that checks your changes before you merge.</p>
104
105
 
105
106
  <h2>What a broken cache costs</h2>
106
107
  <div class="facts">
@@ -1,3 +1,3 @@
1
1
  """CacheCanary: know the moment your Claude prompt cache breaks on Amazon Bedrock."""
2
2
 
3
- __version__ = "0.3.2"
3
+ __version__ = "0.4.0"
@@ -4,6 +4,7 @@
4
4
  cachecanary diff previous.json next.json [--model ID]
5
5
  cachecanary probe request.json --model ID [--region R] [--stream]
6
6
  cachecanary logs FILE_OR_FOLDER ... [--by model|principal|model+principal] [--min-hit 0.5] [--price 3]
7
+ cachecanary langfuse [--since 7d | --from DATE --to DATE] [--environment E] [--by model|name|environment]
7
8
 
8
9
  Exit codes: 0 ok, 1 caching problem found (use to gate CI), 2 could not run (bad input, AWS error).
9
10
  Add --github (before the command) for GitHub Actions annotations and a job summary.
@@ -11,11 +12,13 @@ Add --github (before the command) for GitHub Actions annotations and a job summa
11
12
 
12
13
  import argparse
13
14
  import json
15
+ import os
14
16
  import sys
15
17
  from dataclasses import asdict
18
+ from datetime import datetime, timezone
16
19
  from pathlib import Path
17
20
 
18
- from cachecanary import __version__, diff, lint, logs, pricing, probe
21
+ from cachecanary import __version__, diff, langfuse, lint, logs, pricing, probe
19
22
  from cachecanary import github as gh
20
23
  from cachecanary.request import RequestError, normalize
21
24
 
@@ -163,6 +166,12 @@ def cmd_logs(args) -> int:
163
166
  except (OSError, UnicodeDecodeError, EOFError, json.JSONDecodeError) as exc:
164
167
  raise InputError(f"could not read {path}: {exc}") from exc
165
168
  groups = logs.aggregate(records, by=args.by)
169
+ notes = [f"Skipped {stats.bad_lines} unreadable line(s)."] if stats.bad_lines else []
170
+ return _report(groups, args, "logs", "No invocation log records found.", notes)
171
+
172
+
173
+ def _report(groups, args, source: str, empty: str, notes: list[str]) -> int:
174
+ """Hit rate and dollars per group, shared by `logs` and `langfuse`."""
166
175
  target = args.min_hit if args.min_hit is not None else DEFAULT_TARGET_HIT
167
176
  data, lines, failing = {}, [], False
168
177
  total_cost = total_lost = 0.0
@@ -177,7 +186,7 @@ def cmd_logs(args) -> int:
177
186
  flag, failing = " <-- below threshold", True
178
187
  lines.append(f"{key}: hit {ratio:.0%} over {g.calls} calls "
179
188
  f"(read {g.usage.cache_read}, write {g.usage.cache_write}, uncached {g.usage.uncached_input}, "
180
- f"unparsed {g.unparsed}){flag}")
189
+ f"{'no usage' if source == 'langfuse' else 'unparsed'} {g.unparsed}){flag}")
181
190
  if g.usage.total_input == 0:
182
191
  continue
183
192
  if cost is None:
@@ -202,9 +211,11 @@ def cmd_logs(args) -> int:
202
211
  if any_priced and not args.price:
203
212
  lines.append("List prices: Amazon Bedrock on-demand, Oct 2026 (global. IDs at list, others +10%). "
204
213
  "Use --price for your own rate.")
205
- if stats.bad_lines:
206
- lines.append(f"Skipped {stats.bad_lines} unreadable line(s).")
207
- _emit(data, args.json, lines or ["No invocation log records found."])
214
+ lines.extend(notes)
215
+ if args.json:
216
+ print(json.dumps({"groups": data, "notes": notes} if source == "langfuse" else data, indent=2))
217
+ else:
218
+ print("\n".join(lines if data else [empty, *notes]))
208
219
  if args.github:
209
220
  rows = []
210
221
  for key, d in data.items():
@@ -216,13 +227,59 @@ def cmd_logs(args) -> int:
216
227
  rows.append([key, d["calls"], f"{d['hit_ratio']:.0%}", d["cache_read"], d["cache_write"], d["uncached_input"],
217
228
  d["unparsed"], _money(d["input_cost_usd"]) if priced else "",
218
229
  _money(d["lost_usd"]) if priced else "", "❌" if below else ""])
219
- gh.append_summary("### CacheCanary logs\n\n" + (gh.table(
220
- ["group", "calls", "hit rate", "read", "write", "uncached", "unparsed", "input cost",
221
- f"lost vs {target:.0%} hit", "below threshold"], rows)
222
- if rows else "No invocation log records found."))
230
+ summary = gh.table(["group", "calls", "hit rate", "read", "write", "uncached",
231
+ "no usage" if source == "langfuse" else "unparsed", "input cost",
232
+ f"lost vs {target:.0%} hit", "below threshold"], rows) if rows else empty
233
+ gh.append_summary(f"### CacheCanary {source}\n\n" + summary + "".join(f"\n\n{n}" for n in notes))
223
234
  return EXIT_PROBLEM if failing else EXIT_OK
224
235
 
225
236
 
237
+ def cmd_langfuse(args) -> int:
238
+ if args.file:
239
+ observations, api, capped = [], "file", False
240
+ for path in args.file:
241
+ observations.extend(langfuse.iter_file(Path(path)))
242
+ window = ""
243
+ else:
244
+ now = datetime.now(timezone.utc)
245
+ to_time = args.to or now
246
+ from_time = args.from_ or to_time - args.since
247
+ if from_time >= to_time:
248
+ raise InputError("--from must be earlier than --to")
249
+ client = langfuse.Client(args.host or os.environ.get("LANGFUSE_HOST") or os.environ.get("LANGFUSE_BASE_URL"),
250
+ os.environ.get("LANGFUSE_PUBLIC_KEY", ""), os.environ.get("LANGFUSE_SECRET_KEY", ""),
251
+ progress=langfuse.stderr_progress)
252
+ try:
253
+ fetched = client.fetch_generations(langfuse.iso(from_time), langfuse.iso(to_time) if args.to else None,
254
+ args.environment, args.max_calls)
255
+ finally:
256
+ if sys.stderr.isatty():
257
+ print(file=sys.stderr)
258
+ observations, api, capped = fetched.observations, fetched.api, fetched.capped
259
+ window = f" from {langfuse.iso(from_time)} to {langfuse.iso(to_time)}"
260
+ groups, q = langfuse.aggregate(observations, by=args.by)
261
+ notes = []
262
+ if q.generations:
263
+ notes.append(f"Read {q.generations:,} generations{window}" + (f" (Langfuse API {api})" if api != "file" else "") + ".")
264
+ if capped:
265
+ notes.append(f"Stopped at --max-calls {args.max_calls:,}; older generations in the window were not read.")
266
+ if q.corrected:
267
+ notes.append(f"{q.corrected:,} generation(s): Langfuse's input count did not add up with the total that was sent, "
268
+ "so the uncached input was rebuilt from the total (seen with Strands before 1.54 and LiteLLM's "
269
+ "langfuse callback).")
270
+ if q.total_mismatch:
271
+ notes.append(f"{q.total_mismatch:,} generation(s) carry a total smaller than their parts; their counts were used as sent.")
272
+ if q.no_cache_fields:
273
+ notes.append(f"{q.no_cache_fields:,} generation(s) have no cache fields at all: caching was not used for them, "
274
+ "the integration does not report cache tokens, or Langfuse estimated the tokens itself.")
275
+ if q.no_usage:
276
+ notes.append(f"{q.no_usage:,} generation(s) have no token counts.")
277
+ if q.skipped_other_types:
278
+ notes.append(f"Ignored {q.skipped_other_types:,} non-generation observation(s) (agent and chain spans can repeat "
279
+ "the usage of the calls under them).")
280
+ return _report(groups, args, "langfuse", "No generations found in Langfuse for this window.", notes)
281
+
282
+
226
283
  def _unit(value: str) -> float:
227
284
  f = float(value)
228
285
  if not 0 <= f <= 1:
@@ -240,6 +297,30 @@ def _price(value: str) -> float:
240
297
  return f
241
298
 
242
299
 
300
+ def _since(value: str):
301
+ try:
302
+ return langfuse.parse_since(value)
303
+ except ValueError as exc:
304
+ raise argparse.ArgumentTypeError(str(exc)) from None
305
+
306
+
307
+ def _when(value: str):
308
+ try:
309
+ return langfuse.parse_time(value)
310
+ except ValueError as exc:
311
+ raise argparse.ArgumentTypeError(str(exc)) from None
312
+
313
+
314
+ def _positive(value: str) -> int:
315
+ try:
316
+ n = int(value)
317
+ except ValueError:
318
+ raise argparse.ArgumentTypeError("must be a whole number") from None
319
+ if n < 1:
320
+ raise argparse.ArgumentTypeError("must be at least 1")
321
+ return n
322
+
323
+
243
324
  def build_parser() -> argparse.ArgumentParser:
244
325
  parser = argparse.ArgumentParser(prog="cachecanary", description="CacheCanary: catch silent prompt-cache breakage for Claude on Amazon Bedrock.")
245
326
  parser.add_argument("--version", action="version", version=f"cachecanary {__version__}")
@@ -273,6 +354,27 @@ def build_parser() -> argparse.ArgumentParser:
273
354
  "also the target for the dollars lost figure (default 0.9)")
274
355
  p.add_argument("--price", type=_price, help="your input price in USD per million tokens, instead of list prices")
275
356
  p.set_defaults(func=cmd_logs)
357
+
358
+ p = sub.add_parser("langfuse", help="cache hit rates from the generations in a Langfuse project",
359
+ description="Reads token counts (no prompts or outputs) for the project's generations through "
360
+ "the Langfuse public API. Keys come from LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY; "
361
+ "the host from --host, LANGFUSE_HOST or LANGFUSE_BASE_URL (default "
362
+ f"{langfuse.DEFAULT_HOST}).")
363
+ p.add_argument("--host", help="Langfuse URL, e.g. https://us.cloud.langfuse.com or your self-hosted URL")
364
+ window = p.add_mutually_exclusive_group()
365
+ window.add_argument("--since", type=_since, default=langfuse.parse_since("7d"),
366
+ help="how far back to read, e.g. 24h, 7d, 2w (default 7d)")
367
+ window.add_argument("--from", dest="from_", type=_when, help="start of the window (date or ISO time, UTC)")
368
+ p.add_argument("--to", type=_when, help="end of the window (default now)")
369
+ p.add_argument("--environment", action="append", help="only this Langfuse environment (repeatable)")
370
+ p.add_argument("--by", choices=["model", "name", "environment"], default="model")
371
+ p.add_argument("--min-hit", type=_unit, help="fail if any group's hit ratio is below this (0-1); "
372
+ "also the target for the dollars lost figure (default 0.9)")
373
+ p.add_argument("--price", type=_price, help="your input price in USD per million tokens, instead of list prices")
374
+ p.add_argument("--max-calls", type=_positive, default=20000,
375
+ help="stop after this many generations (default 20000)")
376
+ p.add_argument("--file", action="append", help="read a saved observations API response instead of calling Langfuse")
377
+ p.set_defaults(func=cmd_langfuse)
276
378
  return parser
277
379
 
278
380
 
@@ -280,7 +382,7 @@ def main(argv: list[str] | None = None) -> int:
280
382
  args = build_parser().parse_args(argv)
281
383
  try:
282
384
  return args.func(args)
283
- except (InputError, RequestError, probe.ProbeError) as exc:
385
+ except (InputError, RequestError, probe.ProbeError, langfuse.LangfuseError) as exc:
284
386
  print(f"cachecanary: error: {exc}", file=sys.stderr)
285
387
  if getattr(args, "github", False):
286
388
  print(gh.annotation("error", str(exc), title="CacheCanary could not run"))