cacheverifier 0.7.0__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/.github/workflows/ci.yml +3 -1
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/CHANGELOG.md +10 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/PKG-INFO +95 -34
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/README.md +94 -33
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/__init__.py +1 -1
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/client.py +1 -1
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/examples/gptcache_example.py +3 -3
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/examples/quickstart.py +5 -5
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/pyproject.toml +1 -1
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/.github/workflows/publish.yml +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/.gitignore +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/LICENSE +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/__main__.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/_healthcheck/__init__.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/_healthcheck/_base_models.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/_healthcheck/_finetune.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/_healthcheck/cli.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/_inference.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/integrations/__init__.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/integrations/gptcache.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/integrations/langchain.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/local.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/cacheverifier/py.typed +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/tests/test_client.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/tests/test_gptcache_integration.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/tests/test_healthcheck.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/tests/test_langchain_integration.py +0 -0
- {cacheverifier-0.7.0 → cacheverifier-0.7.1}/tests/test_local.py +0 -0
|
@@ -7,7 +7,9 @@ on:
|
|
|
7
7
|
|
|
8
8
|
jobs:
|
|
9
9
|
test:
|
|
10
|
-
|
|
10
|
+
# ubuntu-latest is 26.04, which setup-python has no 3.9 build for; 3.9
|
|
11
|
+
# stays on 24.04 until it is dropped from requires-python.
|
|
12
|
+
runs-on: ${{ matrix.python-version == '3.9' && 'ubuntu-24.04' || 'ubuntu-latest' }}
|
|
11
13
|
strategy:
|
|
12
14
|
fail-fast: false
|
|
13
15
|
matrix:
|
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.1
|
|
4
|
+
|
|
5
|
+
README only, no code change, so the PyPI page matches the docs: the
|
|
6
|
+
`VerifyResult` table lists `audit`, `verification_id` and `decision_rule`;
|
|
7
|
+
the quickstart reports audited calls; certified thresholds are described as
|
|
8
|
+
informational, valid while traffic resembles the calibration data, and
|
|
9
|
+
covering only verified hits; the offline Health Check notes that the CLI
|
|
10
|
+
defaults to `ms_marco` while hosted fine-tunes default to `nli`; known limits
|
|
11
|
+
cover the stock model's MS MARCO training-data terms.
|
|
12
|
+
|
|
3
13
|
## 0.7.0
|
|
4
14
|
|
|
5
15
|
`entries_to_refresh()` lists the cache entries your feedback reported stale,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cacheverifier
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: Python client for the CacheVerifier semantic-cache verification API, hosted or scored locally
|
|
5
5
|
Project-URL: Homepage, https://www.cacheverifier.com
|
|
6
6
|
Project-URL: Documentation, https://www.cacheverifier.com/docs
|
|
@@ -43,18 +43,44 @@ Description-Content-Type: text/markdown
|
|
|
43
43
|
[](https://pypi.org/project/cacheverifier/)
|
|
44
44
|
[](https://pypi.org/project/cacheverifier/)
|
|
45
45
|
|
|
46
|
-
Python client for **[CacheVerifier](https://www.cacheverifier.com)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
Python client for **[CacheVerifier](https://www.cacheverifier.com)**: verification and
|
|
47
|
+
operations for semantic caches. It checks the cache hits a similarity threshold can't call,
|
|
48
|
+
then keeps your cache right after launch.
|
|
49
|
+
|
|
50
|
+
- **Verify the uncertain hits.** Given a query and a candidate cached answer, approve or
|
|
51
|
+
reject serving it in tens of milliseconds, so a match that is *close but wrong* doesn't
|
|
52
|
+
become a silent error. Fine-tuned on your own feedback.
|
|
53
|
+
- **Know your real error rate.** Randomized audits of approved hits give a conservative
|
|
54
|
+
(Clopper-Pearson) upper bound on how often a served answer is wrong, measured on your
|
|
55
|
+
traffic, not a benchmark.
|
|
56
|
+
- **Certified thresholds.** At fine-tune time, Conformal Risk Control can report a threshold
|
|
57
|
+
that keeps false reuses under a target you set. It is informational (`verify()` doesn't
|
|
58
|
+
apply it), holds only while traffic resembles the calibration data, and covers only the
|
|
59
|
+
hits you send to `verify()`, not the ones your cache serves on similarity alone. An
|
|
60
|
+
optional online (ACI) threshold does change decisions, following what the audits find.
|
|
61
|
+
- **Catch drift and recalibrate.** When traffic moves away from what the verifier learned,
|
|
62
|
+
the dashboard flags it; on Team and Scale you also get a webhook alert and it is
|
|
63
|
+
retrained on your feedback automatically.
|
|
64
|
+
- **Know which entries to fix.** `entries_to_refresh()` lists the cache entries your
|
|
65
|
+
feedback reported stale, or wrong more often than right, with your own cache keys.
|
|
66
|
+
- **Run it in your stack.** On Managed plans, `LocalVerifier` scores with your fine-tuned
|
|
67
|
+
model locally: no network hop, your texts stay with you, nothing metered.
|
|
50
68
|
|
|
51
69
|
CacheVerifier does **not** run your cache or do similarity search. Your cache backend does
|
|
52
70
|
its own lookup first; you call `verify()` only on the candidates in the similarity "gray
|
|
53
71
|
zone", where a plain threshold match might be wrong.
|
|
54
72
|
|
|
73
|
+
| | answers | examples |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| Static-threshold caches | "how similar is it?" | GPTCache, LiteLLM, Portkey, Redis LangCache |
|
|
76
|
+
| Adaptive-threshold caches | "how similar, with a learned per-entry cutoff?" (replaces your cache) | vCache |
|
|
77
|
+
| **Verification layer** | "is this cached answer right for this question?" (sits on top of your cache) | **CacheVerifier** |
|
|
78
|
+
|
|
55
79
|
- Docs / API reference: <https://www.cacheverifier.com/docs>
|
|
56
80
|
- Why similarity ≠ correctness: <https://www.cacheverifier.com/why-similarity-fails>
|
|
57
|
-
- The research behind it (
|
|
81
|
+
- The research behind it (four papers, code and raw results): <https://github.com/imxinchengyou/CacheVerifier>
|
|
82
|
+
- TypeScript / Node: no npm package yet; the docs have a dependency-free `fetch` client and a
|
|
83
|
+
LangChain.js cache wrapper to copy: <https://www.cacheverifier.com/docs#typescript>
|
|
58
84
|
|
|
59
85
|
## Install
|
|
60
86
|
|
|
@@ -73,25 +99,31 @@ are opt-in.
|
|
|
73
99
|
|
|
74
100
|
## Quickstart
|
|
75
101
|
|
|
76
|
-
Get a free API key at <https://www.cacheverifier.com> (
|
|
77
|
-
are free
|
|
102
|
+
Get a free API key at <https://www.cacheverifier.com> (verify, fine-tuning, audits and
|
|
103
|
+
entries to refresh are free, no card).
|
|
78
104
|
|
|
79
105
|
```python
|
|
80
106
|
from cacheverifier import CacheVerifier
|
|
81
107
|
|
|
82
108
|
cv = CacheVerifier(api_key="cv_...")
|
|
83
109
|
|
|
84
|
-
query = "
|
|
85
|
-
candidate = "
|
|
110
|
+
query = "can I change the delivery address after my parcel ships"
|
|
111
|
+
candidate = "You can change the delivery address any time before the parcel ships." # from your cache
|
|
86
112
|
|
|
87
113
|
result = cv.verify(query, candidate)
|
|
88
114
|
if result.approved:
|
|
89
115
|
answer = candidate # verified hit — skip the LLM call
|
|
90
116
|
else:
|
|
91
117
|
answer = call_your_llm(query) # not trustworthy — fall through
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
118
|
+
if result.audit: # a randomized audit of an approval: report whether
|
|
119
|
+
cv.feedback(query, candidate, # the cached answer matched the fresh one
|
|
120
|
+
was_correct=same_answer(answer, candidate),
|
|
121
|
+
verification_id=result.verification_id)
|
|
122
|
+
|
|
123
|
+
# Later, label the cached candidate (not the fresh LLM answer): was it right for this
|
|
124
|
+
# query? (a thumbs-down, a reopened ticket, a reviewer's call)
|
|
125
|
+
cv.feedback(query, candidate, was_correct=candidate_was_right, similarity_score=0.86,
|
|
126
|
+
verification_id=result.verification_id)
|
|
95
127
|
```
|
|
96
128
|
|
|
97
129
|
`verify()` returns a `VerifyResult`:
|
|
@@ -99,7 +131,10 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86)
|
|
|
99
131
|
| field | meaning |
|
|
100
132
|
|---|---|
|
|
101
133
|
| `approved` | serve the cached answer (`True`) or fall through (`False`) |
|
|
102
|
-
| `
|
|
134
|
+
| `audit` | `True` when an approval was drawn for a randomized audit, so `approved` is `False`: call your LLM, serve its answer, and report whether the cached one matched |
|
|
135
|
+
| `verification_id` | set while auditing is on; pass it to `feedback()` |
|
|
136
|
+
| `score` / `threshold` | with `decision_rule="threshold"`, `approved` is `score >= threshold` |
|
|
137
|
+
| `decision_rule` | `"threshold"`, `"joint"` / `"joint_gap"` (a fitted rule that also uses your `similarity_score`), `"aci"` (online threshold), or `"cold_start"` |
|
|
103
138
|
| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` |
|
|
104
139
|
| `latency_ms` | server-side inference time |
|
|
105
140
|
| `degraded` | `True` when the call failed and this result was synthesized client-side |
|
|
@@ -144,23 +179,6 @@ cv = CacheVerifier(api_key="cv_...", fail_open=True)
|
|
|
144
179
|
There is no formal uptime SLA yet, which is the other reason the fallback path
|
|
145
180
|
is a built-in default rather than left to you.
|
|
146
181
|
|
|
147
|
-
## GPTCache
|
|
148
|
-
|
|
149
|
-
Drop the verifier into a GPTCache pipeline as its similarity evaluator — no fork required:
|
|
150
|
-
|
|
151
|
-
```python
|
|
152
|
-
from gptcache import cache
|
|
153
|
-
from cacheverifier.integrations.gptcache import CacheVerifierEvaluation
|
|
154
|
-
|
|
155
|
-
evaluator = CacheVerifierEvaluation(api_key="cv_...")
|
|
156
|
-
cache.init(similarity_evaluation=evaluator, ...)
|
|
157
|
-
|
|
158
|
-
# when you learn a served hit's real outcome:
|
|
159
|
-
evaluator.report_feedback(query, answer, was_correct=False, similarity_score=0.9)
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
See [`examples/gptcache_example.py`](examples/gptcache_example.py).
|
|
163
|
-
|
|
164
182
|
## LangChain
|
|
165
183
|
|
|
166
184
|
`VerifiedCache` wraps any LangChain cache, typically a semantic one, and only
|
|
@@ -185,6 +203,23 @@ set_llm_cache(
|
|
|
185
203
|
)
|
|
186
204
|
```
|
|
187
205
|
|
|
206
|
+
## GPTCache
|
|
207
|
+
|
|
208
|
+
Drop the verifier into a GPTCache pipeline as its similarity evaluator — no fork required:
|
|
209
|
+
|
|
210
|
+
```python
|
|
211
|
+
from gptcache import cache
|
|
212
|
+
from cacheverifier.integrations.gptcache import CacheVerifierEvaluation
|
|
213
|
+
|
|
214
|
+
evaluator = CacheVerifierEvaluation(api_key="cv_...")
|
|
215
|
+
cache.init(similarity_evaluation=evaluator, ...)
|
|
216
|
+
|
|
217
|
+
# when you learn a served hit's real outcome:
|
|
218
|
+
evaluator.report_feedback(query, answer, was_correct=False, similarity_score=0.9)
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
See [`examples/gptcache_example.py`](examples/gptcache_example.py).
|
|
222
|
+
|
|
188
223
|
## Entries to refresh
|
|
189
224
|
|
|
190
225
|
Send your cache's own key with feedback, and the service tells you which
|
|
@@ -273,12 +308,14 @@ fully offline. `--emit-summary` writes an aggregate-only JSON file (AUCs, counts
|
|
|
273
308
|
no query or answer text) that's safe to share for a human read.
|
|
274
309
|
|
|
275
310
|
**Choosing the base model.** `--base-model` takes the same values as the hosted
|
|
276
|
-
service's `base_model` fine-tune parameter
|
|
311
|
+
service's `base_model` fine-tune parameter. The defaults differ: the CLI starts from
|
|
312
|
+
`ms_marco`, hosted fine-tunes from `nli`. To compare a local run with a hosted fine-tune,
|
|
313
|
+
pass `--base-model nli`.
|
|
277
314
|
|
|
278
315
|
| `--base-model` | model | use for |
|
|
279
316
|
|---|---|---|
|
|
280
|
-
| `ms_marco` (default) | `cross-encoder/ms-marco-MiniLM-L6-v2` | English traffic |
|
|
281
|
-
| `nli` | `cross-encoder/nli-MiniLM2-L6-H768` | English; entailment pretraining, stronger on negation / entity swaps |
|
|
317
|
+
| `ms_marco` (CLI default) | `cross-encoder/ms-marco-MiniLM-L6-v2` | English traffic |
|
|
318
|
+
| `nli` (hosted default) | `cross-encoder/nli-MiniLM2-L6-H768` | English; entailment pretraining, stronger on negation / entity swaps |
|
|
282
319
|
| `multilingual` | `BAAI/bge-reranker-base` | Chinese and other non-English traffic (~6 GB RAM to fine-tune) |
|
|
283
320
|
| `multilingual_small` | `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` | same, smaller (~3 GB RAM) |
|
|
284
321
|
|
|
@@ -308,6 +345,30 @@ verdict
|
|
|
308
345
|
IMPROVED -- fine-tuning on your own data helps this traffic
|
|
309
346
|
```
|
|
310
347
|
|
|
348
|
+
## Recommended production setup
|
|
349
|
+
|
|
350
|
+
- Keep randomized audits on (the default for new accounts) and report each audited call
|
|
351
|
+
with `feedback(..., verification_id=r.verification_id)`, or `result=r` on
|
|
352
|
+
`LocalVerifier`. That is what makes the measured error rate real.
|
|
353
|
+
- Fine-tune with `cost_ratio=` (how many times worse a wrong reuse is than an extra LLM
|
|
354
|
+
call) instead of the default threshold. ACI (hosted only) is optional: it lets the
|
|
355
|
+
threshold follow the audits, but on the one real drifting stream we tested it gave no
|
|
356
|
+
clear advantage over a fixed threshold.
|
|
357
|
+
- Send `cache_entry_id=` with feedback so `entries_to_refresh()` hands back your keys.
|
|
358
|
+
- At volume, score locally with `LocalVerifier`.
|
|
359
|
+
|
|
360
|
+
## Known limits
|
|
361
|
+
|
|
362
|
+
- The hosted verifier is **English-only** today. Don't send Chinese or other non-English
|
|
363
|
+
traffic to it; multilingual base models are evaluated in the docs but not deployed.
|
|
364
|
+
- Query + answer share a **128-token** input; longer pairs are truncated head and tail.
|
|
365
|
+
- Verify endpoints allow about **20 requests/second per API key**. Use `verify_batch()` or
|
|
366
|
+
`LocalVerifier` beyond that.
|
|
367
|
+
- No formal uptime SLA yet; `verify()` fails closed by default (see above).
|
|
368
|
+
- The shared stock model (`cross-encoder/ms-marco-MiniLM-L6-v2`) is Apache-2.0 but trained
|
|
369
|
+
on MS MARCO, whose terms describe it as for non-commercial research. Fine-tunes start from
|
|
370
|
+
the NLI base (`cross-encoder/nli-MiniLM2-L6-H768`, Apache-2.0) by default.
|
|
371
|
+
|
|
311
372
|
## API surface
|
|
312
373
|
|
|
313
374
|
| method | endpoint |
|
|
@@ -4,18 +4,44 @@
|
|
|
4
4
|
[](https://pypi.org/project/cacheverifier/)
|
|
5
5
|
[](https://pypi.org/project/cacheverifier/)
|
|
6
6
|
|
|
7
|
-
Python client for **[CacheVerifier](https://www.cacheverifier.com)
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
Python client for **[CacheVerifier](https://www.cacheverifier.com)**: verification and
|
|
8
|
+
operations for semantic caches. It checks the cache hits a similarity threshold can't call,
|
|
9
|
+
then keeps your cache right after launch.
|
|
10
|
+
|
|
11
|
+
- **Verify the uncertain hits.** Given a query and a candidate cached answer, approve or
|
|
12
|
+
reject serving it in tens of milliseconds, so a match that is *close but wrong* doesn't
|
|
13
|
+
become a silent error. Fine-tuned on your own feedback.
|
|
14
|
+
- **Know your real error rate.** Randomized audits of approved hits give a conservative
|
|
15
|
+
(Clopper-Pearson) upper bound on how often a served answer is wrong, measured on your
|
|
16
|
+
traffic, not a benchmark.
|
|
17
|
+
- **Certified thresholds.** At fine-tune time, Conformal Risk Control can report a threshold
|
|
18
|
+
that keeps false reuses under a target you set. It is informational (`verify()` doesn't
|
|
19
|
+
apply it), holds only while traffic resembles the calibration data, and covers only the
|
|
20
|
+
hits you send to `verify()`, not the ones your cache serves on similarity alone. An
|
|
21
|
+
optional online (ACI) threshold does change decisions, following what the audits find.
|
|
22
|
+
- **Catch drift and recalibrate.** When traffic moves away from what the verifier learned,
|
|
23
|
+
the dashboard flags it; on Team and Scale you also get a webhook alert and it is
|
|
24
|
+
retrained on your feedback automatically.
|
|
25
|
+
- **Know which entries to fix.** `entries_to_refresh()` lists the cache entries your
|
|
26
|
+
feedback reported stale, or wrong more often than right, with your own cache keys.
|
|
27
|
+
- **Run it in your stack.** On Managed plans, `LocalVerifier` scores with your fine-tuned
|
|
28
|
+
model locally: no network hop, your texts stay with you, nothing metered.
|
|
11
29
|
|
|
12
30
|
CacheVerifier does **not** run your cache or do similarity search. Your cache backend does
|
|
13
31
|
its own lookup first; you call `verify()` only on the candidates in the similarity "gray
|
|
14
32
|
zone", where a plain threshold match might be wrong.
|
|
15
33
|
|
|
34
|
+
| | answers | examples |
|
|
35
|
+
|---|---|---|
|
|
36
|
+
| Static-threshold caches | "how similar is it?" | GPTCache, LiteLLM, Portkey, Redis LangCache |
|
|
37
|
+
| Adaptive-threshold caches | "how similar, with a learned per-entry cutoff?" (replaces your cache) | vCache |
|
|
38
|
+
| **Verification layer** | "is this cached answer right for this question?" (sits on top of your cache) | **CacheVerifier** |
|
|
39
|
+
|
|
16
40
|
- Docs / API reference: <https://www.cacheverifier.com/docs>
|
|
17
41
|
- Why similarity ≠ correctness: <https://www.cacheverifier.com/why-similarity-fails>
|
|
18
|
-
- The research behind it (
|
|
42
|
+
- The research behind it (four papers, code and raw results): <https://github.com/imxinchengyou/CacheVerifier>
|
|
43
|
+
- TypeScript / Node: no npm package yet; the docs have a dependency-free `fetch` client and a
|
|
44
|
+
LangChain.js cache wrapper to copy: <https://www.cacheverifier.com/docs#typescript>
|
|
19
45
|
|
|
20
46
|
## Install
|
|
21
47
|
|
|
@@ -34,25 +60,31 @@ are opt-in.
|
|
|
34
60
|
|
|
35
61
|
## Quickstart
|
|
36
62
|
|
|
37
|
-
Get a free API key at <https://www.cacheverifier.com> (
|
|
38
|
-
are free
|
|
63
|
+
Get a free API key at <https://www.cacheverifier.com> (verify, fine-tuning, audits and
|
|
64
|
+
entries to refresh are free, no card).
|
|
39
65
|
|
|
40
66
|
```python
|
|
41
67
|
from cacheverifier import CacheVerifier
|
|
42
68
|
|
|
43
69
|
cv = CacheVerifier(api_key="cv_...")
|
|
44
70
|
|
|
45
|
-
query = "
|
|
46
|
-
candidate = "
|
|
71
|
+
query = "can I change the delivery address after my parcel ships"
|
|
72
|
+
candidate = "You can change the delivery address any time before the parcel ships." # from your cache
|
|
47
73
|
|
|
48
74
|
result = cv.verify(query, candidate)
|
|
49
75
|
if result.approved:
|
|
50
76
|
answer = candidate # verified hit — skip the LLM call
|
|
51
77
|
else:
|
|
52
78
|
answer = call_your_llm(query) # not trustworthy — fall through
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
79
|
+
if result.audit: # a randomized audit of an approval: report whether
|
|
80
|
+
cv.feedback(query, candidate, # the cached answer matched the fresh one
|
|
81
|
+
was_correct=same_answer(answer, candidate),
|
|
82
|
+
verification_id=result.verification_id)
|
|
83
|
+
|
|
84
|
+
# Later, label the cached candidate (not the fresh LLM answer): was it right for this
|
|
85
|
+
# query? (a thumbs-down, a reopened ticket, a reviewer's call)
|
|
86
|
+
cv.feedback(query, candidate, was_correct=candidate_was_right, similarity_score=0.86,
|
|
87
|
+
verification_id=result.verification_id)
|
|
56
88
|
```
|
|
57
89
|
|
|
58
90
|
`verify()` returns a `VerifyResult`:
|
|
@@ -60,7 +92,10 @@ cv.feedback(query, answer, was_correct=True, similarity_score=0.86)
|
|
|
60
92
|
| field | meaning |
|
|
61
93
|
|---|---|
|
|
62
94
|
| `approved` | serve the cached answer (`True`) or fall through (`False`) |
|
|
63
|
-
| `
|
|
95
|
+
| `audit` | `True` when an approval was drawn for a randomized audit, so `approved` is `False`: call your LLM, serve its answer, and report whether the cached one matched |
|
|
96
|
+
| `verification_id` | set while auditing is on; pass it to `feedback()` |
|
|
97
|
+
| `score` / `threshold` | with `decision_rule="threshold"`, `approved` is `score >= threshold` |
|
|
98
|
+
| `decision_rule` | `"threshold"`, `"joint"` / `"joint_gap"` (a fitted rule that also uses your `similarity_score`), `"aci"` (online threshold), or `"cold_start"` |
|
|
64
99
|
| `model_version` | `"stock"`, `"v<id>"` (fine-tuned), `"cold_start_fail_closed"` / `"cold_start_auto_pending"`, or `"verify_unavailable"` |
|
|
65
100
|
| `latency_ms` | server-side inference time |
|
|
66
101
|
| `degraded` | `True` when the call failed and this result was synthesized client-side |
|
|
@@ -105,23 +140,6 @@ cv = CacheVerifier(api_key="cv_...", fail_open=True)
|
|
|
105
140
|
There is no formal uptime SLA yet, which is the other reason the fallback path
|
|
106
141
|
is a built-in default rather than left to you.
|
|
107
142
|
|
|
108
|
-
## GPTCache
|
|
109
|
-
|
|
110
|
-
Drop the verifier into a GPTCache pipeline as its similarity evaluator — no fork required:
|
|
111
|
-
|
|
112
|
-
```python
|
|
113
|
-
from gptcache import cache
|
|
114
|
-
from cacheverifier.integrations.gptcache import CacheVerifierEvaluation
|
|
115
|
-
|
|
116
|
-
evaluator = CacheVerifierEvaluation(api_key="cv_...")
|
|
117
|
-
cache.init(similarity_evaluation=evaluator, ...)
|
|
118
|
-
|
|
119
|
-
# when you learn a served hit's real outcome:
|
|
120
|
-
evaluator.report_feedback(query, answer, was_correct=False, similarity_score=0.9)
|
|
121
|
-
```
|
|
122
|
-
|
|
123
|
-
See [`examples/gptcache_example.py`](examples/gptcache_example.py).
|
|
124
|
-
|
|
125
143
|
## LangChain
|
|
126
144
|
|
|
127
145
|
`VerifiedCache` wraps any LangChain cache, typically a semantic one, and only
|
|
@@ -146,6 +164,23 @@ set_llm_cache(
|
|
|
146
164
|
)
|
|
147
165
|
```
|
|
148
166
|
|
|
167
|
+
## GPTCache
|
|
168
|
+
|
|
169
|
+
Drop the verifier into a GPTCache pipeline as its similarity evaluator — no fork required:
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
from gptcache import cache
|
|
173
|
+
from cacheverifier.integrations.gptcache import CacheVerifierEvaluation
|
|
174
|
+
|
|
175
|
+
evaluator = CacheVerifierEvaluation(api_key="cv_...")
|
|
176
|
+
cache.init(similarity_evaluation=evaluator, ...)
|
|
177
|
+
|
|
178
|
+
# when you learn a served hit's real outcome:
|
|
179
|
+
evaluator.report_feedback(query, answer, was_correct=False, similarity_score=0.9)
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
See [`examples/gptcache_example.py`](examples/gptcache_example.py).
|
|
183
|
+
|
|
149
184
|
## Entries to refresh
|
|
150
185
|
|
|
151
186
|
Send your cache's own key with feedback, and the service tells you which
|
|
@@ -234,12 +269,14 @@ fully offline. `--emit-summary` writes an aggregate-only JSON file (AUCs, counts
|
|
|
234
269
|
no query or answer text) that's safe to share for a human read.
|
|
235
270
|
|
|
236
271
|
**Choosing the base model.** `--base-model` takes the same values as the hosted
|
|
237
|
-
service's `base_model` fine-tune parameter
|
|
272
|
+
service's `base_model` fine-tune parameter. The defaults differ: the CLI starts from
|
|
273
|
+
`ms_marco`, hosted fine-tunes from `nli`. To compare a local run with a hosted fine-tune,
|
|
274
|
+
pass `--base-model nli`.
|
|
238
275
|
|
|
239
276
|
| `--base-model` | model | use for |
|
|
240
277
|
|---|---|---|
|
|
241
|
-
| `ms_marco` (default) | `cross-encoder/ms-marco-MiniLM-L6-v2` | English traffic |
|
|
242
|
-
| `nli` | `cross-encoder/nli-MiniLM2-L6-H768` | English; entailment pretraining, stronger on negation / entity swaps |
|
|
278
|
+
| `ms_marco` (CLI default) | `cross-encoder/ms-marco-MiniLM-L6-v2` | English traffic |
|
|
279
|
+
| `nli` (hosted default) | `cross-encoder/nli-MiniLM2-L6-H768` | English; entailment pretraining, stronger on negation / entity swaps |
|
|
243
280
|
| `multilingual` | `BAAI/bge-reranker-base` | Chinese and other non-English traffic (~6 GB RAM to fine-tune) |
|
|
244
281
|
| `multilingual_small` | `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` | same, smaller (~3 GB RAM) |
|
|
245
282
|
|
|
@@ -269,6 +306,30 @@ verdict
|
|
|
269
306
|
IMPROVED -- fine-tuning on your own data helps this traffic
|
|
270
307
|
```
|
|
271
308
|
|
|
309
|
+
## Recommended production setup
|
|
310
|
+
|
|
311
|
+
- Keep randomized audits on (the default for new accounts) and report each audited call
|
|
312
|
+
with `feedback(..., verification_id=r.verification_id)`, or `result=r` on
|
|
313
|
+
`LocalVerifier`. That is what makes the measured error rate real.
|
|
314
|
+
- Fine-tune with `cost_ratio=` (how many times worse a wrong reuse is than an extra LLM
|
|
315
|
+
call) instead of the default threshold. ACI (hosted only) is optional: it lets the
|
|
316
|
+
threshold follow the audits, but on the one real drifting stream we tested it gave no
|
|
317
|
+
clear advantage over a fixed threshold.
|
|
318
|
+
- Send `cache_entry_id=` with feedback so `entries_to_refresh()` hands back your keys.
|
|
319
|
+
- At volume, score locally with `LocalVerifier`.
|
|
320
|
+
|
|
321
|
+
## Known limits
|
|
322
|
+
|
|
323
|
+
- The hosted verifier is **English-only** today. Don't send Chinese or other non-English
|
|
324
|
+
traffic to it; multilingual base models are evaluated in the docs but not deployed.
|
|
325
|
+
- Query + answer share a **128-token** input; longer pairs are truncated head and tail.
|
|
326
|
+
- Verify endpoints allow about **20 requests/second per API key**. Use `verify_batch()` or
|
|
327
|
+
`LocalVerifier` beyond that.
|
|
328
|
+
- No formal uptime SLA yet; `verify()` fails closed by default (see above).
|
|
329
|
+
- The shared stock model (`cross-encoder/ms-marco-MiniLM-L6-v2`) is Apache-2.0 but trained
|
|
330
|
+
on MS MARCO, whose terms describe it as for non-commercial research. Fine-tunes start from
|
|
331
|
+
the NLI base (`cross-encoder/nli-MiniLM2-L6-H768`, Apache-2.0) by default.
|
|
332
|
+
|
|
272
333
|
## API surface
|
|
273
334
|
|
|
274
335
|
| method | endpoint |
|
|
@@ -19,7 +19,7 @@ Research behind it: https://github.com/imxinchengyou/CacheVerifier
|
|
|
19
19
|
from cacheverifier.client import CacheVerifier, CacheVerifierError, ShadowVerdict, VerifyResult
|
|
20
20
|
from cacheverifier.local import LocalVerifier, ModelCheckFailed
|
|
21
21
|
|
|
22
|
-
__version__ = "0.7.
|
|
22
|
+
__version__ = "0.7.1"
|
|
23
23
|
__all__ = [
|
|
24
24
|
"CacheVerifier",
|
|
25
25
|
"CacheVerifierError",
|
|
@@ -152,7 +152,7 @@ class CacheVerifier:
|
|
|
152
152
|
from cacheverifier import CacheVerifier
|
|
153
153
|
|
|
154
154
|
cv = CacheVerifier(api_key="cv_...")
|
|
155
|
-
result = cv.verify("
|
|
155
|
+
result = cv.verify("can I change my flight date", "You can change your seat until check-in closes.")
|
|
156
156
|
if result.approved:
|
|
157
157
|
... # serve the cached answer
|
|
158
158
|
else:
|
|
@@ -31,16 +31,16 @@ cache.init(
|
|
|
31
31
|
similarity_evaluation=evaluator,
|
|
32
32
|
)
|
|
33
33
|
|
|
34
|
-
put("how do I
|
|
34
|
+
put("how do I change my flight date?", "Open Manage booking, pick a new date, and pay any fare difference.")
|
|
35
35
|
|
|
36
36
|
# Close-but-different intent. Cosine similarity would likely reuse the
|
|
37
37
|
# cached answer; the verifier call decides whether that is actually safe.
|
|
38
|
-
hit = get("how do I
|
|
38
|
+
hit = get("how do I change my seat on my flight?")
|
|
39
39
|
print("cache returned:", hit)
|
|
40
40
|
|
|
41
41
|
# When you later learn whether a served hit was right:
|
|
42
42
|
evaluator.report_feedback(
|
|
43
|
-
"how do I
|
|
43
|
+
"how do I change my seat on my flight?",
|
|
44
44
|
hit or "",
|
|
45
45
|
was_correct=False,
|
|
46
46
|
)
|
|
@@ -16,17 +16,17 @@ from cacheverifier import CacheVerifier
|
|
|
16
16
|
def call_your_llm(query: str) -> str:
|
|
17
17
|
"""Stand-in for your own LLM call. CacheVerifier doesn't do this part --
|
|
18
18
|
your cache backend already has one."""
|
|
19
|
-
return "
|
|
19
|
+
return "Once a parcel has shipped, contact the carrier to redirect it; the address can't be edited online."
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
def main() -> None:
|
|
23
23
|
api_key = os.environ["CACHEVERIFIER_API_KEY"]
|
|
24
24
|
|
|
25
25
|
# A gray-zone example: your cache's similarity search found a candidate
|
|
26
|
-
# close enough to look like a match, but "
|
|
27
|
-
# difference a plain similarity threshold can't tell apart.
|
|
28
|
-
query = "
|
|
29
|
-
candidate = "
|
|
26
|
+
# close enough to look like a match, but "after" vs "before" it ships is a
|
|
27
|
+
# real difference a plain similarity threshold can't tell apart.
|
|
28
|
+
query = "can I change the delivery address after my parcel ships"
|
|
29
|
+
candidate = "You can change the delivery address any time before the parcel ships."
|
|
30
30
|
|
|
31
31
|
# verify() sits on your request path: it uses a 1s timeout and, if the
|
|
32
32
|
# service is unreachable, returns a degraded result (approved=False here)
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cacheverifier"
|
|
7
|
-
version = "0.7.
|
|
7
|
+
version = "0.7.1"
|
|
8
8
|
description = "Python client for the CacheVerifier semantic-cache verification API, hosted or scored locally"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|