veriquote 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CITATION.cff +4 -3
- package/README.md +227 -121
- package/bin/veriquote.mjs +24 -0
- package/dist/cli/main.d.ts +26 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +262 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/gate.d.ts +42 -0
- package/dist/gate.d.ts.map +1 -0
- package/dist/gate.js +116 -0
- package/dist/gate.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/judge/chat-judge.js +1 -1
- package/dist/judge/chat-judge.js.map +1 -1
- package/dist/source/fetch.d.ts +21 -0
- package/dist/source/fetch.d.ts.map +1 -0
- package/dist/source/fetch.js +70 -0
- package/dist/source/fetch.js.map +1 -0
- package/dist/source/html.d.ts +18 -0
- package/dist/source/html.d.ts.map +1 -0
- package/dist/source/html.js +69 -0
- package/dist/source/html.js.map +1 -0
- package/package.json +18 -3
- package/src/cli/main.ts +305 -0
- package/src/gate.ts +156 -0
- package/src/index.ts +4 -0
- package/src/judge/chat-judge.ts +1 -1
- package/src/source/fetch.ts +78 -0
- package/src/source/html.ts +74 -0
package/CITATION.cff
CHANGED
|
@@ -9,11 +9,12 @@ authors:
|
|
|
9
9
|
orcid: "https://orcid.org/0009-0009-5596-736X"
|
|
10
10
|
repository-code: "https://github.com/rickintoplace/veriquote"
|
|
11
11
|
license: MIT
|
|
12
|
-
version: 0.
|
|
12
|
+
version: 0.2.0
|
|
13
13
|
identifiers:
|
|
14
14
|
- type: doi
|
|
15
|
-
value: 10.5281/zenodo.
|
|
16
|
-
|
|
15
|
+
value: 10.5281/zenodo.21552379
|
|
16
|
+
description: Concept DOI, resolves to the latest version
|
|
17
|
+
date-released: "2026-09-22"
|
|
17
18
|
keywords:
|
|
18
19
|
- large language models
|
|
19
20
|
- retrieval-augmented generation
|
package/README.md
CHANGED
|
@@ -1,69 +1,56 @@
|
|
|
1
1
|
# VeriQuote
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/veriquote)
|
|
4
4
|
[](https://doi.org/10.5281/zenodo.21552379)
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
6
|
+
**Make an LLM quote its sources, then check every quote twice: is it really in
|
|
7
|
+
the source, and does it support the claim?**
|
|
8
|
+
|
|
9
|
+
A `[1]` after a sentence looks like evidence and usually is not checked by
|
|
10
|
+
anyone. In our tests a third of one capable model's "verbatim" quotes were not
|
|
11
|
+
in the source verbatim, another left 69% of its cited answers with at least one
|
|
12
|
+
citation that had nothing behind it, and a judge model happily confirmed quotes
|
|
13
|
+
that were invented. VeriQuote makes the answering model commit to a verbatim
|
|
14
|
+
quote per citation and then verifies each one — deterministically where
|
|
15
|
+
possible, with an LLM only where it has to. TypeScript, zero dependencies, runs in Node and the
|
|
16
|
+
browser.
|
|
17
|
+
|
|
18
|
+
```console
|
|
19
|
+
$ curl -s https://raw.githubusercontent.com/rickintoplace/veriquote/main/examples/ozone-answer.md \
|
|
20
|
+
| npx veriquote check - --source https://en.wikipedia.org/wiki/Ozone_layer
|
|
21
|
+
|
|
22
|
+
4 citation(s) · 1 source(s) · judge: glm-5.3-flash
|
|
23
|
+
|
|
24
|
+
✓ c1 [1] verbatim 1.00 · entailed 1.00
|
|
25
|
+
The ozone layer absorbs 97 to 99 percent of the Sun's medium-frequency ultraviolet light.
|
|
26
|
+
✗ c2 [1] verbatim 1.00 · contradicted 0.00
|
|
27
|
+
It was discovered in 1913 by the British meteorologist G. M. B. Dobson.
|
|
28
|
+
Quote credits Fabry and Buisson, not Dobson
|
|
29
|
+
✗ c3 [1] fuzzy 0.48 · contradicted 0.00
|
|
30
|
+
Under the Montreal Protocol, all CFC production was banned immediately in 1987.
|
|
31
|
+
Context says production capped at 1986 levels, not banned; Quote absent from context; appears fabricated
|
|
32
|
+
✗ c4 [1] quote not in source (best 0.31) · entailed 0.90
|
|
33
|
+
The treaty limited CFC production to the levels of 1986.
|
|
34
|
+
the quoted text does not occur in the source
|
|
35
|
+
! uncited Ozone depletion has since been fully reversed in every region of the atmosphere.
|
|
36
|
+
|
|
37
|
+
REVISE — 3 failed citation(s), 1 uncited sentence(s)
|
|
38
|
+
```
|
|
19
39
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
40
|
+
`c2` quotes the source correctly and still gets the facts wrong — only the judge
|
|
41
|
+
sees that. `c4` invents its quote and the judge calls it supported — only the
|
|
42
|
+
matcher sees that. Without an API key the CLI checks the quotes only; set
|
|
43
|
+
`VERIQUOTE_JUDGE_API_KEY` and `VERIQUOTE_JUDGE_MODEL` (any OpenAI-compatible
|
|
44
|
+
endpoint) for the judge.
|
|
23
45
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
at temperature 0 with a closed class vocabulary and strict output validation.
|
|
27
|
-
- **Model-agnostic.** Works with any answering model and any OpenAI-compatible
|
|
28
|
-
chat-completions endpoint for the judge (OpenAI, OpenRouter, Azure, local
|
|
29
|
-
gateways) or bring your own `EntailmentJudge` (e.g. a local NLI model).
|
|
46
|
+
**Try both checks in the browser:** [`demo/index.html`](demo/index.html) — eight
|
|
47
|
+
examples with recorded judge verdicts, or your own text with your own key.
|
|
30
48
|
|
|
31
49
|
## How it works
|
|
32
50
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
+ citation prompt │ (any model) │
|
|
37
|
-
└────────────┬──────────────┘
|
|
38
|
-
│ answer body with [n]{cX} markers
|
|
39
|
-
│ + EVI1 quote appendix
|
|
40
|
-
▼
|
|
41
|
-
┌───────────────────────────┐
|
|
42
|
-
│ 1. parseAnswer() │ claims, quotes, protocol
|
|
43
|
-
│ (deterministic) │ completeness warnings
|
|
44
|
-
└────────────┬──────────────┘
|
|
45
|
-
▼
|
|
46
|
-
┌───────────────────────────┐
|
|
47
|
-
│ 2. Quote ↔ source match │ exact / normalized /
|
|
48
|
-
│ (deterministic, fuzzy) │ fuzzy %, offsets
|
|
49
|
-
└────────────┬──────────────┘
|
|
50
|
-
▼
|
|
51
|
-
┌───────────────────────────┐
|
|
52
|
-
│ 3. Entailment judge │ entailed / partially /
|
|
53
|
-
│ (LLM, temp 0, optional)│ overstated / insufficient
|
|
54
|
-
└────────────┬──────────────┘ / contradicted + conf.
|
|
55
|
-
▼
|
|
56
|
-
┌───────────────────────────┐
|
|
57
|
-
│ 4. VerificationReport │ per-citation scores +
|
|
58
|
-
│ (transparency for user)│ answer-level summary
|
|
59
|
-
└───────────────────────────┘
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
### The EVI1 protocol
|
|
63
|
-
|
|
64
|
-
The answering model is instructed (via `buildCitationInstructions()`) to end
|
|
65
|
-
every cited sentence with citation markers and a claim marker, and to append a
|
|
66
|
-
machine-readable quote appendix:
|
|
51
|
+
The answering model gets `veriquote prompt` (or `buildCitationInstructions()`)
|
|
52
|
+
in its system prompt. Every cited sentence ends with source and claim markers,
|
|
53
|
+
and the answer ends with a plain-text quote appendix:
|
|
67
54
|
|
|
68
55
|
```
|
|
69
56
|
Vitamin D supplementation reduced fall risk in older adults.[1]{c1}
|
|
@@ -76,43 +63,151 @@ c2|3|"BMD improved with \"high-dose\" regimens"
|
|
|
76
63
|
END_EVI1
|
|
77
64
|
```
|
|
78
65
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
66
|
+
Plain text rather than JSON, so it survives streaming, markdown renderers and
|
|
67
|
+
weak models, and the `[n]` markers stay readable if nothing checks them. Then,
|
|
68
|
+
per citation:
|
|
69
|
+
|
|
70
|
+
1. **Parse** — every cited claim must carry a quote; missing ones are reported.
|
|
71
|
+
2. **Match** — is the quote in the source? Deterministic fuzzy matching that
|
|
72
|
+
tolerates whitespace, typography, OCR noise and elision, with offsets.
|
|
73
|
+
3. **Judge** — does the quote support the claim? An LLM at temperature 0 picks
|
|
74
|
+
`entailed`, `partially_entailed`, `overstated`, `insufficient` or
|
|
75
|
+
`contradicted`, with a support score.
|
|
76
|
+
|
|
77
|
+
The combined score is `min(match, support)`: no judge error can raise a
|
|
78
|
+
citation above what the matcher found, and a judge failure is reported, never
|
|
79
|
+
counted as support. The prompt is a transparency mechanism, not a cure —
|
|
80
|
+
quoting does not make a model hallucinate less, it makes every claim checkable.
|
|
81
|
+
|
|
82
|
+
## Does it actually work?
|
|
83
|
+
|
|
84
|
+
Three benchmarks, kept separate on purpose — one blended number for a two-stage
|
|
85
|
+
pipeline would hide the failures it exists to separate. Everything is in
|
|
86
|
+
[`bench/`](bench), including how to reproduce it.
|
|
87
|
+
|
|
88
|
+
<picture>
|
|
89
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/tango-dark.svg">
|
|
90
|
+
<img alt="Matcher catches 100% of quotes that are not in the source and 0% of real quotes attached to unsupported claims; the judge catches 84% of the latter and cannot see the former; together they cover both." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/tango-light.svg">
|
|
91
|
+
</picture>
|
|
92
|
+
|
|
93
|
+
The two checks are blind in opposite places. The matcher cannot tell whether a
|
|
94
|
+
real quote supports the claim; the judge only compares claim and quote, so a
|
|
95
|
+
fabricated quote that fits the claim sails through it — in
|
|
96
|
+
[`examples/ozone-answer.md`](examples/ozone-answer.md) `deepseek-v4-flash` rates an
|
|
97
|
+
invented quote "entailed 1.00", and only the matcher notices it is not in the
|
|
98
|
+
source. That is why the combined score is `min(textMatchScore, judgeConfidence)`.
|
|
99
|
+
|
|
100
|
+
### Matcher
|
|
101
|
+
|
|
102
|
+
<picture>
|
|
103
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/matcher-dark.svg">
|
|
104
|
+
<img alt="Matcher score ranges per mutation: faithful quotes score 0.66 to 1.0, quotes not in the source 0.14 to 0.40, meaning-changed quotes score high by design." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/matcher-light.svg">
|
|
105
|
+
</picture>
|
|
106
|
+
|
|
107
|
+
2,061 quotes, no API key, no labels — ground truth by construction;
|
|
108
|
+
`npm run bench:matcher` reproduces it in about two seconds. Faithfully copied
|
|
109
|
+
quotes never score below 0.660, missing ones never above 0.396, and the default
|
|
110
|
+
threshold of 0.4 sits in that gap. The orange rows are the point, not an
|
|
111
|
+
embarrassment: a quote whose meaning was changed is still near-verbatim, and
|
|
112
|
+
catching it is the judge's job. Two honest limits: invented prose built from the
|
|
113
|
+
source's own words (adversarial) scores high, and so can hand-written
|
|
114
|
+
fabrications that reuse the source's vocabulary (0.48 in the example above) —
|
|
115
|
+
the CLI therefore fails any citation below 0.5 and leaves the rest to the judge.
|
|
116
|
+
|
|
117
|
+
### Judge
|
|
118
|
+
|
|
119
|
+
<picture>
|
|
120
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/judges-dark.svg">
|
|
121
|
+
<img alt="Five open judge models against ALCE human labels: false green from 16.1% (glm-5.3-flash) to 25.0% (gpt-oss-120b); binary agreement 76.7% to 80.3%, around the TRUE-NLI baseline of 77.6%." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/judges-light.svg">
|
|
122
|
+
</picture>
|
|
123
|
+
|
|
124
|
+
Agreement with the human annotators of
|
|
125
|
+
[ALCE](https://github.com/princeton-nlp/ALCE) (Gao et al., EMNLP 2023, MIT),
|
|
126
|
+
class mapping fixed before any model ran. Open general-purpose models are level
|
|
127
|
+
with a specialised 11B NLI model on ALCE's binary question, and the best reaches
|
|
128
|
+
Cohen's κ 0.53 — the agreement ALCE reports for its own automatic metric.
|
|
129
|
+
**Read the left panel before trusting any of this:** even the best judge calls
|
|
130
|
+
one unsupported citation in six "fully supported", which is why no judge should
|
|
131
|
+
be the only check. The intervals are wide (56 unsupported pairs per run), so
|
|
132
|
+
the ranking between neighbouring models is not settled; size is not what
|
|
133
|
+
decides it — a 3B-active MoE lands ahead of a 120B model.
|
|
134
|
+
|
|
135
|
+
### Does the answering model play along?
|
|
136
|
+
|
|
137
|
+
<picture>
|
|
138
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/protocol-dark.svg">
|
|
139
|
+
<img alt="Protocol compliance: gpt-oss-120b complete in 30.8% of answers, mistral-medium quotes verbatim in 66.5% of citations; gemma-4-31b and llama-3.1-8b shown for comparison." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/protocol-light.svg">
|
|
140
|
+
</picture>
|
|
141
|
+
|
|
142
|
+
Decided mechanically by `parseAnswer()` over 18 tasks, three of which the
|
|
143
|
+
sources deliberately cannot answer. Two ways to fail, neither visible to a
|
|
144
|
+
reader: `gpt-oss-120b` always prints an appendix, but only 30.8% of its answers
|
|
145
|
+
give every citation a quote — the rest are footnotes with nothing behind them.
|
|
146
|
+
`mistral-medium` is almost always complete, yet a third of its "quotes" are not
|
|
147
|
+
in the source: it paraphrased into the quote slot. Both answers look impeccably
|
|
148
|
+
cited. All four models cited nothing on the three unanswerable questions.
|
|
149
|
+
|
|
150
|
+
<details>
|
|
151
|
+
<summary>The numbers behind the figures</summary>
|
|
152
|
+
|
|
153
|
+
| judge model | false green ↓ | binary agreement | Cohen's κ |
|
|
154
|
+
| --- | ---: | ---: | ---: |
|
|
155
|
+
| `glm-5.3-flash` | 16.1% | 80.3% | 0.529 |
|
|
156
|
+
| `qwen3.5-397b-a17b` | 20.8% | 79.7% | 0.483 |
|
|
157
|
+
| `qwen3.6-35b-a3b` | 18.2% | 78.1% | 0.454 |
|
|
158
|
+
| `deepseek-v4-flash` | 18.2% | 78.7% | 0.435 |
|
|
159
|
+
| `gpt-oss-120b` | 25.0% | 76.7% | 0.394 |
|
|
160
|
+
| ALCE's TRUE-NLI (T5-11B) | | 77.6% | |
|
|
161
|
+
|
|
162
|
+
| quote family | n | median score | accepted at 0.4 |
|
|
163
|
+
| --- | ---: | ---: | ---: |
|
|
164
|
+
| faithful but reformatted | 1,121 | 1.000 | 100.0% |
|
|
165
|
+
| near-verbatim, meaning changed | 586 | 0.932 | 100.0% |
|
|
166
|
+
| absent from the source | 187 | 0.249 | 0.0% |
|
|
167
|
+
|
|
168
|
+
| answering model | appendix | complete | verbatim | warning-free |
|
|
169
|
+
| --- | ---: | ---: | ---: | ---: |
|
|
170
|
+
| gpt-oss-120b | 100% | 30.8% | 93.3% | 23.1% |
|
|
171
|
+
| mistral-medium-3.5-128b | 100% | 92.3% | 66.5% | 92.3% |
|
|
172
|
+
| gemma-4-31b-it | 86.7% | 86.7% | 100% | 86.7% |
|
|
173
|
+
| llama-3.1-8b-instruct | 71.4% | 50.0% | 62.8% | 14.3% |
|
|
174
|
+
|
|
175
|
+
Figures are rendered from `bench/results/*.json` by
|
|
176
|
+
[`bench/figures/render.mjs`](bench/figures/render.mjs).
|
|
177
|
+
</details>
|
|
178
|
+
|
|
179
|
+
## Use from an agent
|
|
180
|
+
|
|
181
|
+
An agent that reads sources and writes conclusions is exactly the case this was
|
|
182
|
+
built for. **[`verify-citations`](integrations/verify-citations)** is an
|
|
183
|
+
[Agent Skill](integrations/verify-citations/SKILL.md) that has the agent write
|
|
184
|
+
its sourced answer in the checkable format and run `veriquote check` on it
|
|
185
|
+
before presenting it. The CLI fetches every cited URL itself, so an agent cannot
|
|
186
|
+
pass with its own (truncated, misremembered) copy of a page, and the exit code
|
|
187
|
+
lets it branch without parsing anything:
|
|
86
188
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
deterministic matcher catches it, with a percent score that distinguishes
|
|
93
|
-
light paraphrase (high fuzzy score) from fabrication (low score).
|
|
189
|
+
```
|
|
190
|
+
exit 0 verdict "pass" every cited claim is grounded -> present the answer
|
|
191
|
+
exit 2 verdict "revise" problems[] + instructionsForModel -> fix and re-check
|
|
192
|
+
exit 1 bad input or unreachable source -> do NOT claim the answer was verified
|
|
193
|
+
```
|
|
94
194
|
|
|
95
|
-
|
|
96
|
-
|
|
195
|
+
It works in any host that reads Agent Skills, such as Claude Code, and in
|
|
196
|
+
anything that can run a shell command.
|
|
97
197
|
|
|
98
|
-
##
|
|
198
|
+
## Use as a library
|
|
99
199
|
|
|
100
200
|
```bash
|
|
101
201
|
npm install veriquote
|
|
102
202
|
```
|
|
103
203
|
|
|
104
|
-
## Quickstart
|
|
105
|
-
|
|
106
|
-
### 1. Prompt the answering model
|
|
107
|
-
|
|
108
204
|
```ts
|
|
109
205
|
import { buildCitationInstructions } from 'veriquote';
|
|
110
206
|
|
|
111
207
|
const systemPrompt = `${yourAssistantPrompt}\n\n${buildCitationInstructions()}`;
|
|
112
|
-
//
|
|
208
|
+
// Give the sources to the model as numbered blocks [1], [2], …
|
|
113
209
|
```
|
|
114
210
|
|
|
115
|
-
### 2. Verify the raw answer
|
|
116
211
|
|
|
117
212
|
```ts
|
|
118
213
|
import { ChatCompletionsJudge, verifyAnswer } from 'veriquote';
|
|
@@ -120,7 +215,7 @@ import { ChatCompletionsJudge, verifyAnswer } from 'veriquote';
|
|
|
120
215
|
const judge = new ChatCompletionsJudge({
|
|
121
216
|
baseUrl: 'https://openrouter.ai/api/v1', // any OpenAI-compatible endpoint
|
|
122
217
|
apiKey: process.env.OPENROUTER_API_KEY, // server-side only!
|
|
123
|
-
model: '
|
|
218
|
+
model: 'your-judge-model', // pick one with bench/judge
|
|
124
219
|
});
|
|
125
220
|
|
|
126
221
|
const report = await verifyAnswer({
|
|
@@ -145,13 +240,10 @@ for (const c of report.citations) {
|
|
|
145
240
|
`report.cleanText` is the answer with all `{cX}` markers removed, ready to
|
|
146
241
|
render (the `[n]` markers remain as human-readable citations).
|
|
147
242
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
is exactly what the NavigNine UI does with tooltips and colored footnotes.
|
|
153
|
-
|
|
154
|
-
## API overview
|
|
243
|
+
`report.cleanText` is the answer without `{cX}` markers, ready to render. Show
|
|
244
|
+
each citation's match score, judge class and combined score next to its
|
|
245
|
+
footnote, so a reader can see which sentence is load-bearing without opening a
|
|
246
|
+
source.
|
|
155
247
|
|
|
156
248
|
| Export | Purpose |
|
|
157
249
|
| --- | --- |
|
|
@@ -162,13 +254,12 @@ is exactly what the NavigNine UI does with tooltips and colored footnotes.
|
|
|
162
254
|
| `matchQuoteAgainstSource(quote, source, options?)` | Deterministic quote matching on its own. |
|
|
163
255
|
| `ChatCompletionsJudge` | Entailment judge for any OpenAI-compatible API. |
|
|
164
256
|
| `EntailmentJudge` (interface) | Bring your own judge (local NLI model, other provider). |
|
|
257
|
+
| `gateReport(report, answer)` | Pass/revise verdict, problem list, uncited sentences, correction prompt. |
|
|
258
|
+
| `fetchSource(url)` / `htmlToText(html)` | Fetch a source independently of the model and extract its text. |
|
|
165
259
|
|
|
166
260
|
All inputs and outputs are plain, serializable data — see
|
|
167
|
-
[`src/types.ts`](src/types.ts)
|
|
168
|
-
[`docs/DESIGN.md`](docs/DESIGN.md)
|
|
169
|
-
thresholds, and design rationale).
|
|
170
|
-
|
|
171
|
-
### Entailment classes
|
|
261
|
+
[`src/types.ts`](src/types.ts) and, for scoring and thresholds,
|
|
262
|
+
[`docs/DESIGN.md`](docs/DESIGN.md).
|
|
172
263
|
|
|
173
264
|
| Class | Confidence band | Meaning |
|
|
174
265
|
| --- | --- | --- |
|
|
@@ -179,39 +270,54 @@ thresholds, and design rationale).
|
|
|
179
270
|
| `contradicted` | 0.0 | Evidence says the opposite. |
|
|
180
271
|
| `error` | — | Judge unavailable for this item (never silently dropped). |
|
|
181
272
|
|
|
273
|
+
## How this differs from the alternatives
|
|
274
|
+
|
|
275
|
+
| | verbatim quote checked | claim↔evidence checked | model-agnostic | runtime |
|
|
276
|
+
| --- | --- | --- | --- | --- |
|
|
277
|
+
| **VeriQuote** | yes, deterministic | yes, pluggable judge | yes | TS, zero deps, browser/edge |
|
|
278
|
+
| [Anthropic Citations API](https://platform.claude.com/docs/en/build-with-claude/citations) | n/a — spans are extracted, so they are real by construction | no | Claude only | hosted |
|
|
279
|
+
| [LettuceDetect](https://github.com/KRLabsOrg/LettuceDetect) | no — no quote protocol | yes, span-level model | yes | Python + model weights |
|
|
280
|
+
| [RAGAS](https://github.com/explodinggradients/ragas) and eval frameworks | no | yes, as an offline metric | yes | Python, offline eval |
|
|
281
|
+
|
|
282
|
+
**Use the Citations API instead** if you are on Claude and only need to know
|
|
283
|
+
that a span is real: it guarantees that by construction, which is stronger than
|
|
284
|
+
any matcher. It does not tell you whether the span supports the sentence built
|
|
285
|
+
on it — for that, pair it with this library's judge and skip the matcher.
|
|
286
|
+
|
|
287
|
+
**Use LettuceDetect instead** if you want unsupported spans flagged in an
|
|
288
|
+
answer that has no citation protocol at all, and you are happy running a model
|
|
289
|
+
in Python. It solves the post-hoc problem; VeriQuote changes what the answering
|
|
290
|
+
model commits to in the first place.
|
|
291
|
+
|
|
292
|
+
VeriQuote's own niche is narrow and worth stating plainly: you want the
|
|
293
|
+
answering model pinned to a quote *before* it generates, you want the
|
|
294
|
+
deterministic half of the check to run anywhere including a browser with no
|
|
295
|
+
dependencies, and you want per-claim numbers to put in front of a reader rather
|
|
296
|
+
than an aggregate score for a dashboard.
|
|
297
|
+
|
|
298
|
+
Closest in spirit is the concurrent academic work by Zhang et al.,
|
|
299
|
+
[“Verifiable by Construction”](https://arxiv.org/abs/2609.15964) (Johns Hopkins,
|
|
300
|
+
2026), which evaluates the same design — inline verbatim quotes, tiered
|
|
301
|
+
verbatim matching, an LLM judge — on clinical guidelines. VeriQuote is the
|
|
302
|
+
deployable library, CLI and agent skill, with the judge measured against human
|
|
303
|
+
labels.
|
|
304
|
+
|
|
182
305
|
## Security
|
|
183
306
|
|
|
184
|
-
- **Keep the
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
- **
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
hallucinated item IDs are rejected.
|
|
192
|
-
- **No dynamic evaluation.** Tolerant JSON recovery is a string-aware scanner;
|
|
193
|
-
nothing is ever `eval`ed.
|
|
194
|
-
- **Failure transparency.** Judge failures degrade to `class: "error"` with a
|
|
195
|
-
`null` score — they are reported, never counted as "supported".
|
|
307
|
+
- **Keep your key on the server.** `ChatCompletionsJudge` needs an API key; in
|
|
308
|
+
your own app, call `verifyAnswer` from a backend. (The demo runs the judge in
|
|
309
|
+
the browser only with a key the visitor enters.)
|
|
310
|
+
- **Source text is untrusted.** Judge inputs are length-capped, stripped of
|
|
311
|
+
control characters and HTML, and pinned as data in the prompt. Output is
|
|
312
|
+
validated against a closed vocabulary; unknown classes, out-of-range scores
|
|
313
|
+
and invented item IDs are rejected. Nothing is ever `eval`ed.
|
|
196
314
|
|
|
197
315
|
## Reproducibility
|
|
198
316
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
or use a self-hosted judge behind the `EntailmentJudge` interface.
|
|
204
|
-
|
|
205
|
-
## Integrations
|
|
206
|
-
|
|
207
|
-
- **[`verify-citations`](integrations/verify-citations/)**: a portable
|
|
208
|
-
[Agent Skill](integrations/verify-citations/SKILL.md) (single `SKILL.md` +
|
|
209
|
-
bundled Node CLI) that runs VeriQuote as an **internal hallucination gate**
|
|
210
|
-
for source-grounded agents: it verifies a cited answer, flags factual
|
|
211
|
-
sentences that carry no citation, and returns a ready-to-use correction
|
|
212
|
-
prompt for a self-correction loop. The same skill works across any
|
|
213
|
-
Agent-Skills host (OpenClaw, Hermes Agent, Claude Code) and any orchestrator
|
|
214
|
-
that can run a Node CLI.
|
|
317
|
+
The matcher is pure: same inputs, same score. The judge runs at temperature 0
|
|
318
|
+
(pass `seed` where the provider supports it), but hosted models are only
|
|
319
|
+
best-effort deterministic; pin the model version, or put a self-hosted model
|
|
320
|
+
behind the `EntailmentJudge` interface.
|
|
215
321
|
|
|
216
322
|
## Citing
|
|
217
323
|
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Thin process wrapper; the command logic lives in src/cli/main.ts.
|
|
3
|
+
import { readFile } from 'node:fs/promises';
|
|
4
|
+
import { main } from '../dist/cli/main.js';
|
|
5
|
+
|
|
6
|
+
async function readStdin() {
|
|
7
|
+
let data = '';
|
|
8
|
+
process.stdin.setEncoding('utf8');
|
|
9
|
+
for await (const chunk of process.stdin) data += chunk;
|
|
10
|
+
return data;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const pkg = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
|
|
14
|
+
|
|
15
|
+
process.exitCode = await main({
|
|
16
|
+
argv: process.argv.slice(2),
|
|
17
|
+
env: process.env,
|
|
18
|
+
stdout: (s) => process.stdout.write(s),
|
|
19
|
+
stderr: (s) => process.stderr.write(s),
|
|
20
|
+
readFile: (path) => readFile(path, 'utf8'),
|
|
21
|
+
readStdin,
|
|
22
|
+
version: pkg.version,
|
|
23
|
+
color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR,
|
|
24
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `veriquote` command line. Kept free of `process` so it can be tested with
|
|
3
|
+
* injected I/O; `bin/veriquote.mjs` wires it to the real process.
|
|
4
|
+
*/
|
|
5
|
+
import type { EntailmentJudge } from '../types.js';
|
|
6
|
+
export interface CliIo {
|
|
7
|
+
argv: string[];
|
|
8
|
+
env: Record<string, string | undefined>;
|
|
9
|
+
stdout: (s: string) => void;
|
|
10
|
+
stderr: (s: string) => void;
|
|
11
|
+
readFile: (path: string) => Promise<string>;
|
|
12
|
+
readStdin: () => Promise<string>;
|
|
13
|
+
fetch?: typeof globalThis.fetch;
|
|
14
|
+
version: string;
|
|
15
|
+
/** Use ANSI colors in human output. */
|
|
16
|
+
color: boolean;
|
|
17
|
+
/** Test hook: replaces the env-configured judge. */
|
|
18
|
+
judge?: EntailmentJudge;
|
|
19
|
+
}
|
|
20
|
+
export declare const EXIT: {
|
|
21
|
+
readonly pass: 0;
|
|
22
|
+
readonly error: 1;
|
|
23
|
+
readonly revise: 2;
|
|
24
|
+
};
|
|
25
|
+
export declare function main(io: CliIo): Promise<number>;
|
|
26
|
+
//# sourceMappingURL=main.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAsC,MAAM,aAAa,CAAC;AAOvF,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACxC,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,uCAAuC;IACvC,KAAK,EAAE,OAAO,CAAC;IACf,oDAAoD;IACpD,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,eAAO,MAAM,IAAI;;;;CAA4C,CAAC;AAkG9D,wBAAsB,IAAI,CAAC,EAAE,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAgCrD"}
|