veriquote 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CITATION.cff +4 -3
- package/README.md +280 -146
- package/bin/veriquote.mjs +24 -0
- package/dist/cli/main.d.ts +26 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +262 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/gate.d.ts +42 -0
- package/dist/gate.d.ts.map +1 -0
- package/dist/gate.js +116 -0
- package/dist/gate.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/judge/chat-judge.d.ts +7 -0
- package/dist/judge/chat-judge.d.ts.map +1 -1
- package/dist/judge/chat-judge.js +3 -1
- package/dist/judge/chat-judge.js.map +1 -1
- package/dist/source/fetch.d.ts +21 -0
- package/dist/source/fetch.d.ts.map +1 -0
- package/dist/source/fetch.js +70 -0
- package/dist/source/fetch.js.map +1 -0
- package/dist/source/html.d.ts +18 -0
- package/dist/source/html.d.ts.map +1 -0
- package/dist/source/html.js +69 -0
- package/dist/source/html.js.map +1 -0
- package/package.json +20 -3
- package/src/cli/main.ts +305 -0
- package/src/gate.ts +156 -0
- package/src/index.ts +4 -0
- package/src/judge/chat-judge.ts +12 -2
- package/src/source/fetch.ts +78 -0
- package/src/source/html.ts +74 -0
package/CITATION.cff
CHANGED
|
@@ -9,11 +9,12 @@ authors:
|
|
|
9
9
|
orcid: "https://orcid.org/0009-0009-5596-736X"
|
|
10
10
|
repository-code: "https://github.com/rickintoplace/veriquote"
|
|
11
11
|
license: MIT
|
|
12
|
-
version: 0.
|
|
12
|
+
version: 0.2.1
|
|
13
13
|
identifiers:
|
|
14
14
|
- type: doi
|
|
15
|
-
value: 10.5281/zenodo.
|
|
16
|
-
|
|
15
|
+
value: 10.5281/zenodo.21552379
|
|
16
|
+
description: Concept DOI, resolves to the latest version
|
|
17
|
+
date-released: "2026-09-22"
|
|
17
18
|
keywords:
|
|
18
19
|
- large language models
|
|
19
20
|
- retrieval-augmented generation
|
package/README.md
CHANGED
|
@@ -1,69 +1,59 @@
|
|
|
1
1
|
# VeriQuote
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/veriquote)
|
|
4
4
|
[](https://doi.org/10.5281/zenodo.21552379)
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
6
|
+
**Make an LLM quote its sources, then check every quote twice: is it really in
|
|
7
|
+
the source, and does it support the claim?**
|
|
8
|
+
|
|
9
|
+
A `[1]` after a sentence looks like evidence, but almost nobody checks it. In
|
|
10
|
+
my tests, a fifth of one capable model's "verbatim" quotes were not in the
|
|
11
|
+
source, another model left two thirds of its cited answers with at least one
|
|
12
|
+
citation that had nothing behind it, and an LLM judge happily confirmed a quote
|
|
13
|
+
that was invented. VeriQuote makes the answering model commit to a verbatim
|
|
14
|
+
quote for every citation, then verifies each one: deterministically where
|
|
15
|
+
possible, with an LLM only where it has to. It is written in TypeScript, has no
|
|
16
|
+
dependencies and runs in Node and in the browser.
|
|
17
|
+
|
|
18
|
+
**Try it in the browser:** [rickinto.place/veriquote](https://rickinto.place/veriquote)
|
|
19
|
+
has eight worked examples, a box for your own text, and the benchmark results as
|
|
20
|
+
interactive charts.
|
|
21
|
+
|
|
22
|
+
```console
|
|
23
|
+
$ curl -s https://raw.githubusercontent.com/rickintoplace/veriquote/main/examples/ozone-answer.md \
|
|
24
|
+
| npx veriquote check - --source https://en.wikipedia.org/wiki/Ozone_layer
|
|
25
|
+
|
|
26
|
+
4 citation(s) · 1 source(s) · judge: glm-5.3-flash
|
|
27
|
+
|
|
28
|
+
✓ c1 [1] verbatim 1.00 · entailed 1.00
|
|
29
|
+
The ozone layer absorbs 97 to 99 percent of the Sun's medium-frequency ultraviolet light.
|
|
30
|
+
✗ c2 [1] verbatim 1.00 · contradicted 0.00
|
|
31
|
+
It was discovered in 1913 by the British meteorologist G. M. B. Dobson.
|
|
32
|
+
Quote credits Fabry and Buisson, not Dobson
|
|
33
|
+
✗ c3 [1] fuzzy 0.48 · contradicted 0.00
|
|
34
|
+
Under the Montreal Protocol, all CFC production was banned immediately in 1987.
|
|
35
|
+
Context says production capped at 1986 levels, not banned; Quote absent from context; appears fabricated
|
|
36
|
+
✗ c4 [1] quote not in source (best 0.31) · entailed 0.90
|
|
37
|
+
The treaty limited CFC production to the levels of 1986.
|
|
38
|
+
the quoted text does not occur in the source
|
|
39
|
+
! uncited Ozone depletion has since been fully reversed in every region of the atmosphere.
|
|
40
|
+
|
|
41
|
+
REVISE — 3 failed citation(s), 1 uncited sentence(s)
|
|
42
|
+
```
|
|
23
43
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
44
|
+
Look at `c2` and `c4`. The quote in `c2` is copied correctly, but the claim
|
|
45
|
+
names the wrong person; only the judge can see that. The quote in `c4` is made
|
|
46
|
+
up, and the judge still calls it supported; only the matcher can see that.
|
|
47
|
+
Without an API key the CLI checks the quotes alone. Set
|
|
48
|
+
`VERIQUOTE_JUDGE_API_KEY` and `VERIQUOTE_JUDGE_MODEL` (any OpenAI-compatible
|
|
49
|
+
endpoint) to add the judge.
|
|
30
50
|
|
|
31
51
|
## How it works
|
|
32
52
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
└────────────┬──────────────┘
|
|
38
|
-
│ answer body with [n]{cX} markers
|
|
39
|
-
│ + EVI1 quote appendix
|
|
40
|
-
▼
|
|
41
|
-
┌───────────────────────────┐
|
|
42
|
-
│ 1. parseAnswer() │ claims, quotes, protocol
|
|
43
|
-
│ (deterministic) │ completeness warnings
|
|
44
|
-
└────────────┬──────────────┘
|
|
45
|
-
▼
|
|
46
|
-
┌───────────────────────────┐
|
|
47
|
-
│ 2. Quote ↔ source match │ exact / normalized /
|
|
48
|
-
│ (deterministic, fuzzy) │ fuzzy %, offsets
|
|
49
|
-
└────────────┬──────────────┘
|
|
50
|
-
▼
|
|
51
|
-
┌───────────────────────────┐
|
|
52
|
-
│ 3. Entailment judge │ entailed / partially /
|
|
53
|
-
│ (LLM, temp 0, optional)│ overstated / insufficient
|
|
54
|
-
└────────────┬──────────────┘ / contradicted + conf.
|
|
55
|
-
▼
|
|
56
|
-
┌───────────────────────────┐
|
|
57
|
-
│ 4. VerificationReport │ per-citation scores +
|
|
58
|
-
│ (transparency for user)│ answer-level summary
|
|
59
|
-
└───────────────────────────┘
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
### The EVI1 protocol
|
|
63
|
-
|
|
64
|
-
The answering model is instructed (via `buildCitationInstructions()`) to end
|
|
65
|
-
every cited sentence with citation markers and a claim marker, and to append a
|
|
66
|
-
machine-readable quote appendix:
|
|
53
|
+
The answering model gets the output of `veriquote prompt` (or
|
|
54
|
+
`buildCitationInstructions()`) in its system prompt. Every cited sentence then
|
|
55
|
+
ends with source and claim markers, and the answer ends with a plain-text quote
|
|
56
|
+
appendix:
|
|
67
57
|
|
|
68
58
|
```
|
|
69
59
|
Vitamin D supplementation reduced fall risk in older adults.[1]{c1}
|
|
@@ -76,51 +66,178 @@ c2|3|"BMD improved with \"high-dose\" regimens"
|
|
|
76
66
|
END_EVI1
|
|
77
67
|
```
|
|
78
68
|
|
|
79
|
-
The
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
69
|
+
The format is plain text rather than JSON, so it survives streaming, Markdown
|
|
70
|
+
renderers and weak models, and the `[n]` markers stay readable if nothing
|
|
71
|
+
checks them. Each citation then goes through three steps:
|
|
72
|
+
|
|
73
|
+
1. **Parse.** Every cited claim must carry a quote. Missing quotes are reported.
|
|
74
|
+
2. **Match.** Is the quote in the source? Deterministic fuzzy matching that
|
|
75
|
+
tolerates whitespace, typography, OCR noise and elisions, and reports where
|
|
76
|
+
the quote was found.
|
|
77
|
+
3. **Judge.** Does the quote support the claim? An LLM at temperature 0 answers
|
|
78
|
+
`entailed`, `partially_entailed`, `overstated`, `insufficient` or
|
|
79
|
+
`contradicted`, with a support score.
|
|
80
|
+
|
|
81
|
+
A citation's combined score is the lower of its match score and its support
|
|
82
|
+
score. A judge error can therefore never lift a citation above what the matcher
|
|
83
|
+
found, and a failed judge call is reported as an error, never counted as
|
|
84
|
+
support. Quoting does not make a model hallucinate less. It makes every claim
|
|
85
|
+
checkable, which is the point.
|
|
86
|
+
|
|
87
|
+
## Benchmarks
|
|
88
|
+
|
|
89
|
+
There are three benchmarks, kept separate on purpose: one blended number for a
|
|
90
|
+
two-stage pipeline would hide exactly the failures it exists to separate.
|
|
91
|
+
Everything is in [`bench/`](bench), including how to reproduce it.
|
|
92
|
+
|
|
93
|
+
<picture>
|
|
94
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/tango-dark.svg">
|
|
95
|
+
<img alt="The matcher blocks all 187 quotes that are not in the source and none of the 586 real quotes whose meaning was changed; the judge blocks 84% of unsupported citations and never sees the source; together they cover both kinds of failure." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/tango-light.svg">
|
|
96
|
+
</picture>
|
|
97
|
+
|
|
98
|
+
The two checks are blind in opposite places. The matcher never reads the claim,
|
|
99
|
+
so it cannot tell whether a real quote supports it. The judge never sees the
|
|
100
|
+
source, so an invented quote that fits the claim passes it: in the example
|
|
101
|
+
above, the judge rates the made-up quote in `c4` as "entailed 0.90".
|
|
102
|
+
|
|
103
|
+
### Matcher
|
|
104
|
+
|
|
105
|
+
<picture>
|
|
106
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/matcher-dark.svg">
|
|
107
|
+
<img alt="Matcher score ranges for fifteen kinds of quote damage: faithful quotes score between 0.66 and 1.0, quotes that are not in the source between 0.14 and 0.40, and quotes whose meaning was changed score high because they are still near-verbatim." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/matcher-light.svg">
|
|
108
|
+
</picture>
|
|
109
|
+
|
|
110
|
+
The matcher benchmark needs no API key and no labels, because the right answer
|
|
111
|
+
is known by construction: 2,061 quotes built from five Wikipedia articles, then
|
|
112
|
+
copied faithfully, reformatted, altered or replaced in fifteen different ways. `npm run bench:matcher` reproduces it in
|
|
113
|
+
about two seconds. Faithfully copied quotes never score below 0.660, quotes
|
|
114
|
+
that are not in the source never above 0.396, and the default threshold of 0.4
|
|
115
|
+
sits in that gap. The amber rows are there on purpose: a quote whose meaning was
|
|
116
|
+
changed is still near-verbatim, and catching it is the judge's job.
|
|
117
|
+
|
|
118
|
+
There are two honest limits. Invented prose built from the source's own words
|
|
119
|
+
(an adversarial case) scores high, and so can a hand-written fabrication that
|
|
120
|
+
reuses the source's vocabulary, like `c3` above at 0.48. The CLI therefore
|
|
121
|
+
fails every citation below 0.5 and leaves the rest to the judge.
|
|
122
|
+
|
|
123
|
+
### Judge
|
|
124
|
+
|
|
125
|
+
<picture>
|
|
126
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/judges-dark.svg">
|
|
127
|
+
<img alt="Five open judge models against the human labels of ALCE: they catch between 75% and 84% of unsupported citations, and agree with the annotators on 76.7% to 80.3% of pairs, around the 77.6% of the TRUE NLI model." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/judges-light.svg">
|
|
128
|
+
</picture>
|
|
129
|
+
|
|
130
|
+
The judge is measured against the human annotators of ALCE[^alce], with the
|
|
131
|
+
mapping from VeriQuote's classes to ALCE's labels fixed before any model ran.
|
|
132
|
+
*Caught* is the share of citations the annotators marked as unsupported that
|
|
133
|
+
the judge did not call fully supported. On ALCE's yes-or-no question ("does the
|
|
134
|
+
source fully support the claim?"), general-purpose open models agree with the
|
|
135
|
+
annotators about as often as TRUE[^true], the specialised 11B NLI model that
|
|
136
|
+
ALCE uses for its own automatic scores (77.6%). The best of them, `glm-5.3-flash`,
|
|
137
|
+
reaches Cohen's κ 0.53 on the three-way labels (full, partial or no support).
|
|
138
|
+
For comparison, ALCE reports κ 0.525 between its automatic metric and the
|
|
139
|
+
annotators on citation precision.
|
|
140
|
+
|
|
141
|
+
Before trusting any of this, look at the left panel: even the best judge lets
|
|
142
|
+
one unsupported citation in six through as fully supported. No judge should be
|
|
143
|
+
the only check. With only 56 unsupported pairs per run the intervals are wide,
|
|
144
|
+
so neighbouring models are not really separated. Reasoning helps: with it
|
|
145
|
+
switched off, all three hybrid models catch less and agree less, in exchange
|
|
146
|
+
for answering five to ten times faster.
|
|
147
|
+
|
|
148
|
+
### Answering models
|
|
149
|
+
|
|
150
|
+
<picture>
|
|
151
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/protocol-dark.svg">
|
|
152
|
+
<img alt="Protocol compliance for eight answering models: qwen3.6, qwen3.5, deepseek-v4-flash and glm-5.3-flash are close to 100% complete and verbatim; gpt-oss-120b is complete in 33.3% of answers; mistral-medium quotes verbatim in 81.0% of citations; llama-3.1-8b manages about half." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/protocol-light.svg">
|
|
153
|
+
</picture>
|
|
154
|
+
|
|
155
|
+
Each model answered 18 questions twice, with the citation instructions in its
|
|
156
|
+
system prompt, and `parseAnswer()` decided mechanically whether it complied.
|
|
157
|
+
Every raw answer is in
|
|
158
|
+
[`bench/results/protocol-answers.jsonl`](bench/results/protocol-answers.jsonl).
|
|
159
|
+
The open models recommended here comply almost perfectly. The failures of the
|
|
160
|
+
others are invisible to a reader: `gpt-oss-120b` almost always prints an
|
|
161
|
+
appendix, yet only a third of its answers give every citation a quote, and
|
|
162
|
+
`mistral-medium` is nearly always complete, yet a fifth of its "quotes" are
|
|
163
|
+
paraphrases. Both answers look impeccably cited. On the three questions the
|
|
164
|
+
sources cannot answer, no model attached a citation without a real quote.
|
|
165
|
+
|
|
166
|
+
<details>
|
|
167
|
+
<summary>The numbers behind the figures</summary>
|
|
168
|
+
|
|
169
|
+
| judge model | caught ↑ | agreement | Cohen's κ |
|
|
170
|
+
| --- | ---: | ---: | ---: |
|
|
171
|
+
| `glm-5.3-flash` | 83.9% | 80.3% | 0.529 |
|
|
172
|
+
| `deepseek-v4-flash` | 81.8% | 78.7% | 0.435 |
|
|
173
|
+
| `qwen3.6-35b-a3b` | 81.8% | 78.1% | 0.454 |
|
|
174
|
+
| `qwen3.5-397b-a17b` | 79.2% | 79.7% | 0.483 |
|
|
175
|
+
| `gpt-oss-120b` | 75.0% | 76.7% | 0.394 |
|
|
176
|
+
| `qwen3.5-397b-a17b`, no reasoning | 78.6% | 77.7% | 0.410 |
|
|
177
|
+
| `qwen3.6-35b-a3b`, no reasoning | 78.6% | 77.3% | 0.397 |
|
|
178
|
+
| `glm-5.3-flash`, no reasoning | 75.6% | 74.0% | 0.373 |
|
|
179
|
+
| TRUE (T5-11B), as reported by ALCE[^alce] | | 77.6% | |
|
|
180
|
+
|
|
181
|
+
| quote family | n | median score | accepted at 0.4 |
|
|
182
|
+
| --- | ---: | ---: | ---: |
|
|
183
|
+
| faithful but reformatted | 1,121 | 1.000 | 100.0% |
|
|
184
|
+
| near-verbatim, meaning changed | 586 | 0.932 | 100.0% |
|
|
185
|
+
| not in the source | 187 | 0.249 | 0.0% |
|
|
186
|
+
|
|
187
|
+
| answering model | appendix | complete | verbatim | warning-free |
|
|
188
|
+
| --- | ---: | ---: | ---: | ---: |
|
|
189
|
+
| qwen3.6-35b-a3b | 100.0% | 100.0% | 100.0% | 100.0% |
|
|
190
|
+
| qwen3.5-397b-a17b | 100.0% | 100.0% | 100.0% | 100.0% |
|
|
191
|
+
| deepseek-v4-flash | 100.0% | 100.0% | 99.2% | 93.3% |
|
|
192
|
+
| glm-5.3-flash | 96.6% | 96.6% | 99.6% | 96.6% |
|
|
193
|
+
| gemma-4-31b-it | 80.0% | 80.0% | 96.6% | 80.0% |
|
|
194
|
+
| mistral-medium-3.5-128b | 96.7% | 93.3% | 81.0% | 96.7% |
|
|
195
|
+
| gpt-oss-120b | 96.7% | 33.3% | 91.1% | 33.3% |
|
|
196
|
+
| llama-3.1-8b-instruct | 66.7% | 43.3% | 50.3% | 20.0% |
|
|
197
|
+
|
|
198
|
+
The figures are rendered from `bench/results/*.json` by
|
|
199
|
+
[`bench/figures/render.mjs`](bench/figures/render.mjs).
|
|
200
|
+
</details>
|
|
201
|
+
|
|
202
|
+
## Use from an agent
|
|
203
|
+
|
|
204
|
+
An agent that reads sources and writes conclusions is exactly the case
|
|
205
|
+
VeriQuote was built for. [`verify-citations`](integrations/verify-citations) is
|
|
206
|
+
an [Agent Skill](integrations/verify-citations/SKILL.md) that has the agent
|
|
207
|
+
write its sourced answer in the checkable format and run `veriquote check` on
|
|
208
|
+
it before showing it to you. The CLI fetches every cited URL itself, so the
|
|
209
|
+
agent cannot pass with its own truncated or misremembered copy of a page. The
|
|
210
|
+
exit code tells the agent what to do next without parsing anything:
|
|
84
211
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
- A quote can be **paraphrased or fabricated**: the claim may even be true,
|
|
91
|
-
but the "quote" is not in the source. The entailment judge might pass; the
|
|
92
|
-
deterministic matcher catches it, with a percent score that distinguishes
|
|
93
|
-
light paraphrase (high fuzzy score) from fabrication (low score).
|
|
212
|
+
```
|
|
213
|
+
exit 0 verdict "pass" every cited claim is grounded -> present the answer
|
|
214
|
+
exit 2 verdict "revise" problems[] + instructionsForModel -> fix and re-check
|
|
215
|
+
exit 1 bad input or unreachable source -> do NOT claim the answer was verified
|
|
216
|
+
```
|
|
94
217
|
|
|
95
|
-
The
|
|
96
|
-
|
|
218
|
+
The skill works in any host that reads Agent Skills, such as Claude Code, and
|
|
219
|
+
the CLI in anything that can run a shell command.
|
|
97
220
|
|
|
98
|
-
##
|
|
221
|
+
## Use as a library
|
|
99
222
|
|
|
100
223
|
```bash
|
|
101
224
|
npm install veriquote
|
|
102
225
|
```
|
|
103
226
|
|
|
104
|
-
## Quickstart
|
|
105
|
-
|
|
106
|
-
### 1. Prompt the answering model
|
|
107
|
-
|
|
108
227
|
```ts
|
|
109
228
|
import { buildCitationInstructions } from 'veriquote';
|
|
110
229
|
|
|
111
230
|
const systemPrompt = `${yourAssistantPrompt}\n\n${buildCitationInstructions()}`;
|
|
112
|
-
//
|
|
231
|
+
// Give the sources to the model as numbered blocks [1], [2], …
|
|
113
232
|
```
|
|
114
233
|
|
|
115
|
-
### 2. Verify the raw answer
|
|
116
|
-
|
|
117
234
|
```ts
|
|
118
235
|
import { ChatCompletionsJudge, verifyAnswer } from 'veriquote';
|
|
119
236
|
|
|
120
237
|
const judge = new ChatCompletionsJudge({
|
|
121
238
|
baseUrl: 'https://openrouter.ai/api/v1', // any OpenAI-compatible endpoint
|
|
122
239
|
apiKey: process.env.OPENROUTER_API_KEY, // server-side only!
|
|
123
|
-
model: '
|
|
240
|
+
model: 'your-judge-model', // pick one with bench/judge
|
|
124
241
|
});
|
|
125
242
|
|
|
126
243
|
const report = await verifyAnswer({
|
|
@@ -129,7 +246,7 @@ const report = await verifyAnswer({
|
|
|
129
246
|
{ title: 'Trial A', url: 'https://…', text: extractedFullText1 },
|
|
130
247
|
{ title: 'Trial B', url: 'https://…', text: extractedFullText2 },
|
|
131
248
|
],
|
|
132
|
-
judge, // omit
|
|
249
|
+
judge, // omit to check the quotes only
|
|
133
250
|
});
|
|
134
251
|
|
|
135
252
|
console.log(report.summary);
|
|
@@ -142,82 +259,99 @@ for (const c of report.citations) {
|
|
|
142
259
|
}
|
|
143
260
|
```
|
|
144
261
|
|
|
145
|
-
`report.cleanText` is the answer
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
Render each citation's `textMatch.score` (percent), `entailment.class`, and
|
|
151
|
-
combined `score` next to the footnote — e.g. green/yellow/red per claim. This
|
|
152
|
-
is exactly what the NavigNine UI does with tooltips and colored footnotes.
|
|
153
|
-
|
|
154
|
-
## API overview
|
|
262
|
+
`report.cleanText` is the answer without the `{cX}` markers, ready to render.
|
|
263
|
+
Show each citation's match score, judge class and combined score next to its
|
|
264
|
+
footnote, so a reader can see which sentence carries weight without opening a
|
|
265
|
+
single source.
|
|
155
266
|
|
|
156
267
|
| Export | Purpose |
|
|
157
268
|
| --- | --- |
|
|
158
|
-
| `buildCitationInstructions(options?)` | Prompt block for the answering model
|
|
159
|
-
| `verifyAnswer(options)` |
|
|
160
|
-
| `parseAnswer(answer)` |
|
|
161
|
-
| `parseEvi1Appendix` / `stripEvi1Appendix` / `serializeEvi1Appendix` | Low-level
|
|
162
|
-
| `matchQuoteAgainstSource(quote, source, options?)` |
|
|
163
|
-
| `ChatCompletionsJudge` |
|
|
164
|
-
| `EntailmentJudge` (interface) | Bring your own judge
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
| Class | Confidence band | Meaning |
|
|
269
|
+
| `buildCitationInstructions(options?)` | Prompt block for the answering model; budgets and quote lengths are configurable. |
|
|
270
|
+
| `verifyAnswer(options)` | The full pipeline: parse, match, judge, report. |
|
|
271
|
+
| `parseAnswer(answer)` | Claims, quotes and protocol warnings, without verifying anything. |
|
|
272
|
+
| `parseEvi1Appendix` / `stripEvi1Appendix` / `serializeEvi1Appendix` | Low-level handling of the appendix. |
|
|
273
|
+
| `matchQuoteAgainstSource(quote, source, options?)` | The deterministic matcher on its own. |
|
|
274
|
+
| `ChatCompletionsJudge` | Judge for any OpenAI-compatible API. |
|
|
275
|
+
| `EntailmentJudge` (interface) | Bring your own judge, such as a local NLI model. |
|
|
276
|
+
| `gateReport(report, answer)` | Pass or revise, the list of problems, uncited sentences and a correction prompt. |
|
|
277
|
+
| `fetchSource(url)` / `htmlToText(html)` | Fetch a source independently of the model and extract its text. |
|
|
278
|
+
|
|
279
|
+
All inputs and outputs are plain, serializable data. See
|
|
280
|
+
[`src/types.ts`](src/types.ts) for the data model and
|
|
281
|
+
[`docs/DESIGN.md`](docs/DESIGN.md) for scoring and thresholds.
|
|
282
|
+
|
|
283
|
+
| Class | Support score | Meaning |
|
|
174
284
|
| --- | --- | --- |
|
|
175
|
-
| `entailed` | 0.9–1.0 |
|
|
176
|
-
| `partially_entailed` | 0.5–0.8 |
|
|
177
|
-
| `overstated` | 0.3–0.6 |
|
|
178
|
-
| `insufficient` | 0.1–0.4 |
|
|
179
|
-
| `contradicted` | 0.0 |
|
|
180
|
-
| `error` |
|
|
285
|
+
| `entailed` | 0.9–1.0 | The quote fully covers the claim. |
|
|
286
|
+
| `partially_entailed` | 0.5–0.8 | The core is supported, details are missing. |
|
|
287
|
+
| `overstated` | 0.3–0.6 | The claim is stronger or more general than the quote. |
|
|
288
|
+
| `insufficient` | 0.1–0.4 | The quote is related but does not confirm the claim. |
|
|
289
|
+
| `contradicted` | 0.0 | The quote says the opposite. |
|
|
290
|
+
| `error` | none | The judge failed for this item; it is reported, never dropped. |
|
|
291
|
+
|
|
292
|
+
## How this differs from the alternatives
|
|
293
|
+
|
|
294
|
+
| | verbatim quote checked | claim↔evidence checked | model-agnostic | runtime |
|
|
295
|
+
| --- | --- | --- | --- | --- |
|
|
296
|
+
| **VeriQuote** | yes, deterministic | yes, pluggable judge | yes | TypeScript, no dependencies, browser and edge |
|
|
297
|
+
| [Anthropic Citations API](https://platform.claude.com/docs/en/build-with-claude/citations) | not needed: spans are extracted, so they are real by construction | no | Claude only | hosted |
|
|
298
|
+
| [LettuceDetect](https://github.com/KRLabsOrg/LettuceDetect) | no quote protocol | yes, span-level model | yes | Python and model weights |
|
|
299
|
+
| [RAGAS](https://github.com/explodinggradients/ragas) and eval frameworks | no | yes, as an offline metric | yes | Python, offline evaluation |
|
|
300
|
+
|
|
301
|
+
**Use the Citations API instead** if you are on Claude and only need to know
|
|
302
|
+
that a span is real. It guarantees that by construction, which is stronger than
|
|
303
|
+
any matcher. It does not tell you whether the span supports the sentence built
|
|
304
|
+
on it; for that, pair it with VeriQuote's judge and skip the matcher.
|
|
305
|
+
|
|
306
|
+
**Use LettuceDetect instead** if you want unsupported spans flagged in an
|
|
307
|
+
answer that follows no citation protocol at all, and you are happy to run a
|
|
308
|
+
model in Python. It solves the problem after the fact; VeriQuote changes what
|
|
309
|
+
the answering model commits to in the first place.
|
|
310
|
+
|
|
311
|
+
VeriQuote's own niche is narrow, and worth stating plainly: you want the
|
|
312
|
+
answering model pinned to a quote *before* it writes, you want the
|
|
313
|
+
deterministic half of the check to run anywhere, including a browser, and you
|
|
314
|
+
want per-claim results you can show a reader rather than one score on a
|
|
315
|
+
dashboard.
|
|
316
|
+
|
|
317
|
+
The closest related work is a recent study by Zhang et al.[^zhang], which
|
|
318
|
+
evaluates the same design (inline verbatim quotes, tiered verbatim matching and
|
|
319
|
+
an LLM judge) on clinical guidelines. It found that `claude-opus-5` quoted
|
|
320
|
+
verbatim for 98.0% of its claims but fully substantiated only 37.1% of them.
|
|
321
|
+
VeriQuote is the reusable library, CLI and agent skill for that kind of check,
|
|
322
|
+
with its judge measured against human labels.
|
|
181
323
|
|
|
182
324
|
## Security
|
|
183
325
|
|
|
184
|
-
- **Keep the
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
- **
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
hallucinated item IDs are rejected.
|
|
192
|
-
- **No dynamic evaluation.** Tolerant JSON recovery is a string-aware scanner;
|
|
193
|
-
nothing is ever `eval`ed.
|
|
194
|
-
- **Failure transparency.** Judge failures degrade to `class: "error"` with a
|
|
195
|
-
`null` score — they are reported, never counted as "supported".
|
|
326
|
+
- **Keep your key on the server.** `ChatCompletionsJudge` needs an API key; in
|
|
327
|
+
your own app, call `verifyAnswer` from a backend. The demo runs the judge in
|
|
328
|
+
the browser only with a key the visitor enters.
|
|
329
|
+
- **Source text is untrusted.** Judge inputs are length-capped, stripped of
|
|
330
|
+
control characters and HTML, and marked as data in the prompt. The judge's
|
|
331
|
+
output is checked against a closed vocabulary: unknown classes, out-of-range
|
|
332
|
+
scores and invented item IDs are rejected. Nothing is ever `eval`ed.
|
|
196
333
|
|
|
197
334
|
## Reproducibility
|
|
198
335
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
## Integrations
|
|
206
|
-
|
|
207
|
-
- **[`verify-citations`](integrations/verify-citations/)**: a portable
|
|
208
|
-
[Agent Skill](integrations/verify-citations/SKILL.md) (single `SKILL.md` +
|
|
209
|
-
bundled Node CLI) that runs VeriQuote as an **internal hallucination gate**
|
|
210
|
-
for source-grounded agents: it verifies a cited answer, flags factual
|
|
211
|
-
sentences that carry no citation, and returns a ready-to-use correction
|
|
212
|
-
prompt for a self-correction loop. The same skill works across any
|
|
213
|
-
Agent-Skills host (OpenClaw, Hermes Agent, Claude Code) and any orchestrator
|
|
214
|
-
that can run a Node CLI.
|
|
336
|
+
The matcher is pure: the same inputs always give the same score. The judge
|
|
337
|
+
runs at temperature 0 (pass `seed` where the provider supports it), but hosted
|
|
338
|
+
models are only deterministic on a best-effort basis. For strict
|
|
339
|
+
reproducibility, pin the model version or put a self-hosted model behind the
|
|
340
|
+
`EntailmentJudge` interface.
|
|
215
341
|
|
|
216
342
|
## Citing
|
|
217
343
|
|
|
218
|
-
If you use VeriQuote in academic work, please cite the
|
|
219
|
-
`CITATION.cff`).
|
|
344
|
+
If you use VeriQuote in academic work, please cite it[^veriquote]; the details
|
|
345
|
+
are also in [`CITATION.cff`](CITATION.cff).
|
|
220
346
|
|
|
221
347
|
## License
|
|
222
348
|
|
|
223
349
|
[MIT](LICENSE)
|
|
350
|
+
|
|
351
|
+
[^alce]: Gao, T., Yen, H., Yu, J., & Chen, D. (2023). Enabling large language models to generate text with citations. In *Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing* (pp. 6465–6488). Association for Computational Linguistics. https://doi.org/10.18653/v1/2023.emnlp-main.398
|
|
352
|
+
|
|
353
|
+
[^true]: Honovich, O., Aharoni, R., Herzig, J., Taitelbaum, H., Kukliansy, D., Cohen, V., Scialom, T., Szpektor, I., Hassidim, A., & Matias, Y. (2022). TRUE: Re-evaluating factual consistency evaluation. In *Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies* (pp. 3905–3920). Association for Computational Linguistics. https://doi.org/10.18653/v1/2022.naacl-main.287
|
|
354
|
+
|
|
355
|
+
[^zhang]: Zhang, J., Chen, Y., Commodore-Mensah, Y., & Oberst, M. (2026). *Verifiable by construction: Claim-level evaluation of verbatim citation in clinical question answering* (Version 2) [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2609.15964
|
|
356
|
+
|
|
357
|
+
[^veriquote]: Heilmann, E. (2026). *VeriQuote: Deterministic and semantic verification of quote-grounded LLM citations* (Version 0.2.1) [Computer software]. Zenodo. https://doi.org/10.5281/zenodo.21552379
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Thin process wrapper; the command logic lives in src/cli/main.ts.
|
|
3
|
+
import { readFile } from 'node:fs/promises';
|
|
4
|
+
import { main } from '../dist/cli/main.js';
|
|
5
|
+
|
|
6
|
+
async function readStdin() {
|
|
7
|
+
let data = '';
|
|
8
|
+
process.stdin.setEncoding('utf8');
|
|
9
|
+
for await (const chunk of process.stdin) data += chunk;
|
|
10
|
+
return data;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const pkg = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
|
|
14
|
+
|
|
15
|
+
process.exitCode = await main({
|
|
16
|
+
argv: process.argv.slice(2),
|
|
17
|
+
env: process.env,
|
|
18
|
+
stdout: (s) => process.stdout.write(s),
|
|
19
|
+
stderr: (s) => process.stderr.write(s),
|
|
20
|
+
readFile: (path) => readFile(path, 'utf8'),
|
|
21
|
+
readStdin,
|
|
22
|
+
version: pkg.version,
|
|
23
|
+
color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR,
|
|
24
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `veriquote` command line. Kept free of `process` so it can be tested with
|
|
3
|
+
* injected I/O; `bin/veriquote.mjs` wires it to the real process.
|
|
4
|
+
*/
|
|
5
|
+
import type { EntailmentJudge } from '../types.js';
|
|
6
|
+
export interface CliIo {
|
|
7
|
+
argv: string[];
|
|
8
|
+
env: Record<string, string | undefined>;
|
|
9
|
+
stdout: (s: string) => void;
|
|
10
|
+
stderr: (s: string) => void;
|
|
11
|
+
readFile: (path: string) => Promise<string>;
|
|
12
|
+
readStdin: () => Promise<string>;
|
|
13
|
+
fetch?: typeof globalThis.fetch;
|
|
14
|
+
version: string;
|
|
15
|
+
/** Use ANSI colors in human output. */
|
|
16
|
+
color: boolean;
|
|
17
|
+
/** Test hook: replaces the env-configured judge. */
|
|
18
|
+
judge?: EntailmentJudge;
|
|
19
|
+
}
|
|
20
|
+
export declare const EXIT: {
|
|
21
|
+
readonly pass: 0;
|
|
22
|
+
readonly error: 1;
|
|
23
|
+
readonly revise: 2;
|
|
24
|
+
};
|
|
25
|
+
export declare function main(io: CliIo): Promise<number>;
|
|
26
|
+
//# sourceMappingURL=main.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAsC,MAAM,aAAa,CAAC;AAOvF,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACxC,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,uCAAuC;IACvC,KAAK,EAAE,OAAO,CAAC;IACf,oDAAoD;IACpD,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,eAAO,MAAM,IAAI;;;;CAA4C,CAAC;AAkG9D,wBAAsB,IAAI,CAAC,EAAE,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAgCrD"}
|