veriquote 0.1.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CITATION.cff CHANGED
@@ -9,11 +9,12 @@ authors:
9
9
  orcid: "https://orcid.org/0009-0009-5596-736X"
10
10
  repository-code: "https://github.com/rickintoplace/veriquote"
11
11
  license: MIT
12
- version: 0.1.1
12
+ version: 0.2.1
13
13
  identifiers:
14
14
  - type: doi
15
- value: 10.5281/zenodo.21552380
16
- date-released: "2026-07-25"
15
+ value: 10.5281/zenodo.21552379
16
+ description: Concept DOI, resolves to the latest version
17
+ date-released: "2026-09-22"
17
18
  keywords:
18
19
  - large language models
19
20
  - retrieval-augmented generation
package/README.md CHANGED
@@ -1,69 +1,59 @@
1
1
  # VeriQuote
2
2
 
3
- **Deterministic + semantic verification of quote-grounded LLM citations.**
3
+ [![npm](https://img.shields.io/npm/v/veriquote)](https://www.npmjs.com/package/veriquote)
4
4
  [![DOI](https://zenodo.org/badge/1311867832.svg)](https://doi.org/10.5281/zenodo.21552379)
5
5
 
6
- VeriQuote makes source-grounded assistant answers *auditable*. Instead of
7
- trusting that a `[1]` citation means anything, the answering model must attach
8
- a **verbatim quote** for every cited claim, and VeriQuote then checks per
9
- claim whether:
10
-
11
- 1. the quote **actually occurs in the source** (deterministic fuzzy text
12
- matching with a percent score), and
13
- 2. the quote **actually supports the claim** (a small, temperature-0 LLM judge
14
- classifying entailment strength).
15
-
16
- The result is a transparent, per-citation report telling users exactly which
17
- statements are verbatim-backed and supported, which are overstated, and which
18
- are unsupported or confabulated.
19
-
20
- VeriQuote is extracted from and battle-tested in
21
- [NavigNine](https://navignine.com), a source-grounded research assistant,
22
- which serves as the reference deployment.
6
+ **Make an LLM quote its sources, then check every quote twice: is it really in
7
+ the source, and does it support the claim?**
8
+
9
+ A `[1]` after a sentence looks like evidence, but almost nobody checks it. In
10
+ my tests, a fifth of one capable model's "verbatim" quotes were not in the
11
+ source, another model left two thirds of its cited answers with at least one
12
+ citation that had nothing behind it, and an LLM judge happily confirmed a quote
13
+ that was invented. VeriQuote makes the answering model commit to a verbatim
14
+ quote for every citation, then verifies each one: deterministically where
15
+ possible, with an LLM only where it has to. It is written in TypeScript, has no
16
+ dependencies and runs in Node and in the browser.
17
+
18
+ **Try it in the browser:** [rickinto.place/veriquote](https://rickinto.place/veriquote)
19
+ has eight worked examples, a box for your own text, and the benchmark results as
20
+ interactive charts.
21
+
22
+ ```console
23
+ $ curl -s https://raw.githubusercontent.com/rickintoplace/veriquote/main/examples/ozone-answer.md \
24
+ | npx veriquote check - --source https://en.wikipedia.org/wiki/Ozone_layer
25
+
26
+ 4 citation(s) · 1 source(s) · judge: glm-5.3-flash
27
+
28
+ ✓ c1 [1] verbatim 1.00 · entailed 1.00
29
+ The ozone layer absorbs 97 to 99 percent of the Sun's medium-frequency ultraviolet light.
30
+ ✗ c2 [1] verbatim 1.00 · contradicted 0.00
31
+ It was discovered in 1913 by the British meteorologist G. M. B. Dobson.
32
+ Quote credits Fabry and Buisson, not Dobson
33
+ ✗ c3 [1] fuzzy 0.48 · contradicted 0.00
34
+ Under the Montreal Protocol, all CFC production was banned immediately in 1987.
35
+ Context says production capped at 1986 levels, not banned; Quote absent from context; appears fabricated
36
+ ✗ c4 [1] quote not in source (best 0.31) · entailed 0.90
37
+ The treaty limited CFC production to the levels of 1986.
38
+ the quoted text does not occur in the source
39
+ ! uncited Ozone depletion has since been fully reversed in every region of the atmosphere.
40
+
41
+ REVISE — 3 failed citation(s), 1 uncited sentence(s)
42
+ ```
23
43
 
24
- - **Zero runtime dependencies.** Runs in Node 18, browsers, and edge runtimes.
25
- - **Deterministic by construction.** The text matcher is pure; the judge runs
26
- at temperature 0 with a closed class vocabulary and strict output validation.
27
- - **Model-agnostic.** Works with any answering model and any OpenAI-compatible
28
- chat-completions endpoint for the judge (OpenAI, OpenRouter, Azure, local
29
- gateways) or bring your own `EntailmentJudge` (e.g. a local NLI model).
44
+ Look at `c2` and `c4`. The quote in `c2` is copied correctly, but the claim
45
+ names the wrong person; only the judge can see that. The quote in `c4` is made
46
+ up, and the judge still calls it supported; only the matcher can see that.
47
+ Without an API key the CLI checks the quotes alone. Set
48
+ `VERIQUOTE_JUDGE_API_KEY` and `VERIQUOTE_JUDGE_MODEL` (any OpenAI-compatible
49
+ endpoint) to add the judge.
30
50
 
31
51
  ## How it works
32
52
 
33
- ```
34
- ┌───────────────────────────┐
35
- numbered sources ───► │ Answering LLM │
36
- + citation prompt │ (any model) │
37
- └────────────┬──────────────┘
38
- │ answer body with [n]{cX} markers
39
- │ + EVI1 quote appendix
40
-
41
- ┌───────────────────────────┐
42
- │ 1. parseAnswer() │ claims, quotes, protocol
43
- │ (deterministic) │ completeness warnings
44
- └────────────┬──────────────┘
45
-
46
- ┌───────────────────────────┐
47
- │ 2. Quote ↔ source match │ exact / normalized /
48
- │ (deterministic, fuzzy) │ fuzzy %, offsets
49
- └────────────┬──────────────┘
50
-
51
- ┌───────────────────────────┐
52
- │ 3. Entailment judge │ entailed / partially /
53
- │ (LLM, temp 0, optional)│ overstated / insufficient
54
- └────────────┬──────────────┘ / contradicted + conf.
55
-
56
- ┌───────────────────────────┐
57
- │ 4. VerificationReport │ per-citation scores +
58
- │ (transparency for user)│ answer-level summary
59
- └───────────────────────────┘
60
- ```
61
-
62
- ### The EVI1 protocol
63
-
64
- The answering model is instructed (via `buildCitationInstructions()`) to end
65
- every cited sentence with citation markers and a claim marker, and to append a
66
- machine-readable quote appendix:
53
+ The answering model gets the output of `veriquote prompt` (or
54
+ `buildCitationInstructions()`) in its system prompt. Every cited sentence then
55
+ ends with source and claim markers, and the answer ends with a plain-text quote
56
+ appendix:
67
57
 
68
58
  ```
69
59
  Vitamin D supplementation reduced fall risk in older adults.[1]{c1}
@@ -76,51 +66,178 @@ c2|3|"BMD improved with \"high-dose\" regimens"
76
66
  END_EVI1
77
67
  ```
78
68
 
79
- The protocol is intentionally plain text (not JSON): it survives streaming,
80
- markdown renderers, and weak models. And `[n]` citations remain human-readable
81
- even if a client ignores VeriQuote entirely.
82
-
83
- ### Why two checks?
69
+ The format is plain text rather than JSON, so it survives streaming, Markdown
70
+ renderers and weak models, and the `[n]` markers stay readable if nothing
71
+ checks them. Each citation then goes through three steps:
72
+
73
+ 1. **Parse.** Every cited claim must carry a quote. Missing quotes are reported.
74
+ 2. **Match.** Is the quote in the source? Deterministic fuzzy matching that
75
+ tolerates whitespace, typography, OCR noise and elisions, and reports where
76
+ the quote was found.
77
+ 3. **Judge.** Does the quote support the claim? An LLM at temperature 0 answers
78
+ `entailed`, `partially_entailed`, `overstated`, `insufficient` or
79
+ `contradicted`, with a support score.
80
+
81
+ A citation's combined score is the lower of its match score and its support
82
+ score. A judge error can therefore never lift a citation above what the matcher
83
+ found, and a failed judge call is reported as an error, never counted as
84
+ support. Quoting does not make a model hallucinate less. It makes every claim
85
+ checkable, which is the point.
86
+
87
+ ## Benchmarks
88
+
89
+ There are three benchmarks, kept separate on purpose: one blended number for a
90
+ two-stage pipeline would hide exactly the failures it exists to separate.
91
+ Everything is in [`bench/`](bench), including how to reproduce it.
92
+
93
+ <picture>
94
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/tango-dark.svg">
95
+ <img alt="The matcher blocks all 187 quotes that are not in the source and none of the 586 real quotes whose meaning was changed; the judge blocks 84% of unsupported citations and never sees the source; together they cover both kinds of failure." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/tango-light.svg">
96
+ </picture>
97
+
98
+ The two checks are blind in opposite places. The matcher never reads the claim,
99
+ so it cannot tell whether a real quote supports it. The judge never sees the
100
+ source, so an invented quote that fits the claim passes it: in the example
101
+ above, the judge rates the made-up quote in `c4` as "entailed 0.90".
102
+
103
+ ### Matcher
104
+
105
+ <picture>
106
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/matcher-dark.svg">
107
+ <img alt="Matcher score ranges for fifteen kinds of quote damage: faithful quotes score between 0.66 and 1.0, quotes that are not in the source between 0.14 and 0.40, and quotes whose meaning was changed score high because they are still near-verbatim." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/matcher-light.svg">
108
+ </picture>
109
+
110
+ The matcher benchmark needs no API key and no labels, because the right answer
111
+ is known by construction: 2,061 quotes built from five Wikipedia articles, then
112
+ copied faithfully, reformatted, altered or replaced in fifteen different ways. `npm run bench:matcher` reproduces it in
113
+ about two seconds. Faithfully copied quotes never score below 0.660, quotes
114
+ that are not in the source never above 0.396, and the default threshold of 0.4
115
+ sits in that gap. The amber rows are there on purpose: a quote whose meaning was
116
+ changed is still near-verbatim, and catching it is the judge's job.
117
+
118
+ There are two honest limits. Invented prose built from the source's own words
119
+ (an adversarial case) scores high, and so can a hand-written fabrication that
120
+ reuses the source's vocabulary, like `c3` above at 0.48. The CLI therefore
121
+ fails every citation below 0.5 and leaves the rest to the judge.
122
+
123
+ ### Judge
124
+
125
+ <picture>
126
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/judges-dark.svg">
127
+ <img alt="Five open judge models against the human labels of ALCE: they catch between 75% and 84% of unsupported citations, and agree with the annotators on 76.7% to 80.3% of pairs, around the 77.6% of the TRUE NLI model." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/judges-light.svg">
128
+ </picture>
129
+
130
+ The judge is measured against the human annotators of ALCE[^alce], with the
131
+ mapping from VeriQuote's classes to ALCE's labels fixed before any model ran.
132
+ *Caught* is the share of citations the annotators marked as unsupported that
133
+ the judge did not call fully supported. On ALCE's yes-or-no question ("does the
134
+ source fully support the claim?"), general-purpose open models agree with the
135
+ annotators about as often as TRUE[^true], the specialised 11B NLI model that
136
+ ALCE uses for its own automatic scores (77.6%). The best of them, `glm-5.3-flash`,
137
+ reaches Cohen's κ 0.53 on the three-way labels (full, partial or no support).
138
+ For comparison, ALCE reports κ 0.525 between its automatic metric and the
139
+ annotators on citation precision.
140
+
141
+ Before trusting any of this, look at the left panel: even the best judge lets
142
+ one unsupported citation in six through as fully supported. No judge should be
143
+ the only check. With only 56 unsupported pairs per run the intervals are wide,
144
+ so neighbouring models are not really separated. Reasoning helps: with it
145
+ switched off, all three hybrid models catch less and agree less, in exchange
146
+ for answering five to ten times faster.
147
+
148
+ ### Answering models
149
+
150
+ <picture>
151
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/protocol-dark.svg">
152
+ <img alt="Protocol compliance for eight answering models: qwen3.6, qwen3.5, deepseek-v4-flash and glm-5.3-flash are close to 100% complete and verbatim; gpt-oss-120b is complete in 33.3% of answers; mistral-medium quotes verbatim in 81.0% of citations; llama-3.1-8b manages about half." src="https://raw.githubusercontent.com/rickintoplace/veriquote/main/bench/figures/protocol-light.svg">
153
+ </picture>
154
+
155
+ Each model answered 18 questions twice, with the citation instructions in its
156
+ system prompt, and `parseAnswer()` decided mechanically whether it complied.
157
+ Every raw answer is in
158
+ [`bench/results/protocol-answers.jsonl`](bench/results/protocol-answers.jsonl).
159
+ The open models recommended here comply almost perfectly. The failures of the
160
+ others are invisible to a reader: `gpt-oss-120b` almost always prints an
161
+ appendix, yet only a third of its answers give every citation a quote, and
162
+ `mistral-medium` is nearly always complete, yet a fifth of its "quotes" are
163
+ paraphrases. Both answers look impeccably cited. On the three questions the
164
+ sources cannot answer, no model attached a citation without a real quote.
165
+
166
+ <details>
167
+ <summary>The numbers behind the figures</summary>
168
+
169
+ | judge model | caught ↑ | agreement | Cohen's κ |
170
+ | --- | ---: | ---: | ---: |
171
+ | `glm-5.3-flash` | 83.9% | 80.3% | 0.529 |
172
+ | `deepseek-v4-flash` | 81.8% | 78.7% | 0.435 |
173
+ | `qwen3.6-35b-a3b` | 81.8% | 78.1% | 0.454 |
174
+ | `qwen3.5-397b-a17b` | 79.2% | 79.7% | 0.483 |
175
+ | `gpt-oss-120b` | 75.0% | 76.7% | 0.394 |
176
+ | `qwen3.5-397b-a17b`, no reasoning | 78.6% | 77.7% | 0.410 |
177
+ | `qwen3.6-35b-a3b`, no reasoning | 78.6% | 77.3% | 0.397 |
178
+ | `glm-5.3-flash`, no reasoning | 75.6% | 74.0% | 0.373 |
179
+ | TRUE (T5-11B), as reported by ALCE[^alce] | | 77.6% | |
180
+
181
+ | quote family | n | median score | accepted at 0.4 |
182
+ | --- | ---: | ---: | ---: |
183
+ | faithful but reformatted | 1,121 | 1.000 | 100.0% |
184
+ | near-verbatim, meaning changed | 586 | 0.932 | 100.0% |
185
+ | not in the source | 187 | 0.249 | 0.0% |
186
+
187
+ | answering model | appendix | complete | verbatim | warning-free |
188
+ | --- | ---: | ---: | ---: | ---: |
189
+ | qwen3.6-35b-a3b | 100.0% | 100.0% | 100.0% | 100.0% |
190
+ | qwen3.5-397b-a17b | 100.0% | 100.0% | 100.0% | 100.0% |
191
+ | deepseek-v4-flash | 100.0% | 100.0% | 99.2% | 93.3% |
192
+ | glm-5.3-flash | 96.6% | 96.6% | 99.6% | 96.6% |
193
+ | gemma-4-31b-it | 80.0% | 80.0% | 96.6% | 80.0% |
194
+ | mistral-medium-3.5-128b | 96.7% | 93.3% | 81.0% | 96.7% |
195
+ | gpt-oss-120b | 96.7% | 33.3% | 91.1% | 33.3% |
196
+ | llama-3.1-8b-instruct | 66.7% | 43.3% | 50.3% | 20.0% |
197
+
198
+ The figures are rendered from `bench/results/*.json` by
199
+ [`bench/figures/render.mjs`](bench/figures/render.mjs).
200
+ </details>
201
+
202
+ ## Use from an agent
203
+
204
+ An agent that reads sources and writes conclusions is exactly the case
205
+ VeriQuote was built for. [`verify-citations`](integrations/verify-citations) is
206
+ an [Agent Skill](integrations/verify-citations/SKILL.md) that has the agent
207
+ write its sourced answer in the checkable format and run `veriquote check` on
208
+ it before showing it to you. The CLI fetches every cited URL itself, so the
209
+ agent cannot pass with its own truncated or misremembered copy of a page. The
210
+ exit code tells the agent what to do next without parsing anything:
84
211
 
85
- The two checks fail independently, and both failure modes occur in practice:
86
-
87
- - A quote can be **verbatim yet irrelevant**: the model copied real text that
88
- doesn't support its claim (scope drift, outcome switching, overstatement).
89
- Text match passes; the entailment judge catches it.
90
- - A quote can be **paraphrased or fabricated**: the claim may even be true,
91
- but the "quote" is not in the source. The entailment judge might pass; the
92
- deterministic matcher catches it, with a percent score that distinguishes
93
- light paraphrase (high fuzzy score) from fabrication (low score).
212
+ ```
213
+ exit 0 verdict "pass" every cited claim is grounded -> present the answer
214
+ exit 2 verdict "revise" problems[] + instructionsForModel -> fix and re-check
215
+ exit 1 bad input or unreachable source -> do NOT claim the answer was verified
216
+ ```
94
217
 
95
- The combined per-citation score is conservative:
96
- `min(textMatchScore, judgeConfidence)`.
218
+ The skill works in any host that reads Agent Skills, such as Claude Code, and
219
+ the CLI in anything that can run a shell command.
97
220
 
98
- ## Installation
221
+ ## Use as a library
99
222
 
100
223
  ```bash
101
224
  npm install veriquote
102
225
  ```
103
226
 
104
- ## Quickstart
105
-
106
- ### 1. Prompt the answering model
107
-
108
227
  ```ts
109
228
  import { buildCitationInstructions } from 'veriquote';
110
229
 
111
230
  const systemPrompt = `${yourAssistantPrompt}\n\n${buildCitationInstructions()}`;
112
- // Provide sources as numbered blocks [1], [2], ... in the user/context prompt.
231
+ // Give the sources to the model as numbered blocks [1], [2],
113
232
  ```
114
233
 
115
- ### 2. Verify the raw answer
116
-
117
234
  ```ts
118
235
  import { ChatCompletionsJudge, verifyAnswer } from 'veriquote';
119
236
 
120
237
  const judge = new ChatCompletionsJudge({
121
238
  baseUrl: 'https://openrouter.ai/api/v1', // any OpenAI-compatible endpoint
122
239
  apiKey: process.env.OPENROUTER_API_KEY, // server-side only!
123
- model: 'google/gemini-2.5-flash-lite',
240
+ model: 'your-judge-model', // pick one with bench/judge
124
241
  });
125
242
 
126
243
  const report = await verifyAnswer({
@@ -129,7 +246,7 @@ const report = await verifyAnswer({
129
246
  { title: 'Trial A', url: 'https://…', text: extractedFullText1 },
130
247
  { title: 'Trial B', url: 'https://…', text: extractedFullText2 },
131
248
  ],
132
- judge, // omit for text-match-only verification
249
+ judge, // omit to check the quotes only
133
250
  });
134
251
 
135
252
  console.log(report.summary);
@@ -142,82 +259,99 @@ for (const c of report.citations) {
142
259
  }
143
260
  ```
144
261
 
145
- `report.cleanText` is the answer with all `{cX}` markers removed, ready to
146
- render (the `[n]` markers remain as human-readable citations).
147
-
148
- ### 3. Show it to the user
149
-
150
- Render each citation's `textMatch.score` (percent), `entailment.class`, and
151
- combined `score` next to the footnote — e.g. green/yellow/red per claim. This
152
- is exactly what the NavigNine UI does with tooltips and colored footnotes.
153
-
154
- ## API overview
262
+ `report.cleanText` is the answer without the `{cX}` markers, ready to render.
263
+ Show each citation's match score, judge class and combined score next to its
264
+ footnote, so a reader can see which sentence carries weight without opening a
265
+ single source.
155
266
 
156
267
  | Export | Purpose |
157
268
  | --- | --- |
158
- | `buildCitationInstructions(options?)` | Prompt block for the answering model (budgets and quote-length rules configurable). |
159
- | `verifyAnswer(options)` | Full pipeline: parse match judge report. |
160
- | `parseAnswer(answer)` | Parse claims, evidence, and protocol warnings without verifying. |
161
- | `parseEvi1Appendix` / `stripEvi1Appendix` / `serializeEvi1Appendix` | Low-level EVI1 handling. |
162
- | `matchQuoteAgainstSource(quote, source, options?)` | Deterministic quote matching on its own. |
163
- | `ChatCompletionsJudge` | Entailment judge for any OpenAI-compatible API. |
164
- | `EntailmentJudge` (interface) | Bring your own judge (local NLI model, other provider). |
165
-
166
- All inputs and outputs are plain, serializable data see
167
- [`src/types.ts`](src/types.ts) for the complete, documented data model and
168
- [`docs/DESIGN.md`](docs/DESIGN.md) for the method description (scoring,
169
- thresholds, and design rationale).
170
-
171
- ### Entailment classes
172
-
173
- | Class | Confidence band | Meaning |
269
+ | `buildCitationInstructions(options?)` | Prompt block for the answering model; budgets and quote lengths are configurable. |
270
+ | `verifyAnswer(options)` | The full pipeline: parse, match, judge, report. |
271
+ | `parseAnswer(answer)` | Claims, quotes and protocol warnings, without verifying anything. |
272
+ | `parseEvi1Appendix` / `stripEvi1Appendix` / `serializeEvi1Appendix` | Low-level handling of the appendix. |
273
+ | `matchQuoteAgainstSource(quote, source, options?)` | The deterministic matcher on its own. |
274
+ | `ChatCompletionsJudge` | Judge for any OpenAI-compatible API. |
275
+ | `EntailmentJudge` (interface) | Bring your own judge, such as a local NLI model. |
276
+ | `gateReport(report, answer)` | Pass or revise, the list of problems, uncited sentences and a correction prompt. |
277
+ | `fetchSource(url)` / `htmlToText(html)` | Fetch a source independently of the model and extract its text. |
278
+
279
+ All inputs and outputs are plain, serializable data. See
280
+ [`src/types.ts`](src/types.ts) for the data model and
281
+ [`docs/DESIGN.md`](docs/DESIGN.md) for scoring and thresholds.
282
+
283
+ | Class | Support score | Meaning |
174
284
  | --- | --- | --- |
175
- | `entailed` | 0.9–1.0 | Claim fully covered by the quote. |
176
- | `partially_entailed` | 0.5–0.8 | Core message supported, details missing. |
177
- | `overstated` | 0.3–0.6 | Claim stronger/more general than the evidence. |
178
- | `insufficient` | 0.1–0.4 | Related but does not confirm the claim. |
179
- | `contradicted` | 0.0 | Evidence says the opposite. |
180
- | `error` | | Judge unavailable for this item (never silently dropped). |
285
+ | `entailed` | 0.9–1.0 | The quote fully covers the claim. |
286
+ | `partially_entailed` | 0.5–0.8 | The core is supported, details are missing. |
287
+ | `overstated` | 0.3–0.6 | The claim is stronger or more general than the quote. |
288
+ | `insufficient` | 0.1–0.4 | The quote is related but does not confirm the claim. |
289
+ | `contradicted` | 0.0 | The quote says the opposite. |
290
+ | `error` | none | The judge failed for this item; it is reported, never dropped. |
291
+
292
+ ## How this differs from the alternatives
293
+
294
+ | | verbatim quote checked | claim↔evidence checked | model-agnostic | runtime |
295
+ | --- | --- | --- | --- | --- |
296
+ | **VeriQuote** | yes, deterministic | yes, pluggable judge | yes | TypeScript, no dependencies, browser and edge |
297
+ | [Anthropic Citations API](https://platform.claude.com/docs/en/build-with-claude/citations) | not needed: spans are extracted, so they are real by construction | no | Claude only | hosted |
298
+ | [LettuceDetect](https://github.com/KRLabsOrg/LettuceDetect) | no quote protocol | yes, span-level model | yes | Python and model weights |
299
+ | [RAGAS](https://github.com/explodinggradients/ragas) and eval frameworks | no | yes, as an offline metric | yes | Python, offline evaluation |
300
+
301
+ **Use the Citations API instead** if you are on Claude and only need to know
302
+ that a span is real. It guarantees that by construction, which is stronger than
303
+ any matcher. It does not tell you whether the span supports the sentence built
304
+ on it; for that, pair it with VeriQuote's judge and skip the matcher.
305
+
306
+ **Use LettuceDetect instead** if you want unsupported spans flagged in an
307
+ answer that follows no citation protocol at all, and you are happy to run a
308
+ model in Python. It solves the problem after the fact; VeriQuote changes what
309
+ the answering model commits to in the first place.
310
+
311
+ VeriQuote's own niche is narrow, and worth stating plainly: you want the
312
+ answering model pinned to a quote *before* it writes, you want the
313
+ deterministic half of the check to run anywhere, including a browser, and you
314
+ want per-claim results you can show a reader rather than one score on a
315
+ dashboard.
316
+
317
+ The closest related work is a recent study by Zhang et al.[^zhang], which
318
+ evaluates the same design (inline verbatim quotes, tiered verbatim matching and
319
+ an LLM judge) on clinical guidelines. It found that `claude-opus-5` quoted
320
+ verbatim for 98.0% of its claims but fully substantiated only 37.1% of them.
321
+ VeriQuote is the reusable library, CLI and agent skill for that kind of check,
322
+ with its judge measured against human labels.
181
323
 
182
324
  ## Security
183
325
 
184
- - **Keep the judge server-side.** `ChatCompletionsJudge` needs an API key;
185
- never instantiate it in a browser. Expose a thin authenticated endpoint that
186
- calls `verifyAnswer` instead.
187
- - **Prompt-injection hardening.** Source text is untrusted. Judge inputs are
188
- length-capped, stripped of control characters and HTML, and the judge prompt
189
- pins them as data ("never instructions"). Output is validated against a
190
- closed vocabulary; unknown classes, out-of-range confidences, and
191
- hallucinated item IDs are rejected.
192
- - **No dynamic evaluation.** Tolerant JSON recovery is a string-aware scanner;
193
- nothing is ever `eval`ed.
194
- - **Failure transparency.** Judge failures degrade to `class: "error"` with a
195
- `null` score — they are reported, never counted as "supported".
326
+ - **Keep your key on the server.** `ChatCompletionsJudge` needs an API key; in
327
+ your own app, call `verifyAnswer` from a backend. The demo runs the judge in
328
+ the browser only with a key the visitor enters.
329
+ - **Source text is untrusted.** Judge inputs are length-capped, stripped of
330
+ control characters and HTML, and marked as data in the prompt. The judge's
331
+ output is checked against a closed vocabulary: unknown classes, out-of-range
332
+ scores and invented item IDs are rejected. Nothing is ever `eval`ed.
196
333
 
197
334
  ## Reproducibility
198
335
 
199
- For a fixed answer, fixed sources, and a fixed judge model, results are
200
- reproducible: the matcher is pure, and the judge runs at temperature 0 (pass
201
- `seed` for providers that support it). Note that hosted LLM APIs are
202
- best-effort deterministic; for strict reproducibility, pin the model version
203
- or use a self-hosted judge behind the `EntailmentJudge` interface.
204
-
205
- ## Integrations
206
-
207
- - **[`verify-citations`](integrations/verify-citations/)**: a portable
208
- [Agent Skill](integrations/verify-citations/SKILL.md) (single `SKILL.md` +
209
- bundled Node CLI) that runs VeriQuote as an **internal hallucination gate**
210
- for source-grounded agents: it verifies a cited answer, flags factual
211
- sentences that carry no citation, and returns a ready-to-use correction
212
- prompt for a self-correction loop. The same skill works across any
213
- Agent-Skills host (OpenClaw, Hermes Agent, Claude Code) and any orchestrator
214
- that can run a Node CLI.
336
+ The matcher is pure: the same inputs always give the same score. The judge
337
+ runs at temperature 0 (pass `seed` where the provider supports it), but hosted
338
+ models are only deterministic on a best-effort basis. For strict
339
+ reproducibility, pin the model version or put a self-hosted model behind the
340
+ `EntailmentJudge` interface.
215
341
 
216
342
  ## Citing
217
343
 
218
- If you use VeriQuote in academic work, please cite the Zenodo record (see
219
- `CITATION.cff`).
344
+ If you use VeriQuote in academic work, please cite it[^veriquote]; the details
345
+ are also in [`CITATION.cff`](CITATION.cff).
220
346
 
221
347
  ## License
222
348
 
223
349
  [MIT](LICENSE)
350
+
351
+ [^alce]: Gao, T., Yen, H., Yu, J., & Chen, D. (2023). Enabling large language models to generate text with citations. In *Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing* (pp. 6465–6488). Association for Computational Linguistics. https://doi.org/10.18653/v1/2023.emnlp-main.398
352
+
353
+ [^true]: Honovich, O., Aharoni, R., Herzig, J., Taitelbaum, H., Kukliansy, D., Cohen, V., Scialom, T., Szpektor, I., Hassidim, A., & Matias, Y. (2022). TRUE: Re-evaluating factual consistency evaluation. In *Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies* (pp. 3905–3920). Association for Computational Linguistics. https://doi.org/10.18653/v1/2022.naacl-main.287
354
+
355
+ [^zhang]: Zhang, J., Chen, Y., Commodore-Mensah, Y., & Oberst, M. (2026). *Verifiable by construction: Claim-level evaluation of verbatim citation in clinical question answering* (Version 2) [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2609.15964
356
+
357
+ [^veriquote]: Heilmann, E. (2026). *VeriQuote: Deterministic and semantic verification of quote-grounded LLM citations* (Version 0.2.1) [Computer software]. Zenodo. https://doi.org/10.5281/zenodo.21552379
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env node
2
+ // Thin process wrapper; the command logic lives in src/cli/main.ts.
3
+ import { readFile } from 'node:fs/promises';
4
+ import { main } from '../dist/cli/main.js';
5
+
6
+ async function readStdin() {
7
+ let data = '';
8
+ process.stdin.setEncoding('utf8');
9
+ for await (const chunk of process.stdin) data += chunk;
10
+ return data;
11
+ }
12
+
13
+ const pkg = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
14
+
15
+ process.exitCode = await main({
16
+ argv: process.argv.slice(2),
17
+ env: process.env,
18
+ stdout: (s) => process.stdout.write(s),
19
+ stderr: (s) => process.stderr.write(s),
20
+ readFile: (path) => readFile(path, 'utf8'),
21
+ readStdin,
22
+ version: pkg.version,
23
+ color: Boolean(process.stdout.isTTY) && !process.env.NO_COLOR,
24
+ });
@@ -0,0 +1,26 @@
1
+ /**
2
+ * `veriquote` command line. Kept free of `process` so it can be tested with
3
+ * injected I/O; `bin/veriquote.mjs` wires it to the real process.
4
+ */
5
+ import type { EntailmentJudge } from '../types.js';
6
+ export interface CliIo {
7
+ argv: string[];
8
+ env: Record<string, string | undefined>;
9
+ stdout: (s: string) => void;
10
+ stderr: (s: string) => void;
11
+ readFile: (path: string) => Promise<string>;
12
+ readStdin: () => Promise<string>;
13
+ fetch?: typeof globalThis.fetch;
14
+ version: string;
15
+ /** Use ANSI colors in human output. */
16
+ color: boolean;
17
+ /** Test hook: replaces the env-configured judge. */
18
+ judge?: EntailmentJudge;
19
+ }
20
+ export declare const EXIT: {
21
+ readonly pass: 0;
22
+ readonly error: 1;
23
+ readonly revise: 2;
24
+ };
25
+ export declare function main(io: CliIo): Promise<number>;
26
+ //# sourceMappingURL=main.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAsC,MAAM,aAAa,CAAC;AAOvF,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACxC,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,uCAAuC;IACvC,KAAK,EAAE,OAAO,CAAC;IACf,oDAAoD;IACpD,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,eAAO,MAAM,IAAI;;;;CAA4C,CAAC;AAkG9D,wBAAsB,IAAI,CAAC,EAAE,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAgCrD"}