veriquote 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CITATION.cff +2 -2
  2. package/README.md +91 -36
  3. package/dist/cli/main.js +12 -1
  4. package/dist/cli/main.js.map +1 -1
  5. package/dist/gate.d.ts +1 -1
  6. package/dist/gate.d.ts.map +1 -1
  7. package/dist/gate.js +14 -0
  8. package/dist/gate.js.map +1 -1
  9. package/dist/index.d.ts +2 -1
  10. package/dist/index.d.ts.map +1 -1
  11. package/dist/index.js +2 -1
  12. package/dist/index.js.map +1 -1
  13. package/dist/judge/decisions-judge.d.ts +72 -0
  14. package/dist/judge/decisions-judge.d.ts.map +1 -0
  15. package/dist/judge/decisions-judge.js +160 -0
  16. package/dist/judge/decisions-judge.js.map +1 -0
  17. package/dist/match/fuzzy.d.ts +10 -2
  18. package/dist/match/fuzzy.d.ts.map +1 -1
  19. package/dist/match/fuzzy.js +87 -11
  20. package/dist/match/fuzzy.js.map +1 -1
  21. package/dist/protocol/evi1.d.ts +21 -1
  22. package/dist/protocol/evi1.d.ts.map +1 -1
  23. package/dist/protocol/evi1.js +169 -28
  24. package/dist/protocol/evi1.js.map +1 -1
  25. package/dist/protocol/prompt.d.ts +6 -0
  26. package/dist/protocol/prompt.d.ts.map +1 -1
  27. package/dist/protocol/prompt.js +35 -10
  28. package/dist/protocol/prompt.js.map +1 -1
  29. package/dist/report.d.ts.map +1 -1
  30. package/dist/report.js +52 -15
  31. package/dist/report.js.map +1 -1
  32. package/dist/types.d.ts +23 -2
  33. package/dist/types.d.ts.map +1 -1
  34. package/package.json +3 -3
  35. package/src/cli/main.ts +13 -2
  36. package/src/gate.ts +16 -1
  37. package/src/index.ts +10 -0
  38. package/src/judge/decisions-judge.ts +230 -0
  39. package/src/match/fuzzy.ts +98 -11
  40. package/src/protocol/evi1.ts +174 -29
  41. package/src/protocol/prompt.ts +42 -9
  42. package/src/report.ts +53 -21
  43. package/src/types.ts +20 -2
@@ -11,7 +11,14 @@
11
11
  * c1|5|"verbatim quote from source 5"
12
12
  * END_EVI1
13
13
  *
14
+ * A (claim, source) pair may have several lines when one passage does not
15
+ * cover the whole claim. With `evidenceFirst` prompting the block comes
16
+ * before the answer instead; the parser accepts either position.
17
+ *
14
18
  * Quotes are single-line, with `\n`, `\"` and `\\` escapes.
19
+ *
20
+ * The parser is lenient about how the appendix is framed (see
21
+ * `parseEvi1Appendix`), strict about the evidence lines themselves.
15
22
  */
16
23
 
17
24
  import type { Claim, EvidenceItem, ParsedAnswer } from '../types.js';
@@ -20,10 +27,25 @@ import { collapseWhitespace } from '../match/normalize.js';
20
27
  export const EVI1_START = 'EVI1';
21
28
  export const EVI1_END = 'END_EVI1';
22
29
 
23
- const EVIDENCE_LINE = /^(c\d+)\|(\d+)\|"([\s\S]*)"$/;
30
+ const EVIDENCE_LINE = /^(c\d+)\s*\|\s*(\d+)\s*\|\s*"([\s\S]*)"$/;
24
31
  const CLAIM_MARKER = /\{c(\d+)\}/g;
25
32
  const CITATION_GROUP_BEFORE_CLAIM = /((?:\[\d+\])+)$/;
26
33
 
34
+ // Models and gateways garble the appendix slightly: "**EVI1**", "EVI1:",
35
+ // "EVI1 c1|2|...", a code fence around it, END_EVI11 or no END_EVI1 at all.
36
+ // The lookahead keeps a sentence about the EVI1 gene from opening an appendix.
37
+ const START_LINE = /^[\s*_`>#]*EVI1[\s*_`:]*(?=$|c\d+\s*\|)/;
38
+ const END_LINE = /^[\s*_`]*END_EVI1/;
39
+ const EVIDENCE_PREFIX = /^\s*c\d+\s*\|\s*\d+\s*\|/;
40
+ // Without END_EVI1, the appendix runs as long as lines look like evidence,
41
+ // malformed ones included (they get a warning, not the answer body).
42
+ const EVIDENCE_LIKE = /^\s*c\d+\s*\|/;
43
+ const FENCE_LINE = /^\s*```\s*\w*\s*$/;
44
+ // While streaming, the last line may be the beginning of an EVI1, evidence or END_EVI1 line.
45
+ const PARTIAL_START =
46
+ /^[\s*_`>#]*(?:E(?:V(?:I(?:1[\s*_`:]*(?:c\d*)?)?)?)?)?$|^[\s*_`]*EN(?:D(?:_(?:E(?:V(?:I1?)?)?)?)?)?$|^\s*c\d*(?:\s*\|\s*\d*)?\s*$/;
47
+ const PARTIAL_MARKER = /\{(?:c\d*)?$/;
48
+
27
49
  /** Unescape an EVI1 quote payload (`\\`, `\n`, `\"`) in a single pass. */
28
50
  function unescapeQuote(s: string): string {
29
51
  return s.replace(/\\(.)/g, (_, c: string) => (c === 'n' ? '\n' : c));
@@ -35,41 +57,110 @@ function escapeQuote(s: string): string {
35
57
  }
36
58
 
37
59
  interface AppendixLocation {
38
- /** Index of the line containing EVI1_START. */
39
- startLine: number;
40
- /** Index of the line containing EVI1_END. */
41
- endLine: number;
60
+ /** First line to cut: the EVI1 line (or first evidence line), or a fence opened just before it. */
61
+ cutStart: number;
62
+ /**
63
+ * First line after the appendix: past END_EVI1 and a closing fence, or,
64
+ * without END_EVI1, the first line that is not evidence (an answer after an
65
+ * evidence-first block), else `lines.length`.
66
+ */
67
+ cutEnd: number;
68
+ /** Candidate evidence lines, the text after an inline `EVI1` header included. */
69
+ body: string[];
42
70
  }
43
71
 
44
- function locateAppendix(lines: string[]): AppendixLocation | null {
72
+ const isBlank = (line: string) => !line.trim();
73
+
74
+ /**
75
+ * Whether the EVI1 line at `start` really opens the appendix: the next
76
+ * non-blank line is evidence, or END_EVI1 follows somewhere.
77
+ * A heading "## EVI1" followed by prose does not.
78
+ */
79
+ function opensAppendix(lines: string[], start: number): boolean {
80
+ if (lines[start].replace(START_LINE, '').trim()) return true; // "EVI1 c1|…"
81
+ let i = start + 1;
82
+ while (i < lines.length && (isBlank(lines[i]) || FENCE_LINE.test(lines[i]))) i++;
83
+ if (EVIDENCE_PREFIX.test(lines[i] ?? '')) return true;
84
+ return lines.slice(i).some((l) => END_LINE.test(l));
85
+ }
86
+
87
+ /**
88
+ * Start of the appendix: the last EVI1 line, or else a run of evidence lines
89
+ * that ends the text. -1 if there is none.
90
+ */
91
+ function findAppendixStart(lines: string[]): number {
45
92
  // Scan from the end so answer text that merely mentions "EVI1" is ignored.
46
- for (let end = lines.length - 1; end >= 0; end--) {
47
- if (lines[end].trim() !== EVI1_END) continue;
48
- for (let start = end - 1; start >= 0; start--) {
49
- if (lines[start].trim() === EVI1_START) return { startLine: start, endLine: end };
93
+ for (let i = lines.length - 1; i >= 0; i--) {
94
+ if (START_LINE.test(lines[i]) && opensAppendix(lines, i)) return i;
95
+ }
96
+ let i = lines.length - 1;
97
+ while (i >= 0 && (isBlank(lines[i]) || END_LINE.test(lines[i]) || FENCE_LINE.test(lines[i]))) i--;
98
+ const last = i;
99
+ while (i >= 0 && EVIDENCE_PREFIX.test(lines[i])) i--;
100
+ return i < last ? i + 1 : -1;
101
+ }
102
+
103
+ function locateAppendix(lines: string[]): AppendixLocation | null {
104
+ const start = findAppendixStart(lines);
105
+ if (start < 0) return null;
106
+
107
+ const headerRest = lines[start].replace(START_LINE, '');
108
+ let endLine = END_LINE.test(headerRest) ? start : -1;
109
+ for (let i = start + 1; endLine < 0 && i < lines.length; i++) {
110
+ if (END_LINE.test(lines[i])) endLine = i;
111
+ }
112
+ let stop = endLine >= 0 ? endLine : start + 1;
113
+ if (endLine < 0) {
114
+ while (stop < lines.length && (isBlank(lines[stop]) || FENCE_LINE.test(lines[stop]) || EVIDENCE_LIKE.test(lines[stop]))) stop++;
115
+ // Blank lines before the answer that follows belong to the answer.
116
+ if (stop < lines.length) while (stop > start + 1 && isBlank(lines[stop - 1])) stop--;
117
+ }
118
+ const body = endLine === start ? [] : [headerRest, ...lines.slice(start + 1, stop)];
119
+
120
+ // A fence the model opened around the appendix goes with it, but not one
121
+ // that closes an earlier code block.
122
+ let cutStart = start;
123
+ let openedFence = /^\s*```/.test(lines[start]);
124
+ let before = start - 1;
125
+ while (before >= 0 && isBlank(lines[before])) before--;
126
+ if (!openedFence && before >= 0 && FENCE_LINE.test(lines[before])) {
127
+ const fences = lines.slice(0, before + 1).filter((l) => FENCE_LINE.test(l)).length;
128
+ if (fences % 2 === 1) {
129
+ cutStart = before;
130
+ openedFence = true;
50
131
  }
51
- return null;
52
132
  }
53
- return null;
133
+
134
+ let cutEnd = endLine < 0 ? stop : endLine + 1;
135
+ if (openedFence && endLine >= 0 && !/```/.test(lines[endLine])) {
136
+ let after = cutEnd;
137
+ while (after < lines.length && isBlank(lines[after])) after++;
138
+ if (after < lines.length && FENCE_LINE.test(lines[after])) cutEnd = after + 1;
139
+ }
140
+ return { cutStart, cutEnd, body };
54
141
  }
55
142
 
56
143
  /**
57
144
  * Parse the EVI1 appendix of a raw model answer.
58
145
  * Returns the evidence items plus warnings for malformed lines;
59
- * `null` when no complete appendix is present.
146
+ * `null` when no appendix is present.
147
+ *
148
+ * Tolerant of common deviations: a decorated start line (`**EVI1**`,
149
+ * `EVI1:`, `EVI1 c1|…`), a code fence around the appendix, a missing or
150
+ * garbled `END_EVI1`, and a missing `EVI1` line when the answer ends in
151
+ * evidence lines.
60
152
  */
61
153
  export function parseEvi1Appendix(
62
154
  answer: string,
63
155
  ): { items: EvidenceItem[]; warnings: string[] } | null {
64
- const lines = String(answer).split('\n');
65
- const loc = locateAppendix(lines);
156
+ const loc = locateAppendix(String(answer).split('\n'));
66
157
  if (!loc) return null;
67
158
 
68
159
  const items: EvidenceItem[] = [];
69
160
  const warnings: string[] = [];
70
- for (let i = loc.startLine + 1; i < loc.endLine; i++) {
71
- const line = lines[i].trim();
72
- if (!line) continue;
161
+ for (const raw of loc.body) {
162
+ const line = raw.trim();
163
+ if (!line || FENCE_LINE.test(line)) continue;
73
164
  const m = line.match(EVIDENCE_LINE);
74
165
  if (!m) {
75
166
  warnings.push(`EVI1: malformed evidence line ignored: ${truncate(line, 80)}`);
@@ -85,16 +176,51 @@ export function parseEvi1Appendix(
85
176
  return { items, warnings };
86
177
  }
87
178
 
179
+ function cutAppendix(lines: string[], loc: AppendixLocation): string {
180
+ const head = lines.slice(0, loc.cutStart);
181
+ const tail = lines.slice(loc.cutEnd);
182
+ // An evidence-first block leaves blank lines before the answer.
183
+ if (head.every(isBlank)) {
184
+ head.length = 0;
185
+ while (tail.length && isBlank(tail[0])) tail.shift();
186
+ }
187
+ return head.concat(tail).join('\n').trimEnd();
188
+ }
189
+
88
190
  /** Remove the EVI1 appendix from a raw answer; returns the answer body. */
89
191
  export function stripEvi1Appendix(answer: string): string {
90
192
  const lines = String(answer).split('\n');
91
193
  const loc = locateAppendix(lines);
92
- if (!loc) return answer;
93
- return lines
94
- .slice(0, loc.startLine)
95
- .concat(lines.slice(loc.endLine + 1))
96
- .join('\n')
97
- .trimEnd();
194
+ return loc ? cutAppendix(lines, loc) : answer;
195
+ }
196
+
197
+ /** Whether the last line may be the beginning of an appendix (but not a fence closing a code block). */
198
+ function isPartialStart(lines: string[]): boolean {
199
+ const last = lines[lines.length - 1];
200
+ if (isBlank(last) || !PARTIAL_START.test(last)) return false;
201
+ return !/```/.test(last) || lines.filter((l) => /^\s*```/.test(l)).length % 2 === 1;
202
+ }
203
+
204
+ /**
205
+ * The answer as a reader should see it, also while it is still streaming:
206
+ * the appendix (complete or not) is cut, as is a last line that may be the
207
+ * beginning of one (`E`, `EV`, `EVI`, `EVI1`, `c1|…`), and all `{cX}`
208
+ * markers are removed, including a half-streamed `{c…` at the end.
209
+ * `[n]` citation markers are kept.
210
+ */
211
+ export function stripForDisplay(text: string): string {
212
+ let lines = String(text).split('\n');
213
+ const popBlank = () => {
214
+ while (lines.length && isBlank(lines[lines.length - 1])) lines.pop();
215
+ };
216
+ popBlank();
217
+ while (lines.length && isPartialStart(lines)) {
218
+ lines.pop();
219
+ popBlank();
220
+ }
221
+ const loc = locateAppendix(lines);
222
+ if (loc) lines = cutAppendix(lines, loc).split('\n');
223
+ return stripClaimMarkers(lines.join('\n').replace(PARTIAL_MARKER, '')).trimEnd();
98
224
  }
99
225
 
100
226
  /** Serialize evidence items into an EVI1 appendix block. */
@@ -173,24 +299,43 @@ export function parseAnswer(answer: string): ParsedAnswer {
173
299
 
174
300
  const body = stripEvi1Appendix(answer);
175
301
  const { claims, warnings: claimWarnings } = extractClaims(body);
176
- warnings.push(...claimWarnings);
302
+ // A claim marker without its [n] group still names its sources in the
303
+ // appendix; take them from there instead of losing the claim's evidence.
304
+ const noGroup = new Set<string>();
305
+ for (const c of claims) {
306
+ if (c.sourceIndexes.length) continue;
307
+ const fromAppendix = [...new Set(evidence.filter((e) => e.claimId === c.id).map((e) => e.sourceIndex))];
308
+ if (!fromAppendix.length) continue;
309
+ c.sourceIndexes = fromAppendix;
310
+ noGroup.add(`Claim ${c.id}: no citation group directly before marker.`);
311
+ }
312
+ warnings.push(
313
+ ...claimWarnings.map((w) =>
314
+ noGroup.has(w) ? w.replace(/\.$/, '; sources taken from the EVI1 appendix.') : w,
315
+ ),
316
+ );
177
317
 
178
318
  const cited = new Set(
179
319
  claims.flatMap((c) => c.sourceIndexes.map((n) => `${c.id}|${n}`)),
180
320
  );
321
+ // Several lines for one (claim, source) pair are allowed: a claim may need
322
+ // two passages of the same source. Only repeated quotes are dropped.
181
323
  const evidenced = new Set<string>();
324
+ const seenLines = new Set<string>();
182
325
  const dedupedEvidence: typeof evidence = [];
183
326
  for (const it of evidence) {
184
327
  const key = `${it.claimId}|${it.sourceIndex}`;
185
- if (evidenced.has(key)) {
186
- warnings.push(`EVI1: duplicate evidence for ${key}; keeping first occurrence.`);
328
+ const line = `${key}|${collapseWhitespace(it.quote)}`;
329
+ if (seenLines.has(line)) {
330
+ warnings.push(`EVI1: repeated quote for ${key}; keeping first occurrence.`);
187
331
  continue;
188
332
  }
189
- evidenced.add(key);
333
+ seenLines.add(line);
190
334
  dedupedEvidence.push(it);
191
- if (!cited.has(key)) {
335
+ if (!evidenced.has(key) && !cited.has(key)) {
192
336
  warnings.push(`EVI1: evidence for ${key} has no matching citation in the answer.`);
193
337
  }
338
+ evidenced.add(key);
194
339
  }
195
340
  for (const key of cited) {
196
341
  if (!evidenced.has(key)) {
@@ -13,6 +13,12 @@ export interface CitationPromptOptions {
13
13
  quoteLengthRange?: [number, number];
14
14
  /** Minimum quote length in characters (with stated exceptions). Default 60. */
15
15
  minQuoteLength?: number;
16
+ /**
17
+ * Put the EVI1 block before the answer: the model selects its passages
18
+ * first, then writes only what they support. The parser accepts either
19
+ * position; a streaming client shows nothing until END_EVI1. Default false.
20
+ */
21
+ evidenceFirst?: boolean;
16
22
  }
17
23
 
18
24
  /** Build the EVI1 citation instruction block for the answering model. */
@@ -22,8 +28,25 @@ export function buildCitationInstructions(options: CitationPromptOptions = {}):
22
28
  maxCitationsPerClaim = 2,
23
29
  quoteLengthRange = [80, 240],
24
30
  minQuoteLength = 60,
31
+ evidenceFirst = false,
25
32
  } = options;
26
33
 
34
+ const appendixIntro = evidenceFirst
35
+ ? `Before you write the answer, output the evidence block. Select the passages
36
+ you will rely on first, one line per planned claim and source, then write the
37
+ answer so that every cited sentence states only what its lines support.`
38
+ : 'After you finish the answer, append an appendix:';
39
+ const afterBlock = evidenceFirst
40
+ ? '- After END_EVI1, write the user-facing answer. The claim ids {cX} in the answer must be the ones used in the block.'
41
+ : '- Do NOT output anything after END_EVI1.';
42
+ const important = evidenceFirst
43
+ ? `- The user-facing answer starts AFTER END_EVI1. Nothing but the block comes before it.
44
+ - The block is required only when you use citations. If you use no
45
+ citations, do NOT output EVI1/END_EVI1.`
46
+ : `- The user-facing answer must end BEFORE the EVI1 appendix starts.
47
+ - The appendix is required only when you used citations. If you used no
48
+ citations, do NOT output EVI1/END_EVI1.`;
49
+
27
50
  return `
28
51
  [CITATION BUDGET]
29
52
  - Keep citations sparse. Only cite new, load-bearing information.
@@ -50,17 +73,20 @@ export function buildCitationInstructions(options: CitationPromptOptions = {}):
50
73
  Example: "...water expands when freezing.[2][5]{c1}"
51
74
  Bad: "...freezing.[2] {c1}"
52
75
  Bad: "...freezing.[2].{c1}"
53
- - Every sentence OR bullet that contains citations MUST end with exactly ONE claim marker {cX}.
76
+ - Every sentence OR bullet that contains citations MUST end with exactly ONE claim marker {cX},
77
+ directly after its [n] group. Never write {cX} without the [n] group, even though the
78
+ appendix names the source as well.
54
79
  - X starts at 1 and increases by 1 for each new cited sentence/bullet in THIS answer.
55
80
  - Do NOT place {cX} anywhere else. Uncited sentences get no claim marker.
56
81
 
57
- [QUOTE-BASED EVIDENCE APPENDIX (MANDATORY)]
58
- After you finish the answer, append an appendix:
82
+ [QUOTE-BASED EVIDENCE ${evidenceFirst ? 'BLOCK' : 'APPENDIX'} (MANDATORY)]
83
+ ${appendixIntro}
59
84
  - Start with a line containing exactly: EVI1
60
85
  - Then output one line per (claim, source) pair in this exact format:
61
86
  cX|n|"QUOTE"
87
+ If one passage does not cover the claim, add further lines for the same pair.
62
88
  - End with a line containing exactly: END_EVI1
63
- - Do NOT output anything after END_EVI1.
89
+ ${afterBlock}
64
90
  - Do NOT wrap EVI1..END_EVI1 in code fences or markdown.
65
91
 
66
92
  [COMPLETENESS RULES (MANDATORY)]
@@ -71,10 +97,19 @@ After you finish the answer, append an appendix:
71
97
  - If you cannot provide a verbatim quote for a citation, REMOVE that citation
72
98
  from the answer. Never leave a citation unverified.
73
99
 
100
+ [SUFFICIENCY RULES (MANDATORY)]
101
+ - The quotes of a cited sentence must together contain EVERY detail it states:
102
+ each number, population, time frame, place, condition, comparison and hedge.
103
+ - If one passage does not cover everything, add another line for the same claim
104
+ (same or other cited source). Otherwise remove the unquoted detail from the
105
+ sentence. Never add detail that no quote contains.
106
+
74
107
  Rules for QUOTE:
75
108
  - QUOTE must be copied verbatim from the provided source TEXT (not paraphrased).
76
- - QUOTE must be sufficient to support the claim; prefer the shortest quote that
77
- still supports it (typically ${quoteLengthRange[0]}-${quoteLengthRange[1]} characters).
109
+ - Prefer the shortest quote that still covers its part of the claim
110
+ (typically ${quoteLengthRange[0]}-${quoteLengthRange[1]} characters).
111
+ - Copy one contiguous passage per line. Do NOT shorten a quote with "..." and do
112
+ NOT join separate passages into one quote; write each passage on its own line.
78
113
  - QUOTE must be at least ${minQuoteLength} characters unless it contains a numeric result or is
79
114
  a complete standalone sentence that uniquely supports the claim.
80
115
  - QUOTE must be a single line: escape newlines as \\n and double quotes as \\".
@@ -92,8 +127,6 @@ Rules for QUOTE:
92
127
  claim; when only conclusions are visible, use cautious language.
93
128
 
94
129
  IMPORTANT:
95
- - The user-facing answer must end BEFORE the EVI1 appendix starts.
96
- - The appendix is required only when you used citations. If you used no
97
- citations, do NOT output EVI1/END_EVI1.
130
+ ${important}
98
131
  `.trim();
99
132
  }
package/src/report.ts CHANGED
@@ -40,39 +40,54 @@ export async function verifyAnswer(options: VerifyOptions): Promise<Verification
40
40
  const warnings = [...parsed.warnings];
41
41
  const claimById = new Map(parsed.claims.map((c) => [c.id, c]));
42
42
 
43
- const citations: CitationVerification[] = [];
43
+ // One citation per (claim, source) pair; a pair may carry several quotes.
44
+ const pairs = new Map<string, { claimId: string; sourceIndex: number; quotes: string[] }>();
44
45
  for (const item of parsed.evidence) {
45
- const claim = claimById.get(item.claimId);
46
- const source = sources[item.sourceIndex - 1];
46
+ const key = `${item.claimId}|${item.sourceIndex}`;
47
+ const pair = pairs.get(key) ?? { claimId: item.claimId, sourceIndex: item.sourceIndex, quotes: [] };
48
+ pair.quotes.push(item.quote);
49
+ pairs.set(key, pair);
50
+ }
51
+
52
+ const citations: CitationVerification[] = [];
53
+ for (const [key, pair] of pairs) {
54
+ const source = sources[pair.sourceIndex - 1];
47
55
  if (!source) {
48
- warnings.push(
49
- `EVI1: evidence ${item.claimId}|${item.sourceIndex} references a source that was not provided.`,
50
- );
56
+ warnings.push(`EVI1: evidence ${key} references a source that was not provided.`);
51
57
  continue;
52
58
  }
53
- const textMatch: QuoteMatch = matchQuoteAgainstSource(item.quote, source, match);
59
+ const parts = pair.quotes.map((quote) => ({ quote, textMatch: matchQuoteAgainstSource(quote, source, match) }));
60
+ const textMatch = combineMatches(parts.map((p) => p.textMatch));
61
+ for (const cue of textMatch.omittedCues ?? []) {
62
+ warnings.push(`Citation ${key}: the ellipsis in the quote leaves out "${cue}".`);
63
+ }
54
64
  citations.push({
55
- claimId: item.claimId,
56
- sourceIndex: item.sourceIndex,
57
- claimText: claim?.text ?? '',
58
- quote: item.quote,
65
+ claimId: pair.claimId,
66
+ sourceIndex: pair.sourceIndex,
67
+ claimText: claimById.get(pair.claimId)?.text ?? '',
68
+ quote: pair.quotes.join(' […] '),
59
69
  textMatch,
70
+ ...(parts.length > 1 ? { parts } : {}),
60
71
  score: textMatch.score,
61
72
  });
62
73
  }
63
74
 
64
75
  if (judge) {
65
76
  const judgeable = citations.filter((c) => c.claimText && c.quote);
66
- const inputs: EntailmentInput[] = judgeable.map((c) => ({
67
- id: `${c.claimId}|${c.sourceIndex}`,
68
- claim: c.claimText,
69
- quote: c.quote,
70
- context: contextWindow(
71
- sources[c.sourceIndex - 1],
72
- c.textMatch,
73
- contextWindowChars,
74
- ),
75
- }));
77
+ const inputs: EntailmentInput[] = judgeable.map((c) => {
78
+ const source = sources[c.sourceIndex - 1];
79
+ const parts = c.parts ?? [{ quote: c.quote, textMatch: c.textMatch }];
80
+ return {
81
+ id: `${c.claimId}|${c.sourceIndex}`,
82
+ claim: c.claimText,
83
+ // An ellipsis must not hide what it left out (a "not", a qualifier):
84
+ // the judge rates the passage as the source has it.
85
+ quote: parts
86
+ .map((p) => (p.textMatch.method === 'elided' ? matchedText(source, p.textMatch) ?? p.quote : p.quote))
87
+ .join(' […] '),
88
+ context: parts.map((p) => contextWindow(source, p.textMatch, contextWindowChars)).join(' […] '),
89
+ };
90
+ });
76
91
  const results = await judge.judge(inputs, { signal });
77
92
  if (results.length !== inputs.length) {
78
93
  warnings.push(
@@ -109,6 +124,23 @@ export async function verifyAnswer(options: VerifyOptions): Promise<Verification
109
124
  };
110
125
  }
111
126
 
127
+ /** Several quotes for one pair count as their weakest match, carrying every cue. */
128
+ function combineMatches(matches: QuoteMatch[]): QuoteMatch {
129
+ if (matches.length === 1) return matches[0];
130
+ const weakest = matches.reduce((a, b) => (b.score < a.score ? b : a));
131
+ const cues = [...new Set(matches.flatMap((m) => m.omittedCues ?? []))];
132
+ const { omittedCues: _, ...rest } = weakest;
133
+ return cues.length ? { ...rest, omittedCues: cues } : rest;
134
+ }
135
+
136
+ /** The matched region of the source field, if its offsets are known. */
137
+ function matchedText(source: SourceDocument | undefined, m: QuoteMatch): string | undefined {
138
+ if (!source || m.start === undefined || m.end === undefined) return undefined;
139
+ const extra = /^extraTexts\[(\d+)\]$/.exec(m.field);
140
+ const text = extra ? source.extraTexts?.[Number(extra[1])] : source.text;
141
+ return text?.slice(m.start, m.end);
142
+ }
143
+
112
144
  /** Source text around the matched region, for judge disambiguation. */
113
145
  function contextWindow(
114
146
  source: SourceDocument | undefined,
package/src/types.ts CHANGED
@@ -57,6 +57,13 @@ export type MatchMethod =
57
57
  | 'exact'
58
58
  /** Substring after Unicode/typography normalization and case folding. */
59
59
  | 'normalized'
60
+ /**
61
+ * The quote omits text, marked with an ellipsis ("…", "...", "[…]"): every
62
+ * fragment occurs literally, in order and close together. `start`/`end`
63
+ * cover the whole passage, omitted text included, and that passage is what
64
+ * the judge sees.
65
+ */
66
+ | 'elided'
60
67
  /** Best fuzzy window by character-trigram Dice similarity. */
61
68
  | 'fuzzy'
62
69
  /** No window reached the fuzzy score threshold. */
@@ -66,8 +73,9 @@ export type MatchMethod =
66
73
  export interface QuoteMatch {
67
74
  method: MatchMethod;
68
75
  /**
69
- * Similarity in [0, 1]. `1` only for exact/normalized hits; fuzzy scores
70
- * are capped at 0.99 so a perfect score always implies a literal hit.
76
+ * Similarity in [0, 1]. `1` only for exact/normalized hits; elided and
77
+ * fuzzy scores are capped at 0.99 so a perfect score always implies a
78
+ * literal hit.
71
79
  */
72
80
  score: number;
73
81
  /** Start offset of the matched region in the original source field, if known. */
@@ -76,6 +84,12 @@ export interface QuoteMatch {
76
84
  end?: number;
77
85
  /** Which source field matched: `"text"` or `"extraTexts[i]"`. */
78
86
  field: string;
87
+ /**
88
+ * `elided` only: words in the omitted text that can reverse or limit a
89
+ * statement ("not", "except", "only", "however", …). A hint, not a verdict:
90
+ * the judge sees the full passage and decides.
91
+ */
92
+ omittedCues?: string[];
79
93
  }
80
94
 
81
95
  /** Qualitative entailment classes produced by the LLM judge. */
@@ -122,8 +136,12 @@ export interface CitationVerification {
122
136
  claimId: string;
123
137
  sourceIndex: number;
124
138
  claimText: string;
139
+ /** The quote; several quotes for the same (claim, source) pair are joined with " […] ". */
125
140
  quote: string;
141
+ /** The match of the quote; with several quotes, the weakest one (cues of all of them). */
126
142
  textMatch: QuoteMatch;
143
+ /** One entry per quote, present only when the pair has more than one. */
144
+ parts?: { quote: string; textMatch: QuoteMatch }[];
127
145
  /** Absent when verification ran without a judge. */
128
146
  entailment?: EntailmentResult;
129
147
  /**