veriquote 0.2.2 → 0.3.0

package/src/protocol/prompt.ts

@@ -13,6 +13,12 @@ export interface CitationPromptOptions {
   quoteLengthRange?: [number, number];
   /** Minimum quote length in characters (with stated exceptions). Default 60. */
   minQuoteLength?: number;
+  /**
+   * Put the EVI1 block before the answer: the model selects its passages
+   * first, then writes only what they support. The parser accepts either
+   * position; a streaming client shows nothing until END_EVI1. Default false.
+   */
+  evidenceFirst?: boolean;
 }
 
 /** Build the EVI1 citation instruction block for the answering model. */
@@ -22,8 +28,25 @@ export function buildCitationInstructions(options: CitationPromptOptions = {}):
     maxCitationsPerClaim = 2,
     quoteLengthRange = [80, 240],
     minQuoteLength = 60,
+    evidenceFirst = false,
   } = options;
 
+  const appendixIntro = evidenceFirst
+    ? `Before you write the answer, output the evidence block. Select the passages
+you will rely on first, one line per planned claim and source, then write the
+answer so that every cited sentence states only what its lines support.`
+    : 'After you finish the answer, append an appendix:';
+  const afterBlock = evidenceFirst
+    ? '- After END_EVI1, write the user-facing answer. The claim ids {cX} in the answer must be the ones used in the block.'
+    : '- Do NOT output anything after END_EVI1.';
+  const important = evidenceFirst
+    ? `- The user-facing answer starts AFTER END_EVI1. Nothing but the block comes before it.
+- The block is required only when you use citations. If you use no
+  citations, do NOT output EVI1/END_EVI1.`
+    : `- The user-facing answer must end BEFORE the EVI1 appendix starts.
+- The appendix is required only when you used citations. If you used no
+  citations, do NOT output EVI1/END_EVI1.`;
+
   return `
 [CITATION BUDGET]
 - Keep citations sparse. Only cite new, load-bearing information.
@@ -50,17 +73,20 @@ export function buildCitationInstructions(options: CitationPromptOptions = {}):
   Example: "...water expands when freezing.[2][5]{c1}"
   Bad: "...freezing.[2] {c1}"
   Bad: "...freezing.[2].{c1}"
-- Every sentence OR bullet that contains citations MUST end with exactly ONE claim marker {cX}.
+- Every sentence OR bullet that contains citations MUST end with exactly ONE claim marker {cX},
+  directly after its [n] group. Never write {cX} without the [n] group, even though the
+  appendix names the source as well.
 - X starts at 1 and increases by 1 for each new cited sentence/bullet in THIS answer.
 - Do NOT place {cX} anywhere else. Uncited sentences get no claim marker.
 
-[QUOTE-BASED EVIDENCE APPENDIX (MANDATORY)]
-After you finish the answer, append an appendix:
+[QUOTE-BASED EVIDENCE ${evidenceFirst ? 'BLOCK' : 'APPENDIX'} (MANDATORY)]
+${appendixIntro}
 - Start with a line containing exactly: EVI1
 - Then output one line per (claim, source) pair in this exact format:
   cX|n|"QUOTE"
+  If one passage does not cover the claim, add further lines for the same pair.
 - End with a line containing exactly: END_EVI1
-- Do NOT output anything after END_EVI1.
+${afterBlock}
 - Do NOT wrap EVI1..END_EVI1 in code fences or markdown.
 
 [COMPLETENESS RULES (MANDATORY)]
@@ -71,10 +97,19 @@ After you finish the answer, append an appendix:
 - If you cannot provide a verbatim quote for a citation, REMOVE that citation
   from the answer. Never leave a citation unverified.
 
+[SUFFICIENCY RULES (MANDATORY)]
+- The quotes of a cited sentence must together contain EVERY detail it states:
+  each number, population, time frame, place, condition, comparison and hedge.
+- If one passage does not cover everything, add another line for the same claim
+  (same or other cited source). Otherwise remove the unquoted detail from the
+  sentence. Never add detail that no quote contains.
+
 Rules for QUOTE:
 - QUOTE must be copied verbatim from the provided source TEXT (not paraphrased).
-- QUOTE must be sufficient to support the claim; prefer the shortest quote that
-  still supports it (typically ${quoteLengthRange[0]}-${quoteLengthRange[1]} characters).
+- Prefer the shortest quote that still covers its part of the claim
+  (typically ${quoteLengthRange[0]}-${quoteLengthRange[1]} characters).
+- Copy one contiguous passage per line. Do NOT shorten a quote with "..." and do
+  NOT join separate passages into one quote; write each passage on its own line.
 - QUOTE must be at least ${minQuoteLength} characters unless it contains a numeric result or is
   a complete standalone sentence that uniquely supports the claim.
 - QUOTE must be a single line: escape newlines as \\n and double quotes as \\".
@@ -92,8 +127,6 @@ Rules for QUOTE:
   claim; when only conclusions are visible, use cautious language.
 
 IMPORTANT:
-- The user-facing answer must end BEFORE the EVI1 appendix starts.
-- The appendix is required only when you used citations. If you used no
-  citations, do NOT output EVI1/END_EVI1.
+${important}
 `.trim();
 }

package/src/report.ts

@@ -40,39 +40,54 @@ export async function verifyAnswer(options: VerifyOptions): Promise<Verification
   const warnings = [...parsed.warnings];
   const claimById = new Map(parsed.claims.map((c) => [c.id, c]));
 
-  const citations: CitationVerification[] = [];
+  // One citation per (claim, source) pair; a pair may carry several quotes.
+  const pairs = new Map<string, { claimId: string; sourceIndex: number; quotes: string[] }>();
   for (const item of parsed.evidence) {
-    const claim = claimById.get(item.claimId);
-    const source = sources[item.sourceIndex - 1];
+    const key = `${item.claimId}|${item.sourceIndex}`;
+    const pair = pairs.get(key) ?? { claimId: item.claimId, sourceIndex: item.sourceIndex, quotes: [] };
+    pair.quotes.push(item.quote);
+    pairs.set(key, pair);
+  }
+
+  const citations: CitationVerification[] = [];
+  for (const [key, pair] of pairs) {
+    const source = sources[pair.sourceIndex - 1];
     if (!source) {
-      warnings.push(
-        `EVI1: evidence ${item.claimId}|${item.sourceIndex} references a source that was not provided.`,
-      );
+      warnings.push(`EVI1: evidence ${key} references a source that was not provided.`);
       continue;
     }
-    const textMatch: QuoteMatch = matchQuoteAgainstSource(item.quote, source, match);
+    const parts = pair.quotes.map((quote) => ({ quote, textMatch: matchQuoteAgainstSource(quote, source, match) }));
+    const textMatch = combineMatches(parts.map((p) => p.textMatch));
+    for (const cue of textMatch.omittedCues ?? []) {
+      warnings.push(`Citation ${key}: the ellipsis in the quote leaves out "${cue}".`);
+    }
     citations.push({
-      claimId: item.claimId,
-      sourceIndex: item.sourceIndex,
-      claimText: claim?.text ?? '',
-      quote: item.quote,
+      claimId: pair.claimId,
+      sourceIndex: pair.sourceIndex,
+      claimText: claimById.get(pair.claimId)?.text ?? '',
+      quote: pair.quotes.join(' […] '),
       textMatch,
+      ...(parts.length > 1 ? { parts } : {}),
       score: textMatch.score,
     });
   }
 
   if (judge) {
     const judgeable = citations.filter((c) => c.claimText && c.quote);
-    const inputs: EntailmentInput[] = judgeable.map((c) => ({
-      id: `${c.claimId}|${c.sourceIndex}`,
-      claim: c.claimText,
-      quote: c.quote,
-      context: contextWindow(
-        sources[c.sourceIndex - 1],
-        c.textMatch,
-        contextWindowChars,
-      ),
-    }));
+    const inputs: EntailmentInput[] = judgeable.map((c) => {
+      const source = sources[c.sourceIndex - 1];
+      const parts = c.parts ?? [{ quote: c.quote, textMatch: c.textMatch }];
+      return {
+        id: `${c.claimId}|${c.sourceIndex}`,
+        claim: c.claimText,
+        // An ellipsis must not hide what it left out (a "not", a qualifier):
+        // the judge rates the passage as the source has it.
+        quote: parts
+          .map((p) => (p.textMatch.method === 'elided' ? matchedText(source, p.textMatch) ?? p.quote : p.quote))
+          .join(' […] '),
+        context: parts.map((p) => contextWindow(source, p.textMatch, contextWindowChars)).join(' […] '),
+      };
+    });
     const results = await judge.judge(inputs, { signal });
     if (results.length !== inputs.length) {
       warnings.push(
@@ -109,6 +124,23 @@ export async function verifyAnswer(options: VerifyOptions): Promise<Verification
   };
 }
 
+/** Several quotes for one pair count as their weakest match, carrying every cue. */
+function combineMatches(matches: QuoteMatch[]): QuoteMatch {
+  if (matches.length === 1) return matches[0];
+  const weakest = matches.reduce((a, b) => (b.score < a.score ? b : a));
+  const cues = [...new Set(matches.flatMap((m) => m.omittedCues ?? []))];
+  const { omittedCues: _, ...rest } = weakest;
+  return cues.length ? { ...rest, omittedCues: cues } : rest;
+}
+
+/** The matched region of the source field, if its offsets are known. */
+function matchedText(source: SourceDocument | undefined, m: QuoteMatch): string | undefined {
+  if (!source || m.start === undefined || m.end === undefined) return undefined;
+  const extra = /^extraTexts\[(\d+)\]$/.exec(m.field);
+  const text = extra ? source.extraTexts?.[Number(extra[1])] : source.text;
+  return text?.slice(m.start, m.end);
+}
+
 /** Source text around the matched region, for judge disambiguation. */
 function contextWindow(
   source: SourceDocument | undefined,

package/src/types.ts

@@ -57,6 +57,13 @@ export type MatchMethod =
   | 'exact'
   /** Substring after Unicode/typography normalization and case folding. */
   | 'normalized'
+  /**
+   * The quote omits text, marked with an ellipsis ("…", "...", "[…]"): every
+   * fragment occurs literally, in order and close together. `start`/`end`
+   * cover the whole passage, omitted text included, and that passage is what
+   * the judge sees.
+   */
+  | 'elided'
   /** Best fuzzy window by character-trigram Dice similarity. */
   | 'fuzzy'
   /** No window reached the fuzzy score threshold. */
@@ -66,8 +73,9 @@ export type MatchMethod =
 export interface QuoteMatch {
   method: MatchMethod;
   /**
-   * Similarity in [0, 1]. `1` only for exact/normalized hits; fuzzy scores
-   * are capped at 0.99 so a perfect score always implies a literal hit.
+   * Similarity in [0, 1]. `1` only for exact/normalized hits; elided and
+   * fuzzy scores are capped at 0.99 so a perfect score always implies a
+   * literal hit.
    */
   score: number;
   /** Start offset of the matched region in the original source field, if known. */
@@ -76,6 +84,12 @@ export interface QuoteMatch {
   end?: number;
   /** Which source field matched: `"text"` or `"extraTexts[i]"`. */
   field: string;
+  /**
+   * `elided` only: words in the omitted text that can reverse or limit a
+   * statement ("not", "except", "only", "however", …). A hint, not a verdict:
+   * the judge sees the full passage and decides.
+   */
+  omittedCues?: string[];
 }
 
 /** Qualitative entailment classes produced by the LLM judge. */
@@ -122,8 +136,12 @@ export interface CitationVerification {
   claimId: string;
   sourceIndex: number;
   claimText: string;
+  /** The quote; several quotes for the same (claim, source) pair are joined with " […] ". */
   quote: string;
+  /** The match of the quote; with several quotes, the weakest one (cues of all of them). */
   textMatch: QuoteMatch;
+  /** One entry per quote, present only when the pair has more than one. */
+  parts?: { quote: string; textMatch: QuoteMatch }[];
   /** Absent when verification ran without a judge. */
   entailment?: EntailmentResult;
   /**