veriquote 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CITATION.cff +33 -0
  2. package/LICENSE +21 -0
  3. package/README.md +223 -0
  4. package/dist/index.d.ts +20 -0
  5. package/dist/index.d.ts.map +1 -0
  6. package/dist/index.js +20 -0
  7. package/dist/index.js.map +1 -0
  8. package/dist/judge/chat-judge.d.ts +53 -0
  9. package/dist/judge/chat-judge.d.ts.map +1 -0
  10. package/dist/judge/chat-judge.js +205 -0
  11. package/dist/judge/chat-judge.js.map +1 -0
  12. package/dist/judge/json-extract.d.ts +21 -0
  13. package/dist/judge/json-extract.d.ts.map +1 -0
  14. package/dist/judge/json-extract.js +117 -0
  15. package/dist/judge/json-extract.js.map +1 -0
  16. package/dist/match/fuzzy.d.ts +46 -0
  17. package/dist/match/fuzzy.d.ts.map +1 -0
  18. package/dist/match/fuzzy.js +210 -0
  19. package/dist/match/fuzzy.js.map +1 -0
  20. package/dist/match/normalize.d.ts +26 -0
  21. package/dist/match/normalize.d.ts.map +1 -0
  22. package/dist/match/normalize.js +0 -0
  23. package/dist/match/normalize.js.map +1 -0
  24. package/dist/protocol/evi1.d.ts +51 -0
  25. package/dist/protocol/evi1.d.ts.map +1 -0
  26. package/dist/protocol/evi1.js +179 -0
  27. package/dist/protocol/evi1.js.map +1 -0
  28. package/dist/protocol/prompt.d.ts +18 -0
  29. package/dist/protocol/prompt.d.ts.map +1 -0
  30. package/dist/protocol/prompt.js +82 -0
  31. package/dist/protocol/prompt.js.map +1 -0
  32. package/dist/report.d.ts +24 -0
  33. package/dist/report.d.ts.map +1 -0
  34. package/dist/report.js +106 -0
  35. package/dist/report.js.map +1 -0
  36. package/dist/types.d.ts +141 -0
  37. package/dist/types.d.ts.map +1 -0
  38. package/dist/types.js +5 -0
  39. package/dist/types.js.map +1 -0
  40. package/package.json +54 -0
  41. package/src/index.ts +45 -0
  42. package/src/judge/chat-judge.ts +258 -0
  43. package/src/judge/json-extract.ts +103 -0
  44. package/src/match/fuzzy.ts +253 -0
  45. package/src/match/normalize.ts +0 -0
  46. package/src/protocol/evi1.ts +206 -0
  47. package/src/protocol/prompt.ts +99 -0
  48. package/src/report.ts +152 -0
  49. package/src/types.ts +156 -0
@@ -0,0 +1,179 @@
1
+ /**
2
+ * The EVI1 protocol: a plain-text convention that lets an LLM attach
3
+ * verifiable evidence to its answer.
4
+ *
5
+ * In the answer body, every cited sentence ends with citation markers and a
6
+ * claim marker, e.g. `...water expands when freezing.[2][5]{c1}`. After the
7
+ * answer, the model appends:
8
+ *
9
+ * EVI1
10
+ * c1|2|"verbatim quote from source 2"
11
+ * c1|5|"verbatim quote from source 5"
12
+ * END_EVI1
13
+ *
14
+ * Quotes are single-line, with `\n`, `\"` and `\\` escapes.
15
+ */
16
+ import { collapseWhitespace } from '../match/normalize.js';
17
+ export const EVI1_START = 'EVI1';
18
+ export const EVI1_END = 'END_EVI1';
19
+ const EVIDENCE_LINE = /^(c\d+)\|(\d+)\|"([\s\S]*)"$/;
20
+ const CLAIM_MARKER = /\{c(\d+)\}/g;
21
+ const CITATION_GROUP_BEFORE_CLAIM = /((?:\[\d+\])+)$/;
22
+ /** Unescape an EVI1 quote payload (`\\`, `\n`, `\"`) in a single pass. */
23
+ function unescapeQuote(s) {
24
+ return s.replace(/\\(.)/g, (_, c) => (c === 'n' ? '\n' : c));
25
+ }
26
+ /** Escape a quote for serialization into an EVI1 line. */
27
+ function escapeQuote(s) {
28
+ return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\r?\n/g, '\\n');
29
+ }
30
+ function locateAppendix(lines) {
31
+ // Scan from the end so answer text that merely mentions "EVI1" is ignored.
32
+ for (let end = lines.length - 1; end >= 0; end--) {
33
+ if (lines[end].trim() !== EVI1_END)
34
+ continue;
35
+ for (let start = end - 1; start >= 0; start--) {
36
+ if (lines[start].trim() === EVI1_START)
37
+ return { startLine: start, endLine: end };
38
+ }
39
+ return null;
40
+ }
41
+ return null;
42
+ }
43
+ /**
44
+ * Parse the EVI1 appendix of a raw model answer.
45
+ * Returns the evidence items plus warnings for malformed lines;
46
+ * `null` when no complete appendix is present.
47
+ */
48
+ export function parseEvi1Appendix(answer) {
49
+ const lines = String(answer).split('\n');
50
+ const loc = locateAppendix(lines);
51
+ if (!loc)
52
+ return null;
53
+ const items = [];
54
+ const warnings = [];
55
+ for (let i = loc.startLine + 1; i < loc.endLine; i++) {
56
+ const line = lines[i].trim();
57
+ if (!line)
58
+ continue;
59
+ const m = line.match(EVIDENCE_LINE);
60
+ if (!m) {
61
+ warnings.push(`EVI1: malformed evidence line ignored: ${truncate(line, 80)}`);
62
+ continue;
63
+ }
64
+ const sourceIndex = Number.parseInt(m[2], 10);
65
+ if (!Number.isFinite(sourceIndex) || sourceIndex < 1) {
66
+ warnings.push(`EVI1: invalid source index in line: ${truncate(line, 80)}`);
67
+ continue;
68
+ }
69
+ items.push({ claimId: m[1], sourceIndex, quote: unescapeQuote(m[3]) });
70
+ }
71
+ return { items, warnings };
72
+ }
73
+ /** Remove the EVI1 appendix from a raw answer; returns the answer body. */
74
+ export function stripEvi1Appendix(answer) {
75
+ const lines = String(answer).split('\n');
76
+ const loc = locateAppendix(lines);
77
+ if (!loc)
78
+ return answer;
79
+ return lines
80
+ .slice(0, loc.startLine)
81
+ .concat(lines.slice(loc.endLine + 1))
82
+ .join('\n')
83
+ .trimEnd();
84
+ }
85
+ /** Serialize evidence items into an EVI1 appendix block. */
86
+ export function serializeEvi1Appendix(items) {
87
+ const body = items.map((it) => `${it.claimId}|${it.sourceIndex}|"${escapeQuote(it.quote)}"`);
88
+ return [EVI1_START, ...body, EVI1_END].join('\n');
89
+ }
90
+ /**
91
+ * Extract cited claims from an answer body (appendix already stripped).
92
+ *
93
+ * A claim is the text segment ending at a `{cX}` marker, bounded by the start
94
+ * of its line/block or the previous claim marker. Its cited sources are the
95
+ * contiguous `[n]` group immediately preceding the marker, per protocol.
96
+ */
97
+ export function extractClaims(body) {
98
+ const claims = [];
99
+ const warnings = [];
100
+ const seen = new Set();
101
+ for (const block of String(body).split('\n')) {
102
+ if (!block.includes('{c'))
103
+ continue;
104
+ let segmentStart = 0;
105
+ CLAIM_MARKER.lastIndex = 0;
106
+ let m;
107
+ while ((m = CLAIM_MARKER.exec(block)) !== null) {
108
+ const id = `c${m[1]}`;
109
+ const segment = block.slice(segmentStart, m.index);
110
+ segmentStart = m.index + m[0].length;
111
+ const groupMatch = segment.match(CITATION_GROUP_BEFORE_CLAIM);
112
+ const sourceIndexes = groupMatch
113
+ ? [...groupMatch[1].matchAll(/\[(\d+)\]/g)].map((g) => Number.parseInt(g[1], 10))
114
+ : [];
115
+ if (!groupMatch) {
116
+ warnings.push(`Claim ${id}: no citation group directly before marker.`);
117
+ }
118
+ if (seen.has(id)) {
119
+ warnings.push(`Claim ${id}: duplicate claim id; keeping first occurrence.`);
120
+ continue;
121
+ }
122
+ seen.add(id);
123
+ claims.push({
124
+ id,
125
+ text: cleanClaimText(segment),
126
+ sourceIndexes: [...new Set(sourceIndexes)],
127
+ });
128
+ }
129
+ }
130
+ return { claims, warnings };
131
+ }
132
+ /** Strip citation `[n]` and claim `{cX}` markers, collapse whitespace. */
133
+ function cleanClaimText(segment) {
134
+ return collapseWhitespace(segment.replace(/\[\d+\]/g, '').replace(/\{c\d+\}/g, ''));
135
+ }
136
+ /** Remove all `{cX}` claim markers from a text. */
137
+ export function stripClaimMarkers(text) {
138
+ return String(text).replace(/\{c\d+\}/g, '');
139
+ }
140
+ /**
141
+ * Parse a raw model answer end-to-end: strip the appendix, extract claims and
142
+ * evidence, and cross-check completeness (every cited (claim, source) pair
143
+ * should have exactly one evidence item, and vice versa).
144
+ */
145
+ export function parseAnswer(answer) {
146
+ const warnings = [];
147
+ const appendix = parseEvi1Appendix(answer);
148
+ const evidence = appendix?.items ?? [];
149
+ if (appendix)
150
+ warnings.push(...appendix.warnings);
151
+ const body = stripEvi1Appendix(answer);
152
+ const { claims, warnings: claimWarnings } = extractClaims(body);
153
+ warnings.push(...claimWarnings);
154
+ const cited = new Set(claims.flatMap((c) => c.sourceIndexes.map((n) => `${c.id}|${n}`)));
155
+ const evidenced = new Set();
156
+ const dedupedEvidence = [];
157
+ for (const it of evidence) {
158
+ const key = `${it.claimId}|${it.sourceIndex}`;
159
+ if (evidenced.has(key)) {
160
+ warnings.push(`EVI1: duplicate evidence for ${key}; keeping first occurrence.`);
161
+ continue;
162
+ }
163
+ evidenced.add(key);
164
+ dedupedEvidence.push(it);
165
+ if (!cited.has(key)) {
166
+ warnings.push(`EVI1: evidence for ${key} has no matching citation in the answer.`);
167
+ }
168
+ }
169
+ for (const key of cited) {
170
+ if (!evidenced.has(key)) {
171
+ warnings.push(`Citation ${key} has no evidence item in the EVI1 appendix.`);
172
+ }
173
+ }
174
+ return { cleanText: stripClaimMarkers(body), claims, evidence: dedupedEvidence, warnings };
175
+ }
176
+ function truncate(s, max) {
177
+ return s.length <= max ? s : `${s.slice(0, max - 1)}…`;
178
+ }
179
+ //# sourceMappingURL=evi1.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"evi1.js","sourceRoot":"","sources":["../../src/protocol/evi1.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC;AACjC,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAC;AAEnC,MAAM,aAAa,GAAG,8BAA8B,CAAC;AACrD,MAAM,YAAY,GAAG,aAAa,CAAC;AACnC,MAAM,2BAA2B,GAAG,iBAAiB,CAAC;AAEtD,0EAA0E;AAC1E,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,0DAA0D;AAC1D,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AASD,SAAS,cAAc,CAAC,KAAe;IACrC,2EAA2E;IAC3E,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;QACjD,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,QAAQ;YAAE,SAAS;QAC7C,KAAK,IAAI,KAAK,GAAG,GAAG,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9C,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,UAAU;gBAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QACpF,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAc;IAEd,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QACrD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACpC,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,0CAA0C,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9E,SAAS;QACX,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACrD,QAAQ,CAAC,IAAI,CAAC,uCAAuC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;YAC3E,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,CAAC,GAAG;QAAE,OAAO,MAAM,CAAC;IACxB,OAAO,KAAK;SACT,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC;SACvB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;SACpC,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,EAAE,CAAC;AACf,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,qBAAqB,CAAC,KAAqB;IACzD,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CACpB,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,WAAW,KAAK,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CACrE,CAAC;IACF,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACpD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QACpC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,YAAY,CAAC,SAAS,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnD,YAAY,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAErC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC9D,MAAM,aAAa,GAAG,UAAU;gBAC9B,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACjF,CAAC,CAAC,EAAE,CAAC;YACP,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,6CAA6C,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,iDAAiD,CAAC,CAAC;gBAC5E,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACb,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE;gBACF,IAAI,EAAE,cAAc,CAAC,OAAO,CAAC;gBAC7B,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;AAC9B,CAAC;AAED,0EAA0E;AAC1E,SAAS,cAAc,CAAC,OAAe;IACrC,OAAO,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;AACtF,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC;IACvC,IAAI,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAElD,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChE,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;IAEhC,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAClE,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,eAAe,GAAoB,EAAE,CAAC;IAC5C,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,CAAC,IAAI,CAAC,gCAAgC,GAAG,6BAA6B,CAAC,CAAC;YAChF,SAAS;QACX,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC,sBAAsB,GAAG,0CAA0C,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,QAAQ,CAAC,IAAI,CAAC,YAAY,GAAG,6CAA6C,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;AAC7F,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAW;IACtC,OAAO,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;AACzD,CAAC"}
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Canonical instruction block for the answering model. Inject the returned
3
+ * text into the system prompt of any assistant that receives numbered web
4
+ * sources, then verify its output with `verifyAnswer`.
5
+ */
6
+ export interface CitationPromptOptions {
7
+ /** Maximum number of cited sentences/bullets per answer. Default 15. */
8
+ maxCitedClaims?: number;
9
+ /** Maximum citations per sentence/bullet. Default 2. */
10
+ maxCitationsPerClaim?: number;
11
+ /** Preferred quote length range in characters. Default [80, 240]. */
12
+ quoteLengthRange?: [number, number];
13
+ /** Minimum quote length in characters (with stated exceptions). Default 60. */
14
+ minQuoteLength?: number;
15
+ }
16
+ /** Build the EVI1 citation instruction block for the answering model. */
17
+ export declare function buildCitationInstructions(options?: CitationPromptOptions): string;
18
+ //# sourceMappingURL=prompt.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../../src/protocol/prompt.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,qBAAqB;IACpC,wEAAwE;IACxE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,wDAAwD;IACxD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,+EAA+E;IAC/E,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,yEAAyE;AACzE,wBAAgB,yBAAyB,CAAC,OAAO,GAAE,qBAA0B,GAAG,MAAM,CAgFrF"}
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Canonical instruction block for the answering model. Inject the returned
3
+ * text into the system prompt of any assistant that receives numbered web
4
+ * sources, then verify its output with `verifyAnswer`.
5
+ */
6
+ /** Build the EVI1 citation instruction block for the answering model. */
7
+ export function buildCitationInstructions(options = {}) {
8
+ const { maxCitedClaims = 15, maxCitationsPerClaim = 2, quoteLengthRange = [80, 240], minQuoteLength = 60, } = options;
9
+ return `
10
+ [CITATION BUDGET]
11
+ - Keep citations sparse. Only cite new, load-bearing information.
12
+ - Maximum cited sentences/bullets: ${maxCitedClaims} total in the entire answer.
13
+ - Maximum citations per cited sentence/bullet: ${maxCitationsPerClaim}. Prefer 1.
14
+ - Prefer citing only:
15
+ (1) key numeric results/effect sizes,
16
+ (2) central conclusions,
17
+ (3) safety-critical claims,
18
+ (4) definitions that are not common knowledge.
19
+ - Everything else should be uncited explanation/synthesis.
20
+
21
+ [CITATIONS - inline markers]
22
+ - Cite sources inline using numeric markers like [n] immediately after the exact sentence/bullet they support.
23
+ - Multiple citations are written as [1][3] (no commas, no spaces).
24
+ - NEVER cite a source that does not directly support the statement in that sentence/bullet.
25
+ - NEVER invent sources. Only use the provided indices.
26
+ - Do NOT add a "Sources" section at the end. Citations must appear inline only.
27
+
28
+ [CLAIM MARKERS - stable mapping (MANDATORY)]
29
+ - A citation group of one or more markers like [n] or [n][m] must be immediately
30
+ followed by a claim marker {cX} with no characters in between.
31
+ Example: "...water expands when freezing.[2]{c1}"
32
+ Example: "...water expands when freezing.[2][5]{c1}"
33
+ Bad: "...freezing.[2] {c1}"
34
+ Bad: "...freezing.[2].{c1}"
35
+ - Every sentence OR bullet that contains citations MUST end with exactly ONE claim marker {cX}.
36
+ - X starts at 1 and increases by 1 for each new cited sentence/bullet in THIS answer.
37
+ - Do NOT place {cX} anywhere else. Uncited sentences get no claim marker.
38
+
39
+ [QUOTE-BASED EVIDENCE APPENDIX (MANDATORY)]
40
+ After you finish the answer, append an appendix:
41
+ - Start with a line containing exactly: EVI1
42
+ - Then output one line per (claim, source) pair in this exact format:
43
+ cX|n|"QUOTE"
44
+ - End with a line containing exactly: END_EVI1
45
+ - Do NOT output anything after END_EVI1.
46
+ - Do NOT wrap EVI1..END_EVI1 in code fences or markdown.
47
+
48
+ [COMPLETENESS RULES (MANDATORY)]
49
+ - For EVERY cited sentence/bullet {cX}, include an EVI1 line for EACH cited source index [n].
50
+ Example: if the text contains "...[2][5]{c3}", EVI1 MUST contain both:
51
+ c3|2|"..."
52
+ c3|5|"..."
53
+ - If you cannot provide a verbatim quote for a citation, REMOVE that citation
54
+ from the answer. Never leave a citation unverified.
55
+
56
+ Rules for QUOTE:
57
+ - QUOTE must be copied verbatim from the provided source TEXT (not paraphrased).
58
+ - QUOTE must be sufficient to support the claim; prefer the shortest quote that
59
+ still supports it (typically ${quoteLengthRange[0]}-${quoteLengthRange[1]} characters).
60
+ - QUOTE must be at least ${minQuoteLength} characters unless it contains a numeric result or is
61
+ a complete standalone sentence that uniquely supports the claim.
62
+ - QUOTE must be a single line: escape newlines as \\n and double quotes as \\".
63
+
64
+ [SCOPE AND STRENGTH DISCIPLINE (MANDATORY)]
65
+ - A cited claim must match its quote in meaning, scope, and outcome/topic. If the
66
+ quote discusses a different outcome, rewrite the claim or drop the citation.
67
+ - Never expand categories beyond what the quote explicitly states.
68
+ - Never infer missing definitions; if a key term is not defined in the provided
69
+ text, keep the statement generic or label it as unclear.
70
+ - Use absolute words ("proves", "always", "never") only when the quote clearly
71
+ supports that strength. Do not upgrade hedged evidence ("may", "associated")
72
+ to certainty.
73
+ - Prefer sources whose provided text shows the methods/assumptions behind a
74
+ claim; when only conclusions are visible, use cautious language.
75
+
76
+ IMPORTANT:
77
+ - The user-facing answer must end BEFORE the EVI1 appendix starts.
78
+ - The appendix is required only when you used citations. If you used no
79
+ citations, do NOT output EVI1/END_EVI1.
80
+ `.trim();
81
+ }
82
+ //# sourceMappingURL=prompt.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"prompt.js","sourceRoot":"","sources":["../../src/protocol/prompt.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAaH,yEAAyE;AACzE,MAAM,UAAU,yBAAyB,CAAC,UAAiC,EAAE;IAC3E,MAAM,EACJ,cAAc,GAAG,EAAE,EACnB,oBAAoB,GAAG,CAAC,EACxB,gBAAgB,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAC5B,cAAc,GAAG,EAAE,GACpB,GAAG,OAAO,CAAC;IAEZ,OAAO;;;qCAG4B,cAAc;iDACF,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iCA8CpC,gBAAgB,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC;2BAChD,cAAc;;;;;;;;;;;;;;;;;;;;CAoBxC,CAAC,IAAI,EAAE,CAAC;AACT,CAAC"}
@@ -0,0 +1,24 @@
1
+ /**
2
+ * End-to-end verification: parse an EVI1 answer, match every quote against
3
+ * its source deterministically, optionally run the entailment judge, and
4
+ * aggregate everything into a transparency report.
5
+ */
6
+ import type { EntailmentJudge, SourceDocument, VerificationReport } from './types.js';
7
+ import { type MatchOptions } from './match/fuzzy.js';
8
+ export interface VerifyOptions {
9
+ /** Raw model output, including the EVI1 appendix. */
10
+ answer: string;
11
+ /** Sources in citation order: `sources[0]` is `[1]`. */
12
+ sources: SourceDocument[];
13
+ /** Optional semantic judge; without it the report is text-match only. */
14
+ judge?: EntailmentJudge;
15
+ /** Options for the deterministic matcher. */
16
+ match?: MatchOptions;
17
+ /** Characters of source context around the match given to the judge. Default 420. */
18
+ contextWindowChars?: number;
19
+ /** Abort signal forwarded to the judge. */
20
+ signal?: AbortSignal;
21
+ }
22
+ /** Verify one answer against its sources. */
23
+ export declare function verifyAnswer(options: VerifyOptions): Promise<VerificationReport>;
24
+ //# sourceMappingURL=report.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAGV,eAAe,EAEf,cAAc,EACd,kBAAkB,EAEnB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAA2B,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAE9E,MAAM,WAAW,aAAa;IAC5B,qDAAqD;IACrD,MAAM,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,yEAAyE;IACzE,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,6CAA6C;IAC7C,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,qFAAqF;IACrF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,2CAA2C;IAC3C,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,6CAA6C;AAC7C,wBAAsB,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA2EtF"}
package/dist/report.js ADDED
@@ -0,0 +1,106 @@
1
+ /**
2
+ * End-to-end verification: parse an EVI1 answer, match every quote against
3
+ * its source deterministically, optionally run the entailment judge, and
4
+ * aggregate everything into a transparency report.
5
+ */
6
+ import { parseAnswer } from './protocol/evi1.js';
7
+ import { matchQuoteAgainstSource } from './match/fuzzy.js';
8
+ /** Verify one answer against its sources. */
9
+ export async function verifyAnswer(options) {
10
+ const { answer, sources, judge, match, signal } = options;
11
+ const contextWindowChars = options.contextWindowChars ?? 420;
12
+ const parsed = parseAnswer(answer);
13
+ const warnings = [...parsed.warnings];
14
+ const claimById = new Map(parsed.claims.map((c) => [c.id, c]));
15
+ const citations = [];
16
+ for (const item of parsed.evidence) {
17
+ const claim = claimById.get(item.claimId);
18
+ const source = sources[item.sourceIndex - 1];
19
+ if (!source) {
20
+ warnings.push(`EVI1: evidence ${item.claimId}|${item.sourceIndex} references a source that was not provided.`);
21
+ continue;
22
+ }
23
+ const textMatch = matchQuoteAgainstSource(item.quote, source, match);
24
+ citations.push({
25
+ claimId: item.claimId,
26
+ sourceIndex: item.sourceIndex,
27
+ claimText: claim?.text ?? '',
28
+ quote: item.quote,
29
+ textMatch,
30
+ score: textMatch.score,
31
+ });
32
+ }
33
+ if (judge) {
34
+ const judgeable = citations.filter((c) => c.claimText && c.quote);
35
+ const inputs = judgeable.map((c) => ({
36
+ id: `${c.claimId}|${c.sourceIndex}`,
37
+ claim: c.claimText,
38
+ quote: c.quote,
39
+ context: contextWindow(sources[c.sourceIndex - 1], c.textMatch, contextWindowChars),
40
+ }));
41
+ const results = await judge.judge(inputs, { signal });
42
+ if (results.length !== inputs.length) {
43
+ warnings.push(`Judge returned ${results.length} results for ${inputs.length} items; missing items marked as errors.`);
44
+ }
45
+ judgeable.forEach((c, i) => {
46
+ const entailment = results[i] ?? {
47
+ class: 'error',
48
+ confidence: null,
49
+ reasons: ['missing_item'],
50
+ };
51
+ c.entailment = entailment;
52
+ c.score =
53
+ entailment.class === 'error' || entailment.confidence === null
54
+ ? null
55
+ : Math.min(c.textMatch.score, entailment.confidence);
56
+ });
57
+ for (const c of citations) {
58
+ if (!c.claimText) {
59
+ warnings.push(`Citation ${c.claimId}|${c.sourceIndex}: no claim text found for entailment check.`);
60
+ }
61
+ }
62
+ }
63
+ return {
64
+ cleanText: parsed.cleanText,
65
+ claims: parsed.claims,
66
+ citations,
67
+ warnings,
68
+ summary: summarize(citations, Boolean(judge)),
69
+ };
70
+ }
71
+ /** Source text around the matched region, for judge disambiguation. */
72
+ function contextWindow(source, matchResult, windowChars) {
73
+ const text = source?.text ?? '';
74
+ if (!text)
75
+ return '';
76
+ if (matchResult.start !== undefined && matchResult.field === 'text') {
77
+ const start = Math.max(0, matchResult.start - windowChars);
78
+ const end = Math.min(text.length, (matchResult.end ?? matchResult.start) + windowChars);
79
+ return text.slice(start, end);
80
+ }
81
+ return text.slice(0, windowChars * 2);
82
+ }
83
+ function summarize(citations, judged) {
84
+ const n = citations.length;
85
+ if (n === 0) {
86
+ return {
87
+ citationCount: 0,
88
+ verbatimRate: null,
89
+ entailedRate: null,
90
+ meanScore: null,
91
+ minScore: null,
92
+ };
93
+ }
94
+ const verbatim = citations.filter((c) => c.textMatch.method === 'exact' || c.textMatch.method === 'normalized').length;
95
+ const judgedCitations = citations.filter((c) => c.entailment);
96
+ const entailed = judgedCitations.filter((c) => c.entailment?.class === 'entailed').length;
97
+ const scores = citations.map((c) => c.score).filter((s) => s !== null);
98
+ return {
99
+ citationCount: n,
100
+ verbatimRate: verbatim / n,
101
+ entailedRate: judged && judgedCitations.length ? entailed / judgedCitations.length : null,
102
+ meanScore: scores.length ? scores.reduce((a, b) => a + b, 0) / scores.length : null,
103
+ minScore: scores.length ? Math.min(...scores) : null,
104
+ };
105
+ }
106
+ //# sourceMappingURL=report.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAqB,MAAM,kBAAkB,CAAC;AAiB9E,6CAA6C;AAC7C,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAsB;IACvD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC1D,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,GAAG,CAAC;IAE7D,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/D,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,QAAQ,CAAC,IAAI,CACX,kBAAkB,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,6CAA6C,CAChG,CAAC;YACF,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAe,uBAAuB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACjF,SAAS,CAAC,IAAI,CAAC;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,KAAK,EAAE,IAAI,IAAI,EAAE;YAC5B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS;YACT,KAAK,EAAE,SAAS,CAAC,KAAK;SACvB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;QAClE,MAAM,MAAM,GAAsB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtD,EAAE,EAAE,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,WAAW,EAAE;YACnC,KAAK,EAAE,CAAC,CAAC,SAAS;YAClB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,aAAa,CACpB,OAAO,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,EAC1B,CAAC,CAAC,SAAS,EACX,kBAAkB,CACnB;SACF,CAAC,CAAC,CAAC;QACJ,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CACX,kBAAkB,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,yCAAyC,CACvG,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACzB,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI;gBAC/B,KAAK,EAAE,OAAgB;gBACvB,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE,CAAC,cAAc,CAAC;aAC1B,CAAC;YACF,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC;YAC1B,CAAC,CAAC,KAAK;gBACL,UAAU,CAAC,KAAK,KAAK,OAAO,IAAI,UAAU,CAAC,UAAU,KAAK,IAAI;oBAC5D,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CACX,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,WAAW,6CAA6C,CACpF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS;QACT,QAAQ;QACR,OAAO,EAAE,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,SAAS,aAAa,CACpB,MAAkC,EAClC,WAAuB,EACvB,WAAmB;IAEnB,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,KAAK,GAAG,WAAW,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC,CAAC;QACxF,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,SAAS,CAAC,SAAiC,EAAE,MAAe;IACnE,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IAC3B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO;YACL,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,IAAI;YAClB,YAAY,EAAE,IAAI;YAClB,SAAS,EAAE,IAAI;YACf,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAC/B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,OAAO,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,YAAY,CAC7E,CAAC,MAAM,CAAC;IACT,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAC1F,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACpF,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,QAAQ,GAAG,CAAC;QAC1B,YAAY,EAAE,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACzF,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACnF,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;KACrD,CAAC;AACJ,CAAC"}
@@ -0,0 +1,141 @@
1
+ /**
2
+ * Core data model shared across the VeriQuote pipeline.
3
+ */
4
+ /** A retrieved source document that the assistant may cite as `[n]`. */
5
+ export interface SourceDocument {
6
+ /** Optional stable identifier (URL, DOI, internal id). Not used for matching. */
7
+ id?: string;
8
+ /** Human-readable title. */
9
+ title?: string;
10
+ /** Canonical URL of the source, if any. */
11
+ url?: string;
12
+ /** Main extracted text of the source. Quotes are verified against this first. */
13
+ text: string;
14
+ /**
15
+ * Additional text fields to search when the quote is not found in `text`
16
+ * (e.g. abstract, search-result snippet, title). Searched in order.
17
+ */
18
+ extraTexts?: string[];
19
+ }
20
+ /** One `cX|n|"QUOTE"` line from an EVI1 appendix. */
21
+ export interface EvidenceItem {
22
+ /** Claim identifier, e.g. `"c1"`. */
23
+ claimId: string;
24
+ /** 1-based source index as used in `[n]` citation markers. */
25
+ sourceIndex: number;
26
+ /** Verbatim quote the model attributes to the source. */
27
+ quote: string;
28
+ }
29
+ /** A cited claim extracted from the answer body via `{cX}` markers. */
30
+ export interface Claim {
31
+ /** Claim identifier, e.g. `"c1"`. */
32
+ id: string;
33
+ /** Claim text with citation and claim markers removed. */
34
+ text: string;
35
+ /** 1-based source indexes cited directly before the claim marker. */
36
+ sourceIndexes: number[];
37
+ }
38
+ /** Result of parsing a raw model answer that follows the EVI1 protocol. */
39
+ export interface ParsedAnswer {
40
+ /** Answer body with the EVI1 appendix and all `{cX}` markers removed. */
41
+ cleanText: string;
42
+ /** Claims found in the answer body. */
43
+ claims: Claim[];
44
+ /** Evidence items found in the EVI1 appendix. */
45
+ evidence: EvidenceItem[];
46
+ /** Protocol violations detected during parsing (non-fatal). */
47
+ warnings: string[];
48
+ }
49
+ /** How a quote was located inside a source. */
50
+ export type MatchMethod =
51
+ /** Byte-for-byte substring of the raw source text. */
52
+ 'exact'
53
+ /** Substring after Unicode/typography normalization and case folding. */
54
+ | 'normalized'
55
+ /** Best fuzzy window by character-trigram Dice similarity. */
56
+ | 'fuzzy'
57
+ /** No window reached the fuzzy score threshold. */
58
+ | 'not_found';
59
+ /** Deterministic text-match result for one (quote, source) pair. */
60
+ export interface QuoteMatch {
61
+ method: MatchMethod;
62
+ /**
63
+ * Similarity in [0, 1]. `1` only for exact/normalized hits; fuzzy scores
64
+ * are capped at 0.99 so a perfect score always implies a literal hit.
65
+ */
66
+ score: number;
67
+ /** Start offset of the matched region in the original source field, if known. */
68
+ start?: number;
69
+ /** End offset (exclusive) of the matched region, if known. */
70
+ end?: number;
71
+ /** Which source field matched: `"text"` or `"extraTexts[i]"`. */
72
+ field: string;
73
+ }
74
+ /** Qualitative entailment classes produced by the LLM judge. */
75
+ export type EntailmentClass = 'entailed' | 'partially_entailed' | 'overstated' | 'insufficient' | 'contradicted';
76
+ /** Judge output for one (claim, quote) pair. */
77
+ export interface EntailmentResult {
78
+ /** Entailment class, or `"error"` when the judge failed for this item. */
79
+ class: EntailmentClass | 'error';
80
+ /** Degree of support in [0, 1]; `null` when unavailable. */
81
+ confidence: number | null;
82
+ /** Short judge rationales (at most two, in the claim's language). */
83
+ reasons: string[];
84
+ }
85
+ /** Input unit for an entailment judge. */
86
+ export interface EntailmentInput {
87
+ /** Caller-chosen identifier echoed back in the result. */
88
+ id: string;
89
+ /** The claim as stated in the answer. */
90
+ claim: string;
91
+ /** The verbatim quote attributed to the source. */
92
+ quote: string;
93
+ /** Surrounding source text for disambiguation (may be empty). */
94
+ context: string;
95
+ }
96
+ /**
97
+ * Pluggable semantic verifier. Implementations must be deterministic for
98
+ * reproducibility (e.g. temperature 0) and must return exactly one result
99
+ * per input, in input order, using `class: "error"` for individual failures.
100
+ */
101
+ export interface EntailmentJudge {
102
+ judge(items: EntailmentInput[], options?: {
103
+ signal?: AbortSignal;
104
+ }): Promise<EntailmentResult[]>;
105
+ }
106
+ /** Combined verification result for one (claim, source) citation. */
107
+ export interface CitationVerification {
108
+ claimId: string;
109
+ sourceIndex: number;
110
+ claimText: string;
111
+ quote: string;
112
+ textMatch: QuoteMatch;
113
+ /** Absent when verification ran without a judge. */
114
+ entailment?: EntailmentResult;
115
+ /**
116
+ * Conservative combined score: `min(textMatch.score, entailment.confidence)`.
117
+ * When no judge ran, equals `textMatch.score`. `null` if the judge errored.
118
+ */
119
+ score: number | null;
120
+ }
121
+ /** Aggregate statistics over all citations in one answer. */
122
+ export interface VerificationSummary {
123
+ citationCount: number;
124
+ /** Share of citations whose quote is a literal (exact or normalized) hit. */
125
+ verbatimRate: number | null;
126
+ /** Share of judged citations classified `entailed`; `null` without a judge. */
127
+ entailedRate: number | null;
128
+ /** Mean of non-null combined scores; `null` if none. */
129
+ meanScore: number | null;
130
+ /** Minimum of non-null combined scores; `null` if none. */
131
+ minScore: number | null;
132
+ }
133
+ /** Full transparency report for one answer. */
134
+ export interface VerificationReport {
135
+ cleanText: string;
136
+ claims: Claim[];
137
+ citations: CitationVerification[];
138
+ warnings: string[];
139
+ summary: VerificationSummary;
140
+ }
141
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,wEAAwE;AACxE,MAAM,WAAW,cAAc;IAC7B,iFAAiF;IACjF,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,4BAA4B;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qDAAqD;AACrD,MAAM,WAAW,YAAY;IAC3B,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,uEAAuE;AACvE,MAAM,WAAW,KAAK;IACpB,qCAAqC;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,2EAA2E;AAC3E,MAAM,WAAW,YAAY;IAC3B,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,iDAAiD;IACjD,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,+CAA+C;AAC/C,MAAM,MAAM,WAAW;AACrB,sDAAsD;AACpD,OAAO;AACT,yEAAyE;GACvE,YAAY;AACd,8DAA8D;GAC5D,OAAO;AACT,mDAAmD;GACjD,WAAW,CAAC;AAEhB,oEAAoE;AACpE,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,WAAW,CAAC;IACpB;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;CACf;AAED,gEAAgE;AAChE,MAAM,MAAM,eAAe,GACvB,UAAU,GACV,oBAAoB,GACpB,YAAY,GACZ,cAAc,GACd,cAAc,CAAC;AAEnB,gDAAgD;AAChD,MAAM,WAAW,gBAAgB;IAC/B,0EAA0E;IAC1E,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC;IACjC,4DAA4D;IAC5D,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,qEAAqE;IACrE,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,0CAA0C;AAC1C,MAAM,WAAW,eAAe;IAC9B,0DAA0D;IAC1D,EAAE,EAAE,MAAM,CAAC;IACX,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,KAAK,EAAE,eAAe,EAAE,EAAE,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAClG;AAED,qEAAqE;AACrE,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,UAAU,CAAC;IACtB,oDAAoD;IACpD,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;OAGG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,6DAA6D;AAC7D,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,+EAA+E;IAC/E,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,wDAAwD;IACxD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,SAAS,EAAE,oBAAoB,EAAE,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE,mBAAmB,CAAC;CAC9B"}
package/dist/types.js ADDED
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Core data model shared across the VeriQuote pipeline.
3
+ */
4
+ export {};
5
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}
package/package.json ADDED
@@ -0,0 +1,54 @@
1
+ {
2
+ "name": "veriquote",
3
+ "version": "0.1.1",
4
+ "description": "Deterministic + semantic verification of quote-grounded LLM citations (EVI1 protocol): fuzzy verbatim-quote matching and an LLM entailment judge for transparent, per-claim hallucination detection.",
5
+ "keywords": [
6
+ "llm",
7
+ "rag",
8
+ "citations",
9
+ "hallucination",
10
+ "groundedness",
11
+ "verification",
12
+ "nli",
13
+ "entailment",
14
+ "fact-checking",
15
+ "attribution"
16
+ ],
17
+ "license": "MIT",
18
+ "author": "NavigNine (https://navignine.com)",
19
+ "repository": {
20
+ "type": "git",
21
+ "url": "https://github.com/rickintoplace/veriquote.git"
22
+ },
23
+ "type": "module",
24
+ "main": "./dist/index.js",
25
+ "types": "./dist/index.d.ts",
26
+ "exports": {
27
+ ".": {
28
+ "types": "./dist/index.d.ts",
29
+ "import": "./dist/index.js"
30
+ }
31
+ },
32
+ "files": [
33
+ "dist",
34
+ "src",
35
+ "README.md",
36
+ "LICENSE",
37
+ "CITATION.cff"
38
+ ],
39
+ "sideEffects": false,
40
+ "engines": {
41
+ "node": ">=18"
42
+ },
43
+ "scripts": {
44
+ "build": "tsc -p tsconfig.build.json",
45
+ "test": "vitest run",
46
+ "test:watch": "vitest",
47
+ "typecheck": "tsc --noEmit",
48
+ "prepublishOnly": "npm run typecheck && npm run test && npm run build"
49
+ },
50
+ "devDependencies": {
51
+ "typescript": "^5.5.0",
52
+ "vitest": "^3.0.0"
53
+ }
54
+ }