reploid 0.0.0-stage → 0.1.1

package/packages/reploid/src/retrieval/page-text.js

@@ -0,0 +1,182 @@
+/**
+ * @fileoverview Text extraction from fetched pages, shared by the server and device-side extractors so the same
+ * bytes yield the same document text and hash on every route. HTML is reduced to readable paragraphs without a DOM:
+ * scripts, styles and page chrome are dropped, the main content region is preferred, and block elements
+ * become paragraph breaks. Plain text is decoded as-is. PDF and other types are refused before this point.
+ * Environment-neutral: no Node or DOM APIs.
+ */
+
+// A tag whose quoted attributes may contain '>'.
+const ATTRS = String.raw`(?:[^>"']|"[^"]*"|'[^']*')*`;
+const TAG = new RegExp(String.raw`<(\/?)([a-zA-Z][a-zA-Z0-9:-]*)(${ATTRS})>`, 'g');
+const RAW_TEXT = ['script', 'style', 'noscript', 'template', 'svg', 'iframe', 'object', 'canvas', 'math'];
+const CHROME = ['nav', 'header', 'footer', 'aside', 'select', 'button', 'textarea', 'dialog'];
+const BLOCK = new Set(['p', 'div', 'section', 'article', 'main', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'dl', 'table',
+  'thead', 'tbody', 'tfoot', 'blockquote', 'pre', 'figure', 'figcaption', 'address', 'hr', 'details', 'summary', 'fieldset',
+  'legend', 'caption', 'center', 'body', 'html', 'form']);
+const LINE = new Set(['li', 'dt', 'dd', 'br', 'tr', 'option']);
+const CELL = new Set(['td', 'th']);
+
+const NAMED = {
+  amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', zwnj: '', zwj: '', shy: '',
+  ndash: '–', mdash: '—', lsquo: '‘', rsquo: '’', sbquo: '‚', ldquo: '“', rdquo: '”', bdquo: '„', hellip: '…', middot: '·',
+  bull: '•', laquo: '«', raquo: '»', lsaquo: '‹', rsaquo: '›', prime: '′', Prime: '″', dagger: '†', Dagger: '‡', permil: '‰',
+  copy: '©', reg: '®', trade: '™', deg: '°', plusmn: '±', times: '×', divide: '÷', minus: '−', frac12: '½', frac14: '¼',
+  frac34: '¾', sup1: '¹', sup2: '²', sup3: '³', micro: 'µ', para: '¶', sect: '§', cent: '¢', pound: '£', euro: '€', yen: '¥',
+  curren: '¤', iexcl: '¡', iquest: '¿', ordf: 'ª', ordm: 'º', not: '¬', macr: '¯', acute: '´', cedil: '¸', uml: '¨', brvbar: '¦',
+  le: '≤', ge: '≥', ne: '≠', asymp: '≈', infin: '∞', larr: '←', rarr: '→', uarr: '↑', darr: '↓', harr: '\u2194', szlig: 'ß',
+  alpha: 'α', beta: 'β', gamma: 'γ', delta: 'δ', epsilon: 'ε', lambda: 'λ', mu: 'μ', pi: 'π', sigma: 'σ', tau: 'τ', phi: 'φ',
+  omega: 'ω', Delta: 'Δ', Sigma: 'Σ', Omega: 'Ω'
+};
+// Latin-1 letters by name: agrave, Agrave, eacute, ...
+const LETTERS = { a: 'àáâãäå', e: 'èéêë', i: 'ìíîï', o: 'òóôõöø', u: 'ùúûü', y: 'ýÿ', n: 'ñ', c: 'ç' };
+const MARKS = { a: ['grave', 'acute', 'circ', 'tilde', 'uml', 'ring'], e: ['grave', 'acute', 'circ', 'uml'],
+  i: ['grave', 'acute', 'circ', 'uml'], o: ['grave', 'acute', 'circ', 'tilde', 'uml', 'slash'], u: ['grave', 'acute', 'circ', 'uml'],
+  y: ['acute', 'uml'], n: ['tilde'], c: ['cedil'] };
+for (const [base, letters] of Object.entries(LETTERS)) {
+  [...letters].forEach((letter, index) => {
+    NAMED[base + MARKS[base][index]] = letter;
+    const upper = letter.toUpperCase();
+    if (upper !== letter && !(base === 'y' && index === 1)) NAMED[base.toUpperCase() + MARKS[base][index]] = upper;
+  });
+}
+Object.assign(NAMED, { aelig: 'æ', AElig: 'Æ', eth: 'ð', ETH: 'Ð', thorn: 'þ', THORN: 'Þ', oelig: 'œ', OElig: 'Œ', Yuml: 'Ÿ' });
+// Legacy names browsers accept without a semicolon.
+const BARE = new Set(['amp', 'lt', 'gt', 'quot', 'nbsp', 'copy', 'reg']);
+
+const codePoint = value => (Number.isSafeInteger(value) && value > 0 && value <= 0x10ffff && !(value >= 0xd800 && value <= 0xdfff))
+  ? String.fromCodePoint(value) : '�';
+
+/** Decodes numeric and common named character references; unknown names stay as written. */
+export function decodeEntities(text) {
+  return String(text).replace(/&(?:#(\d{1,8})|#[xX]([0-9a-fA-F]{1,7})|([a-zA-Z][a-zA-Z0-9]{1,31}))(;?)/g, (match, decimal, hex, name, semi) => {
+    if (decimal) return codePoint(Number(decimal));
+    if (hex) return codePoint(parseInt(hex, 16));
+    if (!Object.hasOwn(NAMED, name) || (!semi && !BARE.has(name))) return match;
+    return NAMED[name];
+  });
+}
+
+/** Removes tags and decodes entities from a short HTML fragment, such as a search result title or snippet. */
+export function htmlFragmentText(fragment) {
+  return decodeEntities(String(fragment ?? '').replace(/<!--[\s\S]*?(?:-->|$)/g, ' ')
+    .replace(TAG, (match, closing, name) => BLOCK.has(name.toLowerCase()) || LINE.has(name.toLowerCase()) ? ' ' : ''))
+    .replace(/[\s ]+/g, ' ').trim();
+}
+
+const anyTag = name => new RegExp(String.raw`<(\/?)${name}(?=[\s/>])${ATTRS}>`, 'gi');
+
+/** Balanced elements named `name`: [{ start, innerStart, innerEnd, end }]. Unclosed elements run to the end. */
+function elements(html, name) {
+  const found = [], pattern = anyTag(name);
+  let depth = 0, current = null, match;
+  while ((match = pattern.exec(html))) {
+    if (!match[1]) {
+      if (match[0].endsWith('/>')) continue;
+      if (depth === 0) current = { start: match.index, innerStart: match.index + match[0].length };
+      depth++;
+    } else if (depth > 0 && --depth === 0) {
+      found.push({ ...current, innerEnd: match.index, end: match.index + match[0].length });
+      current = null;
+    }
+  }
+  if (current) found.push({ ...current, innerEnd: html.length, end: html.length, unclosed: true });
+  return found;
+}
+
+/** Removes every `name` element with its content; an unclosed one loses only its opening tag. */
+function removeElements(html, name) {
+  let output = '', cursor = 0;
+  for (const element of elements(html, name)) {
+    if (element.start < cursor) continue;
+    output += html.slice(cursor, element.start) + ' ';
+    cursor = element.unclosed ? element.innerStart : element.end;
+  }
+  return output + html.slice(cursor);
+}
+
+function removeRawText(html) {
+  // Raw-text elements cannot nest; an unclosed one hides everything after it, as browsers do.
+  const pattern = new RegExp(String.raw`<(${RAW_TEXT.join('|')})(?=[\s/>])${ATTRS}>[\s\S]*?(?:<\/\1\s*>|$)`, 'gi');
+  return html.replace(pattern, ' ');
+}
+
+function region(html) {
+  const main = elements(html, 'main')[0];
+  if (main) return html.slice(main.innerStart, main.innerEnd);
+  const articles = elements(html, 'article');
+  if (articles.length) {
+    const largest = articles.reduce((best, item) => item.innerEnd - item.innerStart > best.innerEnd - best.innerStart ? item : best);
+    return html.slice(largest.innerStart, largest.innerEnd);
+  }
+  const body = elements(html, 'body')[0];
+  return body ? html.slice(body.innerStart, body.innerEnd) : html;
+}
+
+function render(html) {
+  const marked = html.replace(/[\s ]+/g, ' ').replace(TAG, (match, closing, name) => {
+    const tag = name.toLowerCase();
+    if (BLOCK.has(tag)) return '\n\n';
+    if (LINE.has(tag)) return closing ? '' : '\n';
+    if (CELL.has(tag)) return ' ';
+    return '';
+  });
+  return decodeEntities(marked).replace(/ /g, ' ').split('\n').map(line => line.replace(/[ \t\f\v]+/g, ' ').trim())
+    .join('\n').replace(/\n{3,}/g, '\n\n').trim();
+}
+
+function htmlText(source) {
+  let html = source.replace(/<!--[\s\S]*?(?:-->|$)/g, ' ').replace(/<!\[CDATA\[[\s\S]*?(?:\]\]>|$)/g, ' ')
+    .replace(/<![^>]*>|<\?[^>]*>/g, ' ');
+  html = removeRawText(html);
+  const titleMatch = html.match(new RegExp(String.raw`<title(?=[\s>])${ATTRS}>([\s\S]*?)<\/title\s*>`, 'i'));
+  const title = titleMatch ? htmlFragmentText(titleMatch[1]) : '';
+  for (const head of elements(html, 'head').filter(item => !item.unclosed).reverse()) html = html.slice(0, head.start) + ' ' + html.slice(head.end);
+  const chosen = region(html);
+  // Page chrome is dropped inside the chosen region; if that leaves nothing, the region is kept whole.
+  const cleaned = render(CHROME.reduce(removeElements, chosen));
+  return { text: cleaned || render(chosen), title };
+}
+
+const CHARSET = /charset\s*=\s*["']?([\w.:-]+)/i;
+/** The charset named by the header, else a <meta> declaration in the first 2 KB, else a byte-order mark, else UTF-8. */
+export function detectCharset(body, contentType = '', charset = null) {
+  if (charset) return charset;
+  const declared = String(contentType).match(CHARSET)?.[1];
+  if (declared) return declared;
+  if (body[0] === 0xef && body[1] === 0xbb && body[2] === 0xbf) return 'utf-8';
+  if (body[0] === 0xff && body[1] === 0xfe) return 'utf-16le';
+  if (body[0] === 0xfe && body[1] === 0xff) return 'utf-16be';
+  const head = Array.from(body.subarray(0, 2048), byte => String.fromCharCode(byte)).join('');
+  for (const meta of head.matchAll(new RegExp(String.raw`<meta(?=[\s/])${ATTRS}>`, 'gi'))) {
+    const direct = meta[0].match(/\bcharset\s*=\s*["']?([\w.:-]+)/i)?.[1];
+    if (direct) return direct;
+  }
+  return 'utf-8';
+}
+
+// windows-1252 bytes 0x80-0x9F. Some Node releases decode this label as ISO-8859-1, so it is mapped here.
+const CP1252 = '\u20ac\u0081\u201a\u0192\u201e\u2026\u2020\u2021\u02c6\u2030\u0160\u2039\u0152\u008d\u017d\u008f'
+  + '\u0090\u2018\u2019\u201c\u201d\u2022\u2013\u2014\u02dc\u2122\u0161\u203a\u0153\u009d\u017e\u0178';
+
+function decode(body, label) {
+  let decoder;
+  try { decoder = new TextDecoder(label); } catch { decoder = new TextDecoder('utf-8'); }
+  if (decoder.encoding !== 'windows-1252') return decoder.decode(body);
+  let text = '';
+  for (const byte of body) text += byte >= 0x80 && byte <= 0x9f ? CP1252[byte - 0x80] : String.fromCharCode(byte);
+  return text;
+}
+
+/** { text, title, characters } for an HTML or plain-text body. */
+export function extractDocumentText({ body, contentType = '', charset = null }) {
+  const bytes = body instanceof Uint8Array ? body : new TextEncoder().encode(String(body ?? ''));
+  const source = decode(bytes, detectCharset(bytes, contentType, charset)).replace(/^/, '');
+  const mime = String(contentType).split(';')[0].trim().toLowerCase();
+  if (mime === 'text/plain') {
+    const text = source.replace(/\r\n?/g, '\n').replace(/[ \t]+$/gm, '').replace(/\n{3,}/g, '\n\n').trim();
+    return { text, title: '', characters: text.length };
+  }
+  const { text, title } = htmlText(source);
+  return { text, title, characters: text.length };
+}

package/packages/reploid/src/retrieval/passage.js

@@ -0,0 +1,37 @@
+/**
+ * @fileoverview Chooses the passage of an extracted document that is sent as a numbered source, for the server and
+ * device-side extractors alike. The result is always an exact slice of the document, text === document.slice(start, end),
+ * so its offsets and hashes can be checked against the document later. Relevance is lexical: query content words per paragraph.
+ */
+import { contentWords, wordTokens } from './text.js';
+
+const PARAGRAPH = /\S[\s\S]*?(?=\n[ \t]*\n|$)/g;
+
+function trimmedEnd(text, start, end) {
+  while (end > start && /\s/.test(text[end - 1])) end--;
+  // Never end between the halves of a surrogate pair.
+  if (end > start && /[\ud800-\udbff]/.test(text[end - 1])) end--;
+  return end;
+}
+
+/** { text, start, end }: the best-matching paragraph (earliest on ties), extended with following paragraphs within maxCharacters. */
+export function selectPassage({ text, query = '', maxCharacters }) {
+  if (!Number.isSafeInteger(maxCharacters) || maxCharacters < 1) throw new Error('maxCharacters must be a positive integer');
+  const document = String(text ?? '');
+  const paragraphs = [...document.matchAll(PARAGRAPH)].map(match => ({ start: match.index,
+    end: trimmedEnd(document, match.index, match.index + match[0].length) })).filter(item => item.end > item.start);
+  if (!paragraphs.length) return { text: '', start: 0, end: 0 };
+  const terms = contentWords(query);
+  const score = item => { const words = new Set(wordTokens(document.slice(item.start, item.end))); return terms.filter(term => words.has(term)).length; };
+  let best = 0, bestScore = -1;
+  paragraphs.forEach((item, index) => { const value = score(item); if (value > bestScore) { best = index; bestScore = value; } });
+  const start = paragraphs[best].start;
+  let end = paragraphs[best].end;
+  if (end - start > maxCharacters) {
+    const cut = start + maxCharacters, window = document.slice(start, cut), space = window.search(/\s\S*$/);
+    end = trimmedEnd(document, start, space > maxCharacters * 0.8 ? start + space : cut);
+  } else {
+    for (const next of paragraphs.slice(best + 1)) { if (next.end - start > maxCharacters) break; end = next.end; }
+  }
+  return { text: document.slice(start, end), start, end };
+}

package/packages/reploid/src/retrieval/policy.json

@@ -0,0 +1,14 @@
+{
+  "id": "reploid.retrieval-policy",
+  "version": "1.0.0",
+  "defaults": { "maxResults": 5, "maxSourceCharacters": 1200, "maxContextCharacters": 6000, "timeoutMs": 10000 },
+  "maxima": { "maxResults": 10, "maxSourceCharacters": 4000, "maxContextCharacters": 12000, "timeoutMs": 30000 },
+  "maxQueryCharacters": 512,
+  "maxSupportChecks": 256,
+  "context": {
+    "schema": "reploid.retrieval-context/v1",
+    "instructions": "Answer using only the numbered sources below. Cite each factual sentence with the number of every source that states it, for example [1] or [1][2]. If the sources do not answer the question, say that the retrieved sources do not answer it. Do not name or cite any other source.",
+    "sourcesHeading": "Sources:",
+    "questionLabel": "Question:"
+  }
+}

package/packages/reploid/src/retrieval/retrieval.js

@@ -0,0 +1,23 @@
+/** One retrieval owner per workspace: host-supplied search, extraction and support ports under library policy. */
+import defaults from './policy.json' with { type: 'json' };
+import { verifyCitations } from './citations.js';
+import { mapClaimSpans } from './claim-support.js';
+import { normalizeRetrievalRequest, pendingRetrievalRecord, retrieveSources, validateRetrievalPolicy,
+  validateRetrievalPorts, withSourceContext } from './sources.js';
+
+export function createRetrieval({ provider, extractor = null, supportChecker = null, policy = defaults, now = Date.now }) {
+  validateRetrievalPorts({ provider, extractor, supportChecker });
+  const rules = validateRetrievalPolicy(policy);
+  return Object.freeze({
+    /** Validates a request's opt-in before anything is recorded. */
+    request: (input, content) => normalizeRetrievalRequest(input, content, rules),
+    pending: (request, startedAt) => pendingRetrievalRecord({ request, provider, extractor, policy: rules, startedAt }),
+    /** `answer`: an opaque key for the answer, so per-answer costs can be summed in provider usage logs. */
+    retrieve: (record, signal, { answer = null, onProgress } = {}) => retrieveSources({ record, provider, extractor, signal, now, policy: rules, answer, onProgress }),
+    dispatch: (request, record) => withSourceContext(request, record, rules),
+    // Citation flags plus the claim-to-span mapping and deterministic validity; entailment awaits an independent reviewer.
+    // withVerification (costs.js) attaches the result to its record together with the verification cost entry.
+    verify: async (answer, record, signal) => ({ ...await verifyCitations({ answer, record, checker: supportChecker, signal,
+      maxSupportChecks: rules.maxSupportChecks }), claimSupport: await mapClaimSpans({ answer, record, policy: rules }) })
+  });
+}

package/packages/reploid/src/retrieval/review-queue.js

@@ -0,0 +1,76 @@
+/** A blinded human review queue for citation support. Each entry keeps the packet reviewers see apart from the
+ * sealed generator identity, which only checks independence and is never shown. Two human reviewers judge every
+ * claim; claims they disagree on are listed for a third reviewer, whose review decides them by majority. */
+import { SUPPORT_JUDGMENTS_SCHEMA, applySupportJudgments, assertIndependentReviewer, supportPacketHash,
+  validateGenerator, validateSupportJudgments } from './support-review.js';
+
+export const REVIEW_QUEUE_SCHEMA = 'reploid.claim-support-review-queue/v1';
+const assert = (ok, message) => { if (!ok) throw new Error(message); };
+const copy = value => structuredClone(value);
+const memory = () => { let saved = null; return { load: () => saved, save: value => { saved = copy(value); } }; };
+
+/** `store` persists the queue: `{ load(): state | null, save(state) }`. Required reviews default to two. */
+export function createHumanReviewQueue({ store = memory(), now = Date.now, requiredReviews = 2 } = {}) {
+  assert(typeof store?.load === 'function' && typeof store?.save === 'function', 'Review queue store required');
+  assert(Number.isSafeInteger(requiredReviews) && requiredReviews > 0, 'Invalid required review count');
+  const loaded = store.load();
+  assert(!loaded || loaded.schema === REVIEW_QUEUE_SCHEMA && Array.isArray(loaded.entries), 'Invalid stored review queue');
+  const entries = copy(loaded?.entries || []);
+  const save = () => store.save({ schema: REVIEW_QUEUE_SCHEMA, entries });
+  const entry = id => { const found = entries.find(item => item.packet.id === id); assert(found, 'Review packet not found'); return found; };
+  const disagreements = async item => {
+    const files = [];
+    for (const file of item.judgments) files.push(await validateSupportJudgments({ packet: item.packet, judgments: file, generator: item.generator }));
+    return item.packet.claims.filter(claim => new Set(files.flatMap(decided => decided.has(claim.id) ? [decided.get(claim.id).verdict] : [])).size > 1)
+      .map(claim => claim.id);
+  };
+  const queue = Object.freeze({
+    id: 'reploid.human-support-review', version: '1', kind: 'human-queue',
+    /** Reviewer interface: queues the packet and returns null, because human judgments arrive later through submit(). */
+    async review(packet, { generator, claimSupport }) {
+      validateGenerator(generator);
+      assert(!entries.some(item => item.packet.id === packet.id), 'Review packet already queued');
+      entries.push({ packet: copy(packet), sha256: await supportPacketHash(packet), generator: validateGenerator(generator),
+        claimSupport: copy(claimSupport), queuedAt: now(), judgments: [] });
+      save();
+      return null;
+    },
+    /** Packets still needing a review, without their sealed generator identity. */
+    async pending() {
+      const waiting = [];
+      for (const item of entries) {
+        const third = (await disagreements(item)).length > 0 && item.judgments.length === requiredReviews;
+        if (item.judgments.length < requiredReviews || third) waiting.push({ packet: copy(item.packet), sha256: item.sha256,
+          reviews: item.judgments.length, ...(third ? { thirdReviewClaims: await disagreements(item) } : {}) });
+      }
+      return waiting;
+    },
+    packet: id => copy(entry(id).packet),
+    /** Accepts one human reviewer's judgment file; the generator never reviews, and a reviewer submits once per packet. */
+    async submit(judgments) {
+      assert(judgments?.schema === SUPPORT_JUDGMENTS_SCHEMA && judgments.reviewer?.kind === 'human', 'The queue accepts human judgment files');
+      const item = entry(judgments.packet?.id);
+      assertIndependentReviewer(judgments.reviewer, item.generator);
+      assert(!item.judgments.some(file => file.reviewer.id === judgments.reviewer.id), 'This reviewer already judged the packet');
+      const limit = requiredReviews + ((await disagreements(item)).length ? 1 : 0);
+      assert(item.judgments.length < limit, 'This packet needs no further review');
+      await validateSupportJudgments({ packet: item.packet, judgments, generator: item.generator });
+      item.judgments.push({ ...copy(judgments), receivedAt: now() });
+      save();
+      return queue.status(item.packet.id);
+    },
+    async status(id) {
+      const item = entry(id), disputed = await disagreements(item);
+      return { packetId: id, reviews: item.judgments.length, requiredReviews, disagreements: disputed,
+        needsThirdReview: disputed.length > 0 && item.judgments.length === requiredReviews,
+        complete: item.judgments.length >= requiredReviews && (!disputed.length || item.judgments.length > requiredReviews) };
+    },
+    /** The per-claim result from every judgment received so far. */
+    result(id) {
+      const item = entry(id);
+      return applySupportJudgments({ packet: item.packet, claimSupport: item.claimSupport,
+        judgments: item.judgments.map(({ receivedAt, ...file }) => file), generator: item.generator });
+    }
+  });
+  return queue;
+}

package/packages/reploid/src/retrieval/sources.js

@@ -0,0 +1,263 @@
+/** Retrieval for one request: provider search, optional passage extraction, bounded numbered context
+ * and the provenance record kept with the conversation. Providers and extractors are host ports. */
+import defaults from './policy.json' with { type: 'json' };
+import { SHA256, sha256Text } from './text.js';
+import { COST_BEARERS, retrievalCostEntry, retrievalCostRecord, unknownRetrievalCosts } from './costs.js';
+// Provider and extractor calls are awaited only until the deadline passes or the attempt stops;
+// providers that ignore their signal cannot extend it.
+import { abortable } from '../cancellation.js';
+
+export const RETRIEVAL_RECORD_SCHEMA = 'reploid.retrieval-record/v1';
+export const RETRIEVAL_OUTCOMES = Object.freeze(['pending', 'retrieved', 'empty', 'failed', 'cancelled', 'interrupted']);
+const LIMIT_NAMES = ['maxResults', 'maxSourceCharacters', 'maxContextCharacters', 'timeoutMs'];
+const FAILURES = Object.freeze({
+  'no-results': 'the search returned no results',
+  timeout: 'retrieval exceeded its time limit',
+  'provider-error': 'the search provider failed',
+  'invalid-provider-result': 'the search provider returned an invalid result',
+  'extraction-failed': 'no source text could be extracted',
+  'context-limit': 'no source fits the context limit',
+  cancelled: 'the request was stopped during retrieval',
+  interrupted: 'the connection was interrupted during retrieval'
+});
+
+class RetrievalError extends Error {
+  constructor(code, detail = null) { super(FAILURES[code]); this.code = code; this.detail = detail; }
+}
+const assert = (ok, message) => { if (!ok) throw new Error(message); };
+/** Where a provider's retrieval runs: an operator's server route, or this device. Recorded with the provider. */
+export const RETRIEVAL_SUPPLIES = Object.freeze(['server', 'device', 'contributor']);
+const copy = value => structuredClone(value);
+const port = value => value && typeof value.id === 'string' && value.id && typeof value.version === 'string' && value.version;
+const line = value => String(value).replace(/\s+/g, ' ').trim();
+
+export function validateRetrievalPolicy(policy) {
+  assert(policy && typeof policy.id === 'string' && typeof policy.version === 'string'
+    && LIMIT_NAMES.every(name => Number.isSafeInteger(policy.maxima?.[name]) && policy.maxima[name] > 0
+      && Number.isSafeInteger(policy.defaults?.[name]) && policy.defaults[name] > 0 && policy.defaults[name] <= policy.maxima[name])
+    && policy.maxima.timeoutMs <= 2147483647 && Number.isSafeInteger(policy.maxQueryCharacters) && policy.maxQueryCharacters > 0
+    && Number.isSafeInteger(policy.maxSupportChecks) && policy.maxSupportChecks >= 0
+    && ['schema', 'instructions', 'sourcesHeading', 'questionLabel'].every(name => typeof policy.context?.[name] === 'string'),
+  'Invalid retrieval policy');
+  return copy(policy);
+}
+
+/** Requested limits may only narrow the policy maxima; omitted limits use the policy defaults. */
+export function normalizeRetrievalLimits(requested = {}, policy = defaults) {
+  assert(requested && typeof requested === 'object' && !Array.isArray(requested)
+    && Object.keys(requested).every(name => LIMIT_NAMES.includes(name)), 'Unknown retrieval limit');
+  return Object.fromEntries(LIMIT_NAMES.map(name => {
+    const value = requested[name] ?? policy.defaults[name];
+    assert(Number.isSafeInteger(value) && value > 0 && value <= policy.maxima[name], `Retrieval ${name} is outside the policy limit`);
+    return [name, value];
+  }));
+}
+
+/** The request's opt-in: the query sent to the provider and its limits. Defaults to the message text. */
+export function normalizeRetrievalRequest(input, content, policy = defaults) {
+  assert(input && typeof input === 'object' && !Array.isArray(input)
+    && Object.keys(input).every(name => ['query', 'limits'].includes(name)), 'Invalid retrieval request');
+  assert(input.query === undefined || typeof input.query === 'string', 'Invalid search query');
+  const query = (input.query ?? content).trim();
+  assert(query, 'Search query is empty');
+  assert(query.length <= policy.maxQueryCharacters, 'Search query exceeds the retrieval limit');
+  return { query, limits: normalizeRetrievalLimits(input.limits, policy) };
+}
+
+export function pendingRetrievalRecord({ request, provider, extractor = null, policy = defaults, startedAt }) {
+  return { schema: RETRIEVAL_RECORD_SCHEMA, policy: { id: policy.id, version: policy.version },
+    provider: { id: provider.id, version: provider.version, ...(provider.supply === undefined ? {} : { supply: provider.supply }) },
+    extractor: extractor ? { id: extractor.id, version: extractor.version } : null,
+    // Every source below came from the provider this request explicitly opted into.
+    sourceUse: 'requester-opt-in', query: request.query, limits: copy(request.limits),
+    outcome: 'pending', error: null, startedAt, finishedAt: null, providerResultCount: null,
+    sources: [], omitted: [], context: null, usage: null, verification: null, costs: null };
+}
+
+export function retrievalFailureMessage(record) {
+  return `Could not retrieve sources: ${FAILURES[record?.error?.code] || 'retrieval did not complete'}.`;
+}
+
+/** Marks a record left pending by a reload. Nothing it would have retrieved was sent; what it spent was not observed. */
+export function interruptRetrievalRecord(record, finishedAt) {
+  if (record?.outcome !== 'pending') return record;
+  return { ...record, outcome: 'interrupted', error: { code: 'interrupted', message: FAILURES.interrupted }, finishedAt,
+    costs: retrievalCostRecord({ retrieval: unknownRetrievalCosts() }) };
+}
+
+const webUrl = value => {
+  try { return typeof value === 'string' && value.length <= 2048 && ['http:', 'https:'].includes(new URL(value).protocol); }
+  catch { return false; }
+};
+function validResult(value) {
+  return value && typeof value === 'object' && typeof value.sourceId === 'string' && /^[\x21-\x7e]{1,256}$/.test(value.sourceId)
+    && webUrl(value.url) && typeof value.title === 'string' && line(value.title) && value.title.length <= 1000
+    && typeof value.excerpt === 'string' && value.excerpt.trim()
+    && Number.isFinite(value.retrievedAt) && value.retrievedAt > 0 && typeof value.contentHash === 'string' && SHA256.test(value.contentHash);
+}
+const count = value => Number.isSafeInteger(value) && value >= 0;
+/** Where a fetched passage sits in the extracted document text the extractor hashed. */
+const validDocument = (value, text) => value === undefined || value && typeof value === 'object' && webUrl(value.url)
+  && typeof value.contentType === 'string' && value.contentType.length <= 200 && SHA256.test(value.sha256)
+  && count(value.start) && count(value.end) && value.end - value.start === text.length && count(value.characters)
+  && value.end <= value.characters && (value.bytes === undefined || count(value.bytes))
+  && (value.status === undefined || Number.isSafeInteger(value.status) && value.status >= 200 && value.status < 300);
+const validPassage = (value, source) => value && typeof value === 'object' && value.sourceId === source.sourceId
+  && (value.title === undefined || typeof value.title === 'string' && value.title.length <= 1000)
+  && typeof value.text === 'string' && value.text.trim() && Number.isFinite(value.retrievedAt) && value.retrievedAt > 0
+  && typeof value.contentHash === 'string' && SHA256.test(value.contentHash) && validDocument(value.document, value.text)
+  && (value.costMicroUsd === undefined || value.costMicroUsd === null || count(value.costMicroUsd));
+/** Provider-reported usage for one query: billable requests and, when priced, the cost in micro-dollars. */
+const validUsage = value => value === undefined || value && typeof value === 'object' && typeof value.provider === 'string'
+  && /^[\x21-\x7e]{1,64}$/.test(value.provider) && count(value.requests) && (value.costMicroUsd === null || count(value.costMicroUsd));
+/** Providers return results, or `{ results, usage }` when they report per-query usage. */
+const providerResults = value => Array.isArray(value) ? { results: value, usage: undefined }
+  : value && typeof value === 'object' && Array.isArray(value.results) ? { results: value.results, usage: value.usage } : null;
+
+/** The retained passage's offsets in the extracted document: the extractor's span, less trimmed and truncated text. */
+function retainedDocument(document, text, retained) {
+  if (!document) return null;
+  const start = document.start + (text.length - text.trimStart().length);
+  return { url: document.url, contentType: document.contentType, characters: document.characters, sha256: document.sha256,
+    start, end: start + retained.length, ...(document.bytes === undefined ? {} : { bytes: document.bytes }),
+    ...(document.status === undefined ? {} : { status: document.status }) };
+}
+
+/** Passages longer than the per-source limit are cut at a word boundary when one is near. */
+function bounded(text, limit) {
+  const clean = text.trim();
+  if (clean.length <= limit) return { text: clean, truncated: false };
+  const cut = clean.slice(0, limit), space = cut.search(/\s\S*$/);
+  return { text: (space > limit * 0.8 ? cut.slice(0, space) : cut).trimEnd(), truncated: true };
+}
+
+const sourceBlock = source => `[${source.number}] ${source.title}\n${source.url}\n${source.passage}`;
+/** The exact numbered context sent with the question. Deterministic for a record's sources and policy. */
+export function renderSourceContext(sources, policy = defaults) {
+  return `${policy.context.instructions}\n\n${policy.context.sourcesHeading}\n\n${sources.map(sourceBlock).join('\n\n')}`;
+}
+
+
+/** Runs one retrieval and returns the finished record. Failures are recorded, never thrown and never filled with sources.
+ * `answer` is an opaque key for the answer this retrieval serves; providers may pass it to their own usage log so
+ * service economics can sum costs per answer. It is never part of the query. */
+export async function retrieveSources({ record, provider, extractor = null, signal, now = Date.now, policy = defaults, answer = null, onProgress }) {
+  const limits = record.limits, deadline = new AbortController();
+  const timer = setTimeout(() => deadline.abort(new RetrievalError('timeout')), limits.timeoutMs);
+  const forward = () => deadline.abort(new RetrievalError('cancelled'));
+  if (signal?.aborted) forward(); else signal?.addEventListener('abort', forward, { once: true });
+  let usage = null, stage = 'search';
+  let completed = 0, available = 0;
+  const progress = (total = null) => {
+    if (signal?.aborted) return;
+    try { onProgress?.({ stage, completed, total, available }); } catch { /* Observation cannot cancel retrieval. */ }
+  };
+  // What this retrieval spent, for its cost record: observed even when it fails.
+  const meter = { searched: false, usage: null, attempted: 0, documents: 0, failed: 0, bytes: 0, costs: [] };
+  const context = { providerId: record.provider.id, ...(answer === null ? {} : { answer }) };
+  const finish = (fields) => ({ ...record, ...fields, usage, finishedAt: now(), costs: retrievalCostRecord({
+    retrieval: retrievalCostEntry({ bearer: provider.costBearer ?? null, service: provider.searchProvider ?? null, meter }) }) });
+  try {
+    progress();
+    let returned;
+    try {
+      returned = providerResults(await abortable(() => { meter.searched = true; return provider.search(record.query, copy(limits), deadline.signal, copy(context)); },
+        deadline.signal));
+    } catch (error) { throw error instanceof RetrievalError ? error : new RetrievalError('provider-error', error); }
+    if (!returned || !returned.results.every(validResult) || !validUsage(returned.usage)) throw new RetrievalError('invalid-provider-result');
+    const results = returned.results;
+    // Recorded before anything else can fail: a failed or empty search may still have been billed.
+    const searchUsage = returned.usage ? { provider: returned.usage.provider, requests: returned.usage.requests,
+      costMicroUsd: returned.usage.costMicroUsd } : null;
+    meter.usage = searchUsage;
+    usage = { search: searchUsage, extraction: { documents: 0, bytes: 0 } };
+    if (!results.length) return finish({ outcome: 'empty', error: { code: 'no-results', message: FAILURES['no-results'] }, providerResultCount: 0 });
+    stage = 'extraction';
+    const omitted = [], seen = new Set(), kept = [];
+    for (const result of results) {
+      if (seen.has(result.sourceId)) continue;
+      seen.add(result.sourceId);
+      if (kept.length < limits.maxResults) kept.push(result); else omitted.push({ sourceId: result.sourceId, reason: 'result-limit' });
+    }
+    if (extractor) meter.attempted = kept.length;
+    progress(kept.length);
+    const passages = await Promise.allSettled(kept.map(source => (extractor
+      ? abortable(() => extractor.extract({ source: copy(source), query: record.query, limits: copy(limits), signal: deadline.signal, ...copy(context) }),
+        deadline.signal)
+        .then(value => { if (!validPassage(value, source)) throw new Error('Invalid extracted passage'); return { ...value, origin: 'extract' }; })
+      : Promise.resolve({ sourceId: source.sourceId, text: source.excerpt, retrievedAt: source.retrievedAt,
+        contentHash: source.contentHash, origin: 'search-excerpt' }))
+      .then(value => { available++; return value; })
+      .finally(() => { completed++; progress(kept.length); })));
+    if (extractor) for (const passage of passages) {
+      if (passage.status === 'rejected') { meter.failed++; continue; }
+      const bytes = passage.value.document?.bytes;
+      meter.documents++;
+      meter.bytes = meter.bytes === null || bytes === undefined ? null : meter.bytes + bytes;
+      meter.costs.push(passage.value.costMicroUsd ?? null);
+    }
+    const timedOut = deadline.signal.aborted && deadline.signal.reason?.code === 'timeout';
+    if (deadline.signal.aborted && !timedOut) throw deadline.signal.reason;
+    const candidates = [];
+    kept.forEach((source, index) => {
+      const passage = passages[index];
+      if (passage.status === 'rejected') { omitted.push({ sourceId: source.sourceId,
+        reason: passage.reason?.code === 'timeout' ? 'extraction-timeout' : 'extraction-failed' }); return; }
+      const { text, truncated } = bounded(passage.value.text, limits.maxSourceCharacters);
+      const document = retainedDocument(passage.value.document, passage.value.text, text);
+      if (document) { usage.extraction.documents++; usage.extraction.bytes += document.bytes ?? 0; }
+      // A fetched page supplies its own address and title, so retained and forwarded sources keep nothing from the
+      // search response beyond transient use: search terms (such as Brave's) forbid storing or redistributing results.
+      const page = passage.value.document;
+      candidates.push({ sourceId: source.sourceId, url: page ? page.url : source.url,
+        title: page ? line(passage.value.title ?? '') || page.url : line(source.title), retrievedAt: passage.value.retrievedAt,
+        contentHash: passage.value.contentHash, passage: text, passageOrigin: passage.value.origin, truncated,
+        ...(document ? { document } : {}) });
+    });
+    if (!candidates.length) throw new RetrievalError(timedOut ? 'timeout' : 'extraction-failed');
+    // Sources are numbered in provider order; any that would push the context past its limit are omitted.
+    const sources = [];
+    for (const candidate of candidates) {
+      const next = [...sources, { number: sources.length + 1, ...candidate }];
+      if (renderSourceContext(next, policy).length <= limits.maxContextCharacters) sources.push(next.at(-1));
+      else omitted.push({ sourceId: candidate.sourceId, reason: 'context-limit' });
+    }
+    if (!sources.length) throw new RetrievalError('context-limit');
+    stage = 'context';
+    progress(kept.length);
+    for (const source of sources) source.passageHash = await sha256Text(source.passage);
+    const content = renderSourceContext(sources, policy);
+    const digest = await sha256Text(content);
+    if (signal?.aborted) throw new RetrievalError('cancelled');
+    return finish({ outcome: 'retrieved', providerResultCount: results.length, sources, omitted,
+      partial: passages.some(passage => passage.status === 'rejected'),
+      context: { schema: policy.context.schema, characters: content.length, digest } });
+  } catch (cause) {
+    const error = cause instanceof RetrievalError ? cause : new RetrievalError('provider-error', cause);
+    return finish({ outcome: error.code === 'cancelled' ? 'cancelled' : 'failed', sources: [], context: null,
+      error: { code: error.code, message: FAILURES[error.code], stage } });
+  } finally {
+    clearTimeout(timer); signal?.removeEventListener('abort', forward);
+  }
+}
+
+/** The request executors receive: the latest user turn carries the numbered sources, checked against the record's digest. */
+export async function withSourceContext(request, record, policy = defaults) {
+  assert(record?.outcome === 'retrieved' && record.sources.length, retrievalFailureMessage(record));
+  const content = renderSourceContext(record.sources, policy);
+  assert(record.context?.schema === policy.context.schema && await sha256Text(content) === record.context.digest,
+    'Retrieved context does not match its recorded digest');
+  const { retrieval, ...executed } = request, messages = request.messages.map(message => ({ ...message }));
+  const last = messages.at(-1);
+  assert(last?.role === 'user', 'Retrieval requires a user question');
+  last.content = `${content}\n\n${record.query !== last.content ? `Search query: ${record.query}\n` : ''}${policy.context.questionLabel} ${last.content}`;
+  return { ...executed, messages };
+}
+
+export function validateRetrievalPorts({ provider, extractor = null, supportChecker = null }) {
+  assert(port(provider) && typeof provider.search === 'function', 'Retrieval requires a search provider');
+  assert(provider.costBearer === undefined || COST_BEARERS.includes(provider.costBearer), 'Invalid search provider cost bearer');
+  assert(provider.supply === undefined || RETRIEVAL_SUPPLIES.includes(provider.supply), 'Invalid search provider supply');
+  assert(extractor === null || port(extractor) && typeof extractor.extract === 'function', 'Invalid passage extractor');
+  assert(supportChecker === null || port(supportChecker) && typeof supportChecker.check === 'function', 'Invalid citation support checker');
+}