wolfy-reader 0.1.0

package/src/search/matcher.ts

@@ -0,0 +1,187 @@
+/**
+ * The headless full-text matcher. Normalized literal substring search over a
+ * section's extracted text, yielding jumpable hits. Imports only `src/core`
+ * (`capturePosition`, `Position`, `Book`, `Section`) and platform primitives, so it
+ * runs under `node:test` with no browser.
+ *
+ * The whole-book scan is a lazy async generator: it decodes and scans one section at
+ * a time and yields hits as it goes, so the book is never buffered whole and a
+ * consumer that `break`s stops the scan (the generator's cleanup runs on return).
+ */
+import { capturePosition, type Book, type Position, type Section } from '../core/index.ts';
+import { collapseWhitespace as collapse } from '../core/text.ts';
+
+import { extractSectionText } from './extract.ts';
+import { normalizeQuery, normalizeText } from './normalize.ts';
+
+/** One search hit: the matched text, a clean context window, and a jumpable position. */
+export interface SearchHit {
+  /** The verbatim matched run, in the section's raw text. */
+  readonly text: string;
+  /**
+   * The match surrounded by context, trimmed to whole words at both ends via
+   * `Intl.Segmenter` — no leading/trailing partial word.
+   */
+  readonly context: string;
+  /**
+   * A content-addressed `Position` at the match, built with `capturePosition`. Pass
+   * it to `reader.goTo(hit.position)` to land on the hit's page.
+   */
+  readonly position: Position;
+  /** 0-based index of the section this hit falls in, in reading order. */
+  readonly sectionIndex: number;
+}
+
+export interface SearchOptions {
+  /**
+   * Characters of surrounding context to aim for on each side of the match before
+   * word-boundary trimming. Default 40.
+   */
+  readonly contextRadius?: number;
+  /** Locale for word segmentation of the context. Defaults to the runtime default. */
+  readonly locale?: string;
+}
+
+const DEFAULT_CONTEXT_RADIUS = 40;
+
+/**
+ * All hits for `query` within one section's already-extracted `rawText`. Pure over
+ * strings — no I/O — so it is the unit the section scan and the tests both drive.
+ * A blank (whitespace-only or empty) query yields nothing.
+ */
+export function* matchText(
+  rawText: string,
+  query: string,
+  sectionId: string,
+  sectionIndex: number,
+  options: SearchOptions = {},
+): Generator<SearchHit> {
+  const needle = normalizeQuery(query);
+  if (needle === '') return;
+
+  const { text: haystack, map } = normalizeText(rawText);
+  const radius = options.contextRadius ?? DEFAULT_CONTEXT_RADIUS;
+
+  let from = 0;
+  for (;;) {
+    const at = haystack.indexOf(needle, from);
+    if (at === -1) return;
+    const end = at + needle.length;
+
+    // Map the normalized match span back onto the raw text. `map[end]` points at the
+    // start of the character *after* the match, which — when that character is a
+    // collapsed space — lies past a run of raw whitespace; trim it so the matched
+    // text ends on real content. The anchor still starts at `rawStart`.
+    const rawStart = map[at]!;
+    const rawEnd = trimEnd(rawText, rawStart, map[end]!);
+    const matched = rawText.slice(rawStart, rawEnd);
+
+    // Context window in normalized space, then mapped back and word-trimmed on the
+    // raw text so the shown context carries no partial word at either edge.
+    const ctxNormStart = Math.max(0, at - radius);
+    const ctxNormEnd = Math.min(haystack.length, end + radius);
+    const rawCtxStart = map[ctxNormStart]!;
+    const rawCtxEnd = map[ctxNormEnd]!;
+    const context = trimToWords(
+      rawText,
+      rawCtxStart,
+      rawCtxEnd,
+      rawStart,
+      rawEnd,
+      options.locale,
+    );
+
+    yield {
+      text: matched,
+      context,
+      position: capturePosition(rawText, rawStart, sectionId),
+      sectionIndex,
+    };
+
+    // Advance past this match; guard against a zero-length needle (impossible here,
+    // an empty needle returned above) so the loop always progresses.
+    from = end > at ? end : at + 1;
+  }
+}
+
+/**
+ * Trims a raw context window `[start, end)` inward to whole-word boundaries via
+ * `Intl.Segmenter`, never cutting into the match span `[matchStart, matchEnd)`. A
+ * word straddling the left edge is dropped forward to its end (the next word's
+ * start); a word straddling the right edge is dropped back to its start. The result
+ * is whitespace-collapsed for a compact one-line context.
+ */
+function trimToWords(
+  text: string,
+  start: number,
+  end: number,
+  matchStart: number,
+  matchEnd: number,
+  locale: string | undefined,
+): string {
+  // Segment a slice wider than the window so a word straddling an edge is visible
+  // whole. Offsets are relative to the wide slice, so add `lo`.
+  const lo = Math.max(0, Math.min(start, matchStart) - 24);
+  const hi = Math.min(text.length, Math.max(end, matchEnd) + 24);
+  let head = start;
+  let tail = end;
+  for (const seg of wordSegmenter(locale).segment(text.slice(lo, hi))) {
+    if (!seg.isWordLike) continue;
+    const segStart = lo + seg.index;
+    const segEnd = segStart + seg.segment.length;
+    // A word-like segment straddling the left edge: move head to its end (the next
+    // word's start), but never past the match start.
+    if (segStart < start && segEnd > start) head = Math.min(matchStart, segEnd);
+    // A word-like segment straddling the right edge: move tail to its start, but
+    // never before the match end.
+    if (segStart < end && segEnd > end) tail = Math.max(matchEnd, segStart);
+  }
+  return collapse(text.slice(head, tail));
+}
+
+/** Pull `end` back past any trailing whitespace, never before `start`. */
+function trimEnd(text: string, start: number, end: number): number {
+  let at = end;
+  while (at > start && /\s/.test(text[at - 1]!)) at -= 1;
+  return at;
+}
+
+let cachedWordSegmenter: { locale: string | undefined; segmenter: Intl.Segmenter } | undefined;
+
+function wordSegmenter(locale: string | undefined): Intl.Segmenter {
+  if (cachedWordSegmenter === undefined || cachedWordSegmenter.locale !== locale) {
+    cachedWordSegmenter = { locale, segmenter: new Intl.Segmenter(locale, { granularity: 'word' }) };
+  }
+  return cachedWordSegmenter.segmenter;
+}
+
+/**
+ * Lazily scans every section of `book` for `query`, one section at a time, yielding
+ * each hit as it is found. The book is never buffered whole: each section's bytes are
+ * loaded, extracted, matched and released before the next is touched. A consumer that
+ * stops early (`break`/`return`) halts the scan — no later section is loaded.
+ *
+ * `onSection` (test-only instrumentation seam) fires with each section's index the
+ * moment before it is decoded, so a test can prove laziness by counting decodes.
+ */
+export async function* searchBook(
+  book: Book,
+  query: string,
+  options: SearchOptions & { onSection?: (index: number) => void } = {},
+): AsyncGenerator<SearchHit> {
+  const needle = normalizeQuery(query);
+  if (needle === '') return;
+
+  const sections = book.sections;
+  for (let index = 0; index < sections.length; index += 1) {
+    const section: Section = sections[index]!;
+    options.onSection?.(index);
+    const bytes = await section.load();
+    // The section's own resolver rides along so extraction shows an <img> the
+    // way the frame will: rendered (no text) or substituted by its alt.
+    const rawText = extractSectionText(bytes, section.resolve?.bind(section));
+    for (const hit of matchText(rawText, query, section.id, index, options)) {
+      yield hit;
+    }
+  }
+}

package/src/search/normalize.ts

@@ -0,0 +1,122 @@
+/**
+ * Search normalization — the folding policy that decides when two strings "match"
+ * for the purpose of full-text search. Applied to both the query and the extracted
+ * section text before literal substring matching.
+ *
+ * Policy (see docs/domains/search.md ## Key decisions):
+ * - **Case-insensitive.** Folded with `toLowerCase()`.
+ * - **Unicode NFC.** Each grapheme is canonically composed, so a pre-composed `é`
+ *   and a decomposed `e`+combining-acute match.
+ * - **Diacritics are preserved.** `café` does not match `cafe`. Stripping marks is a
+ *   locale-sensitive judgement (German ä, Turkish dotless ı) better left to a later,
+ *   opt-in dimension than baked into the default.
+ * - **Smart punctuation folded to ASCII.** Curly quotes → straight, en/em dashes and
+ *   the minus sign → hyphen-minus, non-breaking and other Unicode spaces → a normal
+ *   space. A reader types `don't`, the book prints `don’t`.
+ * - **Whitespace collapsed.** Every run of whitespace (including the newlines that
+ *   land between block elements during extraction) becomes a single space, and
+ *   leading/trailing space folds off, so a query never fails on invisible layout
+ *   whitespace.
+ *
+ * The fold must stay anchorable: the matcher works on the normalized string but each
+ * hit's text and `Position` anchor into the **raw extracted** text. So the fold is
+ * built grapheme by grapheme alongside an offset map from every normalized index
+ * back to the raw UTF-16 offset it came from. The raw text handed to
+ * `capturePosition` is never NFC-rewritten (it must stay byte-identical to what the
+ * paginator measures against), so composition happens only inside the folded copy.
+ */
+
+/** A normalized string plus the map from its indices back to raw UTF-16 offsets. */
+export interface NormalizedText {
+  /** The folded, whitespace-collapsed text the matcher searches. */
+  readonly text: string;
+  /**
+   * `map[i]` is the raw UTF-16 offset of the character that produced normalized
+   * code unit `i`; `map[text.length]` is the raw text length. Monotonic
+   * non-decreasing, so a normalized span `[a, b)` maps to the raw span
+   * `[map[a], map[b])`.
+   */
+  readonly map: readonly number[];
+}
+
+const SMART: ReadonlyMap<string, string> = new Map([
+  ['‘', "'"], // ‘ left single quote
+  ['’', "'"], // ’ right single quote / apostrophe
+  ['‚', "'"], // ‚ single low-9 quote
+  ['‛', "'"], // ‛ single high-reversed-9 quote
+  ['“', '"'], // “ left double quote
+  ['”', '"'], // ” right double quote
+  ['„', '"'], // „ double low-9 quote
+  ['‟', '"'], // ‟ double high-reversed-9 quote
+  ['′', "'"], // ′ prime
+  ['″', '"'], // ″ double prime
+  ['‐', '-'], // ‐ hyphen
+  ['‑', '-'], // ‑ non-breaking hyphen
+  ['‒', '-'], // ‒ figure dash
+  ['–', '-'], // – en dash
+  ['—', '-'], // — em dash
+  ['―', '-'], // ― horizontal bar
+  ['−', '-'], // − minus sign
+]);
+
+const WHITESPACE = /^\s$/;
+
+let graphemeSegmenter: Intl.Segmenter | undefined;
+
+function segmenter(): Intl.Segmenter {
+  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });
+  return graphemeSegmenter;
+}
+
+/** True when every code point in a grapheme is whitespace. */
+function isWhitespace(grapheme: string): boolean {
+  for (const ch of grapheme) {
+    if (!WHITESPACE.test(ch) && ch !== '​' && ch !== '') return false;
+  }
+  return true;
+}
+
+/** Folds a single grapheme under the punctuation/NFC/case policy. */
+function foldGrapheme(grapheme: string): string {
+  const mapped = grapheme.length === 1 ? (SMART.get(grapheme) ?? grapheme) : grapheme;
+  return mapped.normalize('NFC').toLowerCase();
+}
+
+/**
+ * Folds `raw` under the search policy and returns the normalized text plus the
+ * index map back to raw UTF-16 offsets. A match found in `text` maps cleanly onto a
+ * raw span for `capturePosition`.
+ */
+export function normalizeText(raw: string): NormalizedText {
+  let text = '';
+  const map: number[] = [];
+  let pendingSpace = false;
+  let sawNonSpace = false;
+  let rawOffset = 0;
+
+  for (const { segment } of segmenter().segment(raw)) {
+    if (isWhitespace(segment)) {
+      if (sawNonSpace) pendingSpace = true;
+      rawOffset += segment.length;
+      continue;
+    }
+    if (pendingSpace) {
+      map.push(rawOffset); // the collapsed space anchors at the start of the text it precedes
+      text += ' ';
+      pendingSpace = false;
+    }
+    const folded = foldGrapheme(segment);
+    for (let unit = 0; unit < folded.length; unit += 1) map.push(rawOffset);
+    text += folded;
+    sawNonSpace = true;
+    rawOffset += segment.length;
+  }
+
+  map.push(raw.length); // trailing whitespace folds off; the terminal maps to raw end
+  return { text, map };
+}
+
+/** The query folded to the same space as {@link normalizeText}'s `text`. */
+export function normalizeQuery(query: string): string {
+  return normalizeText(query).text;
+}

package/src/view/allowlist.ts

@@ -0,0 +1,273 @@
+/**
+ * What is allowed to survive sanitization, stated as allowlists.
+ *
+ * A blocklist loses to the next vector the platform invents; an allowlist only
+ * ever loses formatting. Anything absent from these tables is either unwrapped
+ * (the element goes, its text stays — a book is text before it is markup) or,
+ * when its children are not readable content, discarded whole.
+ */
+
+function set(...names: string[]): ReadonlySet<string> {
+  return new Set(names);
+}
+
+const NONE: ReadonlySet<string> = new Set<string>();
+
+export const HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml';
+export const SVG_NAMESPACE = 'http://www.w3.org/2000/svg';
+export const XLINK_NAMESPACE = 'http://www.w3.org/1999/xlink';
+
+export const HTML_GLOBAL_ATTRIBUTES = set(
+  'class',
+  'dir',
+  'epub:type',
+  'hidden',
+  'id',
+  'lang',
+  'role',
+  'style',
+  'title',
+  'translate',
+  'xml:lang',
+);
+
+const CELL = set('abbr', 'align', 'colspan', 'headers', 'rowspan', 'scope', 'valign', 'width');
+
+export const HTML_ELEMENTS: ReadonlyMap<string, ReadonlySet<string>> = new Map([
+  ['a', set('href')],
+  ['abbr', NONE],
+  ['address', NONE],
+  ['article', NONE],
+  ['aside', NONE],
+  ['b', NONE],
+  ['bdi', NONE],
+  ['bdo', NONE],
+  ['big', NONE],
+  ['blockquote', set('cite')],
+  ['br', NONE],
+  ['caption', set('align')],
+  ['center', NONE],
+  ['cite', NONE],
+  ['code', NONE],
+  ['col', set('align', 'span', 'valign', 'width')],
+  ['colgroup', set('align', 'span', 'valign', 'width')],
+  ['dd', NONE],
+  ['del', set('cite', 'datetime')],
+  ['details', set('open')],
+  ['dfn', NONE],
+  ['div', NONE],
+  ['dl', NONE],
+  ['dt', NONE],
+  ['em', NONE],
+  ['figcaption', NONE],
+  ['figure', NONE],
+  ['footer', NONE],
+  ['h1', NONE],
+  ['h2', NONE],
+  ['h3', NONE],
+  ['h4', NONE],
+  ['h5', NONE],
+  ['h6', NONE],
+  ['header', NONE],
+  ['hgroup', NONE],
+  ['hr', NONE],
+  ['i', NONE],
+  ['img', set('alt', 'height', 'src', 'width')],
+  ['ins', set('cite', 'datetime')],
+  ['kbd', NONE],
+  ['li', set('value')],
+  ['link', set('href', 'media', 'rel', 'type')],
+  ['main', NONE],
+  ['mark', NONE],
+  ['nav', NONE],
+  ['ol', set('reversed', 'start', 'type')],
+  ['p', NONE],
+  ['pre', NONE],
+  ['q', set('cite')],
+  ['rp', NONE],
+  ['rt', NONE],
+  ['ruby', NONE],
+  ['s', NONE],
+  ['samp', NONE],
+  ['section', NONE],
+  ['small', NONE],
+  ['span', NONE],
+  ['strike', NONE],
+  ['strong', NONE],
+  ['style', set('media', 'type')],
+  ['sub', NONE],
+  ['summary', NONE],
+  ['sup', NONE],
+  ['table', set('align', 'border', 'cellpadding', 'cellspacing', 'summary', 'width')],
+  ['tbody', set('align', 'valign')],
+  ['td', CELL],
+  ['tfoot', set('align', 'valign')],
+  ['th', CELL],
+  ['thead', set('align', 'valign')],
+  ['time', set('datetime')],
+  ['tr', set('align', 'valign')],
+  ['tt', NONE],
+  ['u', NONE],
+  ['ul', NONE],
+  ['var', NONE],
+  ['wbr', NONE],
+]);
+
+/**
+ * Elements that go with everything inside them, because their children are not
+ * prose: raw-text elements whose content is source rather than text, form
+ * controls, and void elements with no children to keep.
+ *
+ * Everything else absent from the allowlist is *unwrapped* instead — the
+ * element goes, its text stays. That distinction is load-bearing twice over. A
+ * book is text before it is markup, and a malformed one nests half a chapter
+ * inside a stray element: an </noscript> end tag after an open <p> is ignored
+ * by the HTML parser, so an unclosed <noscript> swallows everything after it,
+ * and discarding that element would delete the rest of the chapter.
+ *
+ * <noscript> is also why unwrapping is the safer option and not merely the
+ * kinder one. Its content is markup when a document is parsed with scripting
+ * off and raw text when it is parsed with scripting on, so an element that
+ * survives into the frame is re-read in the other mode and attribute
+ * boundaries move under it — the mutation-XSS shape this pipeline exists to
+ * defeat. Removing the element removes the reinterpretation.
+ *
+ * The table itself lives in `src/core/reading-text.ts` (re-exported here under
+ * its frozen name): discarding decides not only what renders but what the
+ * frame's *text* is, and the headless search extractor must mirror it exactly
+ * so a hit's anchor exists in the text the frame resolves against. One table,
+ * two consumers — never fork it.
+ */
+export { DISCARDED_HTML_ELEMENTS as HTML_DISCARDED } from '../core/reading-text.ts';
+
+export const SVG_GLOBAL_ATTRIBUTES = set(
+  'class',
+  'clip-path',
+  'clip-rule',
+  'color',
+  'display',
+  'dominant-baseline',
+  'fill',
+  'fill-opacity',
+  'fill-rule',
+  'font-family',
+  'font-size',
+  'font-style',
+  'font-weight',
+  'id',
+  'lang',
+  'letter-spacing',
+  'mask',
+  'opacity',
+  'overflow',
+  'paint-order',
+  'role',
+  'shape-rendering',
+  'space',
+  'stop-color',
+  'stop-opacity',
+  'stroke',
+  'stroke-dasharray',
+  'stroke-dashoffset',
+  'stroke-linecap',
+  'stroke-linejoin',
+  'stroke-opacity',
+  'stroke-width',
+  'style',
+  'text-anchor',
+  'transform',
+  'vector-effect',
+  'visibility',
+  'word-spacing',
+);
+
+const GRADIENT = set(
+  'cx',
+  'cy',
+  'fr',
+  'fx',
+  'fy',
+  'gradientTransform',
+  'gradientUnits',
+  'r',
+  'spreadMethod',
+  'x1',
+  'x2',
+  'y1',
+  'y2',
+);
+
+const TEXT_POSITION = set('dx', 'dy', 'rotate', 'textLength', 'x', 'y');
+
+export const SVG_ELEMENTS: ReadonlyMap<string, ReadonlySet<string>> = new Map([
+  ['a', set('href')],
+  ['circle', set('cx', 'cy', 'r')],
+  ['clipPath', set('clipPathUnits')],
+  ['defs', NONE],
+  ['desc', NONE],
+  ['ellipse', set('cx', 'cy', 'rx', 'ry')],
+  ['g', NONE],
+  ['image', set('height', 'href', 'preserveAspectRatio', 'width', 'x', 'y')],
+  ['line', set('x1', 'x2', 'y1', 'y2')],
+  ['linearGradient', GRADIENT],
+  ['marker', set('markerHeight', 'markerUnits', 'markerWidth', 'orient', 'refX', 'refY', 'viewBox')],
+  ['mask', set('height', 'maskContentUnits', 'maskUnits', 'width', 'x', 'y')],
+  ['path', set('d', 'pathLength')],
+  ['pattern', set('height', 'patternContentUnits', 'patternTransform', 'patternUnits', 'viewBox', 'width', 'x', 'y')],
+  ['polygon', set('points')],
+  ['polyline', set('points')],
+  ['radialGradient', GRADIENT],
+  ['rect', set('height', 'rx', 'ry', 'width', 'x', 'y')],
+  ['stop', set('offset')],
+  ['style', set('media', 'type')],
+  ['svg', set('height', 'preserveAspectRatio', 'version', 'viewBox', 'width', 'x', 'y')],
+  ['symbol', set('preserveAspectRatio', 'viewBox')],
+  ['text', TEXT_POSITION],
+  ['title', NONE],
+  ['tspan', TEXT_POSITION],
+  ['use', set('height', 'href', 'width', 'x', 'y')],
+]);
+
+/**
+ * SMIL animation is on this list, not merely absent from the allowlist, because
+ * `<animate attributeName="href" to="javascript:...">` turns an inert element
+ * into a live one after sanitization has already run.
+ *
+ * Shared from `src/core/reading-text.ts` for the same reason as HTML_DISCARDED:
+ * the search extractor mirrors what discarding removes from the frame's text.
+ */
+export { DISCARDED_SVG_ELEMENTS as SVG_DISCARDED } from '../core/reading-text.ts';
+
+/** Attributes carrying a reference, by element local name. */
+export const URL_ATTRIBUTES: ReadonlyMap<string, ReadonlySet<string>> = new Map([
+  ['a', set('href')],
+  ['blockquote', set('cite')],
+  ['del', set('cite')],
+  ['image', set('href')],
+  ['img', set('src')],
+  ['ins', set('cite')],
+  ['link', set('href')],
+  ['q', set('cite')],
+  ['use', set('href')],
+]);
+
+const LINK_SCHEMES = set('http', 'https', 'mailto', 'tel');
+const MEDIA_SCHEMES = set('data', 'http', 'https');
+
+/**
+ * Whether a reference may stay on the attribute at all. Relative references are
+ * kept so they can be resolved to blob URLs; remote ones are kept so CSP is the
+ * layer that refuses them, and refuses them observably.
+ */
+export function isAllowedReference(element: string, attribute: string, kind: string, scheme: string | undefined): boolean {
+  // <use href> reaches into the same document and nowhere else: a data: URI
+  // there is a document-injection vector, not an image reference.
+  if (element === 'use') return kind === 'fragment';
+  if (kind === 'fragment') return element === 'a';
+  if (kind === 'relative') return true;
+  if (kind !== 'scheme' || scheme === undefined) return false;
+  if (attribute === 'src' || (element === 'image' && attribute === 'href') || element === 'link') {
+    return MEDIA_SCHEMES.has(scheme);
+  }
+  return LINK_SCHEMES.has(scheme);
+}