pi-unsloth-webtools 0.2.5 → 0.3.1

package/README.md

@@ -26,33 +26,60 @@ Mirrors Unsloth Studio's `web_search` tool:
 
 - Searches exactly like Studio's pinned `ddgs==9.14.4` `DDGS.text()`: the same seven engines
   (duckduckgo, brave, google, mojeek, yahoo, yandex, wikipedia; bing is disabled upstream),
-  the same provider deduplication, href-dedupe aggregator with frequency ordering, and the
-  same `SimpleFilterRanker` re-ranking. Formats results identically: `Title:` / `URL:` /
+  the same provider deduplication, href-dedupe aggregator with frequency ordering (hrefs are
+  canonicalized first — `utm_*`/tracking parameters and fragments are dropped and the URL is
+  re-serialized, collapsing host-case, default-port, and trailing-slash variants — so the
+  same page found via different tracking links collapses), and the same `SimpleFilterRanker`
+  re-ranking. Formats results identically: `Title:` / `URL:` /
   `Snippet:` blocks separated by `---`, ending with the hint to pass `{"url": "<URL>"}` to
   read a full page.
-- Accepts an optional `url` parameter; when given, fetches that page's text instead of searching.
+- Accepts an optional `url` parameter; when given, fetches that page's text instead of
+  searching (optionally truncated with `maxChars`).
 - Rate-limit, timeout, and empty-result messages mirror Studio's `_search_failure_message`.
+- Transient engine failures (network errors or null responses) are retried once with a short
+  backoff inside the same timeout budget (a retry that cannot fit in the remaining budget is
+  skipped); timeouts and cancellations are never retried.
+- Sweeps stop as soon as enough results are gathered: engines still in flight are aborted
+  instead of being allowed to run to their timeout.
 
 ### web_fetch
 
 Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 
 - URL scheme normalization (bare hosts like `google.com` become `https://google.com`).
-- URL validation: http/https only, no credentials or encoded hostnames, hostname/port checks.
+- URL validation: http/https only, no credentials or encoded hostnames, hostname/port checks (any
+  port 1–65535 is permitted; SSRF protection is enforced at the resolved-IP layer, not by port
+  allowlists).
+  Canonical public IPv4 literals are accepted like IPv6 literals; private literals are still
+  blocked at the resolved-IP layer.
 - DNS resolution with SSRF protection: every resolved address is validated against
   private/loopback/link-local/CGNAT/documentation/multicast/reserved ranges, then the validated IP
   is pinned for the connection (custom `lookup` + SNI `servername`), so DNS cannot rebind between
-  validation and fetch.
+  validation and fetch; resolution shares the caller's abort signal and the overall deadline,
+  so a stuck resolver cannot outlive the fetch.
+  When a host publishes both IPv4 and IPv6 addresses, IPv4 is preferred (broken IPv6 routes
+  cannot stall a fetch), and a connection failure falls back to the next validated address for
+  the same host before giving up.
 - GitHub repo root pages are rewritten to the unauthenticated README API
-  (`Accept: application/vnd.github.raw+json`), falling back to the HTML page on failure.
-- HTTP 4xx/5xx responses are reported as errors with the status reason instead of
+  (`Accept: application/vnd.github.raw+json`), falling back to the raw README URL
+  (`raw.githubusercontent.com`, no API rate limit) and then to the HTML page on failure.
   returning error-page content.
 - Up to 4 redirect hops, each re-validated and re-resolved against the same rules.
 - 512 KiB download cap (10 MiB for PDFs), overall deadline + per-hop socket timeouts, abort-aware
-  (`signal` cancels mid-flight).
+  (`signal` cancels mid-flight). Fetches cut off by a download cap are marked with a trailing
+  truncation notice, so a partial page is not mistaken for a complete one.
+- Responses sent with a `Content-Encoding` of gzip, deflate, or brotli (servers that ignore the
+  `Accept-Encoding: identity` request) are decompressed while streaming, so the download caps
+  bound the decoded page text (a gzip'd PDF still gets the 10 MiB PDF budget via magic sniffing on
+  the decoded head) and a stream cut by the cap or a mid-stream decode failure returns the readable
+  decoded prefix with the truncation notice instead of a binary-content error. A 64 MiB
+  decoded-output cap bounds a compressed bomb.
+- Long fetches and searches report a short progress note to the session before they start, so
+  slow tool calls are not silent.
 - PDF text extraction via the official MuPDF.js engine (the same C library pymupdf wraps):
   object streams, all filters, ToUnicode fonts, encryption detection, and a
-  pymupdf4llm-style markdown layer (headings, bold/italic, code fences, links, tables)
+  pymupdf4llm-style markdown layer (headings, bold/italic, code fences, links, tables),
+  running header/footer and page-number stripping,
   with Studio's corrupted/incomplete fallback to plain text.
 - Content sniffing: MIME allow/deny, binary magic signatures, PDF magic detection, and charset
   decoding (BOM sniffing for UTF-8/16/32 first, then the declared charset, `<meta charset>` sniffing for
@@ -62,10 +89,16 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   stripping (`hidden`, `aria-hidden`, inline styles); `<article>`/`<main>` main-content scoping
   with link-density header stripping; boilerplate-line removal.
 - No page-size budget: fetched pages and PDFs are returned in full (Studio's window-aware
-  cap is deliberately dropped; the optional `maxChars` parameter still truncates when given).
+  cap is deliberately dropped; the optional `maxChars` parameter still truncates when given,
+  on `web_fetch` and on `web_search`'s url mode).
   The 512 KiB / 10 MiB download caps still bound the raw fetch.
 - HTML entity decoding replicates CPython's `html.unescape` (full 2,231-entry HTML5 table,
   longest-prefix rule, Windows-1252 numeric mappings), matching Studio byte-for-byte.
+- Fetched HTML pages are prefixed with the decoded document `<title>`, so the model can
+  see which page it is reading. `Author:` (`meta name=author` / `article:author` / `dc.creator`),
+  `Date:` (`article:published_time` / `dc.date` / `date`) and `Site:` (`og:site_name` /
+  `application-name`) lines are added when declared, so the model can judge recency and
+  provenance.
 
 ## Known differences from Studio
 
@@ -73,6 +106,11 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   whole line instead of per-span; superscript, subscript, underline, strikeout, and
   highlight markers are not emitted. Tables use a conservative text-grid detector:
   aligned text tables are detected, drawn-rule-only tables are not.
+- Running headers and footers: lines repeated at the same page-edge position on at
+  least half the pages (two pages minimum) are dropped from the markdown layer, as is
+  any numeric-only line at a fixed edge position where page numbers appear on at least
+  half the pages (so a one-off number sharing that position is dropped too, while fused
+  labels like `Page 3 of 12` survive). Studio and pymupdf4llm return them verbatim.
 - Search engines: Node's `fetch` TLS fingerprint differs from ddgs's `primp`
   impersonation, so Google/Brave/Yahoo/Yandex may block or serve consent pages more
   aggressively (a blocked engine simply contributes no results). User agents are a
@@ -85,6 +123,13 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   worst-case wall time.
 - Proxies: Studio routes through environment proxies; this port always connects
   directly with DNS pinning (deliberately out of scope).
+- Dedup and titles: the aggregator keys on canonicalized hrefs (`utm_*`/tracking parameters
+  and fragments stripped, then the URL re-serialized); fetched HTML pages are prefixed with
+  the document `<title>`. Studio keys on raw hrefs and returns the converted body alone.
+- Upstream drift: current ddgs ships ten backends (adding bing, startpage, grokipedia),
+  requires a `vqd` token for DuckDuckGo, and exposes an `extract()` mode. This port
+  deliberately pins the Studio snapshot — seven engines, bing disabled upstream, no vqd,
+  no pagination — so engine behavior matches Studio rather than ddgs head.
 
 ## Development
 
@@ -119,7 +164,8 @@ The suite ports Unsloth Studio's own tests for these tools:
   ASCII85Decode, font `/Differences` encodings, pymupdf4llm-style headings/links/tables
 - `test/entities.test.ts`: `decodeHtmlEntities` parity with CPython `html.unescape`,
   legacy refs, longest-prefix rule, Windows-1252 numeric mappings, invalid codepoints
-- `test/smoke.test.ts`: live network checks against real hosts
+- `test/smoke.test.ts`: live network checks against real hosts, including a per-engine
+  result-health sweep (at least three engines must return well-formed results)
 
 The seams (`seams.resolve` / `seams.request` / `rawFetch`) replace the network stack
 with fakes, mirroring how the Studio suite monkeypatches `_validate_and_resolve_host`

package/engines.ts

@@ -1,5 +1,5 @@
 import { randomBytes } from "node:crypto";
-import { decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
+import { collapseWhitespace, decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { randomUserAgent } from "./user-agents.ts";
 export class EmptySweepError extends Error {
@@ -35,7 +35,7 @@ export function normalizeText(raw: string): string {
   text = decodeHtmlEntities(text);
   text = text.normalize("NFC");
   text = text.replace(/[\p{Cc}\p{Cf}\p{Co}\p{Cs}\p{Cn}]/gu, "");
-  return text.trim().split(/\s+/).join(" ");
+  return collapseWhitespace(text);
 }
 
 export function normalizeUrl(url: string): string {
@@ -47,6 +47,41 @@ export function normalizeUrl(url: string): string {
   }
 }
 
+const TRACKING_PARAM_NAMES = new Set([
+  "_hsenc",
+  "_hsmi",
+  "dclid",
+  "fbclid",
+  "gbraid",
+  "gclid",
+  "gclsrc",
+  "igshid",
+  "mc_cid",
+  "mc_eid",
+  "msclkid",
+  "srsltid",
+  "twclid",
+  "wbraid",
+  "yclid",
+]);
+
+export function canonicalizeHref(href: string): string {
+  if (!href) return "";
+  try {
+    const url = new URL(href);
+    for (const key of [...url.searchParams.keys()]) {
+      const lower = key.toLowerCase();
+      if (lower.startsWith("utm_") || TRACKING_PARAM_NAMES.has(lower)) {
+        url.searchParams.delete(key);
+      }
+    }
+    url.hash = "";
+    return url.toString();
+  } catch {
+    return href;
+  }
+}
+
 export interface DomNode {
   tag: string;
   attrs: Record<string, string>;
@@ -382,7 +417,7 @@ export function extractResults(
       ["body", elementsXpath.body],
     ] as const;
     for (const [key, value] of entries) {
-      const data = xpathText(value, item).join("").trim().split(/\s+/).join(" ");
+      const data = collapseWhitespace(xpathText(value, item).join(""));
       if (!data) continue;
       result[key] = key === "href" ? normalizeUrl(data) : normalizeText(data);
     }
@@ -444,15 +479,22 @@ export interface Engine {
   ): Promise<SearchResult[] | null>;
 }
 
+interface HttpRequestOptions {
+  headers?: Record<string, string>;
+  cookies?: Record<string, string>;
+  timeoutMs: number;
+  signal?: AbortSignal;
+}
+
+interface HttpOptions extends HttpRequestOptions {
+  method?: string;
+  body?: string;
+}
+
 async function httpGet(
   url: string,
   params: Record<string, string>,
-  options: {
-    headers?: Record<string, string>;
-    cookies?: Record<string, string>;
-    timeoutMs: number;
-    signal?: AbortSignal;
-  },
+  options: HttpRequestOptions,
 ): Promise<string | null> {
   const target = new URL(url);
   for (const [key, value] of Object.entries(params)) target.searchParams.set(key, value);
@@ -462,17 +504,15 @@ async function httpGet(
 async function httpPost(
   url: string,
   data: Record<string, string>,
-  options: {
-    headers?: Record<string, string>;
-    cookies?: Record<string, string>;
-    timeoutMs: number;
-    signal?: AbortSignal;
-  },
+  options: HttpRequestOptions,
 ): Promise<string | null> {
   return httpFetch(url, { ...options, method: "POST", body: new URLSearchParams(data).toString() });
 }
 
 const MAX_ENGINE_RESPONSE_BYTES = 5 * 1024 * 1024;
+const ENGINE_RETRY_BACKOFF_MS = 250;
+
+const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
 
 async function readBodyCapped(response: Response): Promise<string | null> {
   const declared = Number(response.headers.get("content-length") ?? "0");
@@ -494,16 +534,15 @@ async function readBodyCapped(response: Response): Promise<string | null> {
   return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
 }
 
+function mapFetchError(err: unknown): never {
+  if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
+  if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
+  throw err;
+}
+
 async function httpFetch(
   url: string,
-  options: {
-    method?: string;
-    body?: string;
-    headers?: Record<string, string>;
-    cookies?: Record<string, string>;
-    timeoutMs: number;
-    signal?: AbortSignal;
-  },
+  options: HttpOptions,
 ): Promise<string | null> {
   const headers: Record<string, string> = {
     "User-Agent": options.headers?.["User-Agent"] ?? randomUserAgent(),
@@ -528,17 +567,20 @@ async function httpFetch(
       signal: AbortSignal.any(signals),
     });
   } catch (err) {
-    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
-    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
-    throw err;
+    throw mapFetchError(err);
+  }
+  if (response.status !== 200) {
+    try {
+      await response.body?.cancel();
+    } catch {
+      return null;
+    }
+    return null;
   }
-  if (response.status !== 200) return null;
   try {
     return await readBodyCapped(response);
   } catch (err) {
-    if (err instanceof DOMException && err.name === "TimeoutError") throw new SearchTimeoutError();
-    if (err instanceof DOMException && err.name === "AbortError") throw new SearchCancelled();
-    throw err;
+    throw mapFetchError(err);
   }
 }
 
@@ -649,7 +691,7 @@ const MOJEEK: Engine = {
 const YAHOO: Engine = {
   name: "yahoo",
   provider: "bing",
-  async search(query, ctx, timeoutMs, signal) {
+  async search(query, _ctx, timeoutMs, signal) {
     const ylt = tokenUrlSafe(18);
     const ylu = tokenUrlSafe(35);
     const html = await httpGet(
@@ -675,7 +717,7 @@ const YAHOO: Engine = {
 const YANDEX: Engine = {
   name: "yandex",
   provider: "yandex",
-  async search(query, ctx, timeoutMs, signal) {
+  async search(query, _ctx, timeoutMs, signal) {
     const searchid = 1000000 + Math.floor(Math.random() * 9000000);
     const html = await httpGet(
       "https://yandex.com/search/site/",
@@ -734,7 +776,7 @@ const WIKIPEDIA: Engine = {
   },
 };
 
-const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, BRAVE, GOOGLE, MOJEEK, YAHOO, YANDEX, WIKIPEDIA];
+export const TEXT_ENGINES: Engine[] = [DUCKDUCKGO, BRAVE, GOOGLE, MOJEEK, YAHOO, YANDEX, WIKIPEDIA];
 
 export class ResultsAggregator {
   private cache = new Map<string, SearchResult>();
@@ -745,10 +787,10 @@ export class ResultsAggregator {
   }
 
   append(item: SearchResult): void {
-    const key = item.href;
+    const key = canonicalizeHref(item.href);
     const existing = this.cache.get(key);
     if (!existing || item.body.length > existing.body.length) {
-      this.cache.set(key, item);
+      this.cache.set(key, { ...item, href: key });
     }
     this.counter.set(key, (this.counter.get(key) ?? 0) + 1);
   }
@@ -821,6 +863,7 @@ export async function autoTextSearch(
   const seenProviders = new Set<string>();
   const aggregator = new ResultsAggregator();
   const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
+  const controller = new AbortController();
   let timedOut = false;
   let cancelled = false;
   const uniqueProviders = new Set(engines.map((e) => e.provider)).size;
@@ -828,20 +871,50 @@ export async function autoTextSearch(
   let i = 0;
   let pending: Promise<void>[] = [];
   const run = async (engine: Engine) => {
-    const remaining = Math.max(1, deadline - Date.now());
-    try {
-      const results = await engine.search(query, ctx, remaining, signal);
-      if (results && results.length) {
-        aggregator.extend(results);
-        seenProviders.add(engine.provider);
+    let results: SearchResult[] | null = null;
+    for (let attempt = 0; attempt < 2 && results === null; attempt++) {
+      if (controller.signal.aborted) return;
+      const budgetLeft = deadline - Date.now();
+      if (budgetLeft <= 0) return;
+      if (attempt > 0 && budgetLeft < ENGINE_RETRY_BACKOFF_MS) return;
+      const remaining = Math.max(1, budgetLeft);
+      const engineSignal = signal
+        ? AbortSignal.any([signal, controller.signal])
+        : controller.signal;
+      try {
+        results = await engine.search(query, ctx, remaining, engineSignal);
+      } catch (e) {
+        if (e instanceof SearchCancelled) {
+          if (controller.signal.aborted && !signal?.aborted) return;
+          cancelled = true;
+          return;
+        }
+        if (e instanceof SearchTimeoutError) {
+          timedOut = true;
+          return;
+        }
+      }
+      if (results === null && attempt === 0) {
+        const backoff = Math.min(ENGINE_RETRY_BACKOFF_MS, Math.max(0, deadline - Date.now()));
+        if (backoff > 0) await sleep(backoff);
+        if (signal?.aborted) {
+          cancelled = true;
+          return;
+        }
+        if (controller.signal.aborted) return;
       }
-    } catch (e) {
-      if (e instanceof SearchCancelled) cancelled = true;
-      if (e instanceof SearchTimeoutError) timedOut = true;
+    }
+    if (results && results.length) {
+      aggregator.extend(results);
+      seenProviders.add(engine.provider);
+      if (aggregator.size >= maxResults) controller.abort();
     }
   };
   while (i < engines.length) {
-    if (aggregator.size >= maxResults) break;
+    if (aggregator.size >= maxResults || cancelled) {
+      controller.abort();
+      break;
+    }
     const engine = engines[i++];
     if (seenProviders.has(engine.provider)) continue;
     pending.push(run(engine));

package/html-to-md.ts

@@ -194,7 +194,7 @@ class HeaderFrame {
 const CHARREF_RE = /&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)/g;
 
 export function decodeHtmlEntities(text: string): string {
-  return text.replace(CHARREF_RE, (whole, s: string) => {
+  return text.replace(CHARREF_RE, (_whole, s: string) => {
     if (s[0] === "#") {
       const hex = s[1] === "x" || s[1] === "X";
       const num = parseInt(s.slice(hex ? 2 : 1).replace(/;+$/, ""), hex ? 16 : 10);
@@ -214,6 +214,10 @@ export function decodeHtmlEntities(text: string): string {
   });
 }
 
+export function collapseWhitespace(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}
+
 
 interface HtmlHandlers {
   handleStartTag(name: string, attrs: AttrDict): void;
@@ -224,8 +228,11 @@ interface HtmlHandlers {
   handleCharRef(name: string): void;
 }
 
-const START_TAG_NAME_RE = /^[a-zA-Z][^\s/>]*/;
-const ATTR_NAME_RE = /^[^\s=/>]+/;
+const START_TAG_NAME_RE = /[a-zA-Z][^\s/>]*/y;
+const ATTR_NAME_RE = /[^\s=/>]+/y;
+const ENTITY_NAMED_RE = /&([A-Za-z][A-Za-z0-9.-]*);/y;
+const ENTITY_NUMERIC_RE = /&#([xX][0-9a-fA-F]+|[0-9]+);?/y;
+const ENTITY_LEGACY_RE = /&([A-Za-z][A-Za-z0-9.-]*)(?=[^A-Za-z0-9]|$)/y;
 
 const RAW_TEXT_TAGS = [
   "script",
@@ -238,9 +245,23 @@ const RAW_TEXT_TAGS = [
   "noframes",
 ];
 
-const RAW_TEXT_CLOSERS: Record<string, RegExp> = Object.fromEntries(
-  RAW_TEXT_TAGS.map((name) => [name, new RegExp(`</${name}\\s*>`, "i")]),
-);
+function findRawTextClose(
+  input: string,
+  lowerInput: string,
+  from: number,
+  name: string,
+): { start: number; end: number } | null {
+  const needle = `</${name}`;
+  let pos = from;
+  while (true) {
+    const hit = lowerInput.indexOf(needle, pos);
+    if (hit === -1) return null;
+    let j = hit + needle.length;
+    while (j < input.length && /\s/.test(input[j])) j++;
+    if (j < input.length && input[j] === ">") return { start: hit, end: j + 1 };
+    pos = hit + 1;
+  }
+}
 
 function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, boolean] {
   const attrs: AttrDict = {};
@@ -253,10 +274,11 @@ function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, bo
       pos++;
       continue;
     }
-    const nameMatch = ATTR_NAME_RE.exec(input.slice(pos));
+    ATTR_NAME_RE.lastIndex = pos;
+    const nameMatch = ATTR_NAME_RE.exec(input);
     if (!nameMatch) return [attrs, -1, false];
     const name = nameMatch[0].toLowerCase();
-    pos += nameMatch[0].length;
+    pos = nameMatch.index + nameMatch[0].length;
     while (pos < input.length && /\s/.test(input[pos])) pos++;
     let value: string | null = null;
     if (pos < input.length && input[pos] === "=") {
@@ -281,64 +303,69 @@ function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, bo
 }
 
 function scanTag(
-  html: string,
+  input: string,
   i: number,
 ): { end: number; kind: "comment" | "decl" | "end" | "start" | "startend"; name?: string; attrs?: AttrDict } | null {
-  const rest = html.slice(i + 1);
-  if (rest.startsWith("!--")) {
-    const close = html.indexOf("-->", i + 4);
+  if (input.startsWith("!--", i + 1)) {
+    const close = input.indexOf("-->", i + 4);
     if (close === -1) return null;
     return { end: close + 3, kind: "decl" };
   }
-  if (rest.startsWith("!") || rest.startsWith("?")) {
+  const next = input[i + 1];
+  if (next === "!" || next === "?") {
     let j = i + 2;
-    while (j < html.length && html[j] !== ">") j++;
-    if (j >= html.length) return null;
+    while (j < input.length && input[j] !== ">") j++;
+    if (j >= input.length) return null;
     return { end: j + 1, kind: "decl" };
   }
-  if (rest.startsWith("/")) {
+  if (next === "/") {
     let j = i + 2;
-    while (j < html.length && /[\s>]/.test(html[j]) === false) j++;
-    const name = html.slice(i + 2, j).toLowerCase();
+    while (j < input.length && !/[\s>]/.test(input[j])) j++;
+    const name = input.slice(i + 2, j).toLowerCase();
     if (!name) return null;
-    while (j < html.length && html[j] !== ">") j++;
-    if (j >= html.length) return null;
+    while (j < input.length && input[j] !== ">") j++;
+    if (j >= input.length) return null;
     return { end: j + 1, kind: "end", name };
   }
-  const nameMatch = START_TAG_NAME_RE.exec(rest);
+  START_TAG_NAME_RE.lastIndex = i + 1;
+  const nameMatch = START_TAG_NAME_RE.exec(input);
   if (!nameMatch) return null;
   const name = nameMatch[0].toLowerCase();
-  const [attrs, next, selfClosing] = parseAttrsUntilClose(html, i + 1 + nameMatch[0].length);
-  if (next === -1) return null;
-  return { end: next, kind: selfClosing ? "startend" : "start", name, attrs };
+  const [attrs, nextPos, selfClosing] = parseAttrsUntilClose(input, i + 1 + nameMatch[0].length);
+  if (nextPos === -1) return null;
+  return { end: nextPos, kind: selfClosing ? "startend" : "start", name, attrs };
 }
 
 
 export function feedHtml(input: string, handlers: HtmlHandlers): void {
-  const emitText = (text: string) => {
-    if (!text) return;
-    let pos = 0;
-    while (pos < text.length) {
-      const amp = text.indexOf("&", pos);
-      if (amp === -1) {
-        handlers.handleData(text.slice(pos));
+  let lowerInput: string | null = null;
+  const emitTextRange = (start: number, end: number) => {
+    if (end <= start) return;
+    let pos = start;
+    while (pos < end) {
+      const amp = input.indexOf("&", pos);
+      if (amp === -1 || amp >= end) {
+        handlers.handleData(input.slice(pos, end));
         return;
       }
-      if (amp > pos) handlers.handleData(text.slice(pos, amp));
-      const named = /^&([A-Za-z][A-Za-z0-9.-]*);/.exec(text.slice(amp));
-      if (named) {
+      if (amp > pos) handlers.handleData(input.slice(pos, amp));
+      ENTITY_NAMED_RE.lastIndex = amp;
+      const named = ENTITY_NAMED_RE.exec(input);
+      if (named && named.index === amp && amp + named[0].length <= end) {
         handlers.handleEntityRef(named[1]);
         pos = amp + named[0].length;
         continue;
       }
-      const numeric = /^&#(?:[xX]([0-9a-fA-F]+)|([0-9]+));/.exec(text.slice(amp));
-      if (numeric) {
-        handlers.handleCharRef(numeric[1] ?? numeric[2]);
+      ENTITY_NUMERIC_RE.lastIndex = amp;
+      const numeric = ENTITY_NUMERIC_RE.exec(input);
+      if (numeric && numeric.index === amp && amp + numeric[0].length <= end) {
+        handlers.handleCharRef(numeric[1]);
         pos = amp + numeric[0].length;
         continue;
       }
-      const legacy = /^&([A-Za-z][A-Za-z0-9.-]*)(?=[^A-Za-z0-9]|$)/.exec(text.slice(amp));
-      if (legacy) {
+      ENTITY_LEGACY_RE.lastIndex = amp;
+      const legacy = ENTITY_LEGACY_RE.exec(input);
+      if (legacy && legacy.index === amp && amp + legacy[0].length <= end) {
         handlers.handleEntityRef(legacy[1]);
         pos = amp + legacy[0].length;
         continue;
@@ -360,28 +387,32 @@ export function feedHtml(input: string, handlers: HtmlHandlers): void {
       i++;
       continue;
     }
-    emitText(input.slice(textStart, i));
+    emitTextRange(textStart, i);
     if (tag.kind === "start") {
-      const rawCloser = RAW_TEXT_CLOSERS[tag.name!];
-      if (rawCloser) {
-        const after = input.slice(tag.end);
-        const closeMatch = rawCloser.exec(after);
-        if (closeMatch) {
-          handlers.handleStartTag(tag.name!, tag.attrs!);
-          emitText(after.slice(0, closeMatch.index));
-          handlers.handleEndTag(tag.name!);
-          textStart = tag.end + closeMatch.index + closeMatch[0].length;
+      const rawName = tag.name!;
+      if (RAW_TEXT_TAGS.includes(rawName)) {
+        lowerInput ??= input.toLowerCase();
+        const close = findRawTextClose(input, lowerInput, tag.end, rawName);
+        if (close) {
+          handlers.handleStartTag(rawName, tag.attrs!);
+          emitTextRange(tag.end, close.start);
+          handlers.handleEndTag(rawName);
+          textStart = close.end;
           i = textStart;
           continue;
         }
       }
-      handlers.handleStartTag(tag.name!, tag.attrs!);
+      handlers.handleStartTag(rawName, tag.attrs!);
     } else if (tag.kind === "startend") handlers.handleStartEndTag(tag.name!, tag.attrs!);
     else if (tag.kind === "end") handlers.handleEndTag(tag.name!);
     textStart = tag.end;
     i = tag.end;
   }
-  emitText(input.slice(textStart));
+  emitTextRange(textStart, input.length);
+}
+
+function popMarksAbove(marks: number[], index: number): void {
+  while (marks.length && marks[marks.length - 1] >= index) marks.pop();
 }
 
 class MarkdownRenderer {
@@ -524,8 +555,8 @@ class MarkdownRenderer {
   }
 
   private finishLink(): void {
-    const text = this.linkTextParts.join("").replace(/\s+/g, " ").trim();
-    const headingText = this.linkHeadingParts.join("").replace(/\s+/g, " ").trim();
+    const text = collapseWhitespace(this.linkTextParts.join(""));
+    const headingText = collapseWhitespace(this.linkHeadingParts.join(""));
     const href = this.linkHref ?? "";
     this.inLink = false;
     this.linkTextParts = [];
@@ -571,12 +602,8 @@ class MarkdownRenderer {
       }
       if (closeAt === null) break;
       this.truncateOpenTags(closeAt);
-      while (this.hiddenMarks.length && this.hiddenMarks[this.hiddenMarks.length - 1] >= closeAt) {
-        this.hiddenMarks.pop();
-      }
-      while (this.headingMarks.length && this.headingMarks[this.headingMarks.length - 1] >= closeAt) {
-        this.headingMarks.pop();
-      }
+      popMarksAbove(this.hiddenMarks, closeAt);
+      popMarksAbove(this.headingMarks, closeAt);
       this.closeHeaderFrames(closeAt);
     }
   }
@@ -692,12 +719,8 @@ class MarkdownRenderer {
       for (let i = this.openTags.length - 1; i >= 0; i--) {
         if (this.openTags[i] === tag) {
           this.truncateOpenTags(i);
-          while (this.hiddenMarks.length && this.hiddenMarks[this.hiddenMarks.length - 1] >= i) {
-            this.hiddenMarks.pop();
-          }
-          while (this.headingMarks.length && this.headingMarks[this.headingMarks.length - 1] >= i) {
-            this.headingMarks.pop();
-          }
+          popMarksAbove(this.hiddenMarks, i);
+          popMarksAbove(this.headingMarks, i);
           this.closeHeaderFrames(i, tag === "header");
           break;
         }
@@ -715,7 +738,11 @@ class MarkdownRenderer {
     return !suppressed;
   }
 
-  handleStartEndTag(_name: string, _attrs: AttrDict): void {}
+  handleStartEndTag(name: string, attrs: AttrDict): void {
+    if (!VOID_TAGS.has(name)) return;
+    this.handleStartTag(name, attrs);
+    this.handleEndTag(name);
+  }
   handleStartTag(tag: string, attrs: AttrDict): void {
     if (this.skipDepth) {
       if (SKIP_TAGS.has(tag)) this.skipDepth++;