pi-unsloth-webtools 0.3.0 → 0.4.0

package/README.md

@@ -39,6 +39,8 @@ Mirrors Unsloth Studio's `web_search` tool:
 - Transient engine failures (network errors or null responses) are retried once with a short
   backoff inside the same timeout budget (a retry that cannot fit in the remaining budget is
   skipped); timeouts and cancellations are never retried.
+- Sweeps stop as soon as enough results are gathered: engines still in flight are aborted
+  instead of being allowed to run to their timeout.
 
 ### web_fetch
 
@@ -48,16 +50,22 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
 - URL validation: http/https only, no credentials or encoded hostnames, hostname/port checks (any
   port 1–65535 is permitted; SSRF protection is enforced at the resolved-IP layer, not by port
   allowlists).
+  Canonical public IPv4 literals are accepted like IPv6 literals; private literals are still
+  blocked at the resolved-IP layer (non-canonical numeric encodings like `0x7f.0.0.1` /
+  `013.0.0.1` / `2130706433` are rejected by URL validation before DNS).
 - DNS resolution with SSRF protection: every resolved address is validated against
   private/loopback/link-local/CGNAT/documentation/multicast/reserved ranges, then the validated IP
   is pinned for the connection (custom `lookup` + SNI `servername`), so DNS cannot rebind between
   validation and fetch; resolution shares the caller's abort signal and the overall deadline,
   so a stuck resolver cannot outlive the fetch.
+  When a host publishes both IPv4 and IPv6 addresses, IPv4 is preferred (broken IPv6 routes
+  cannot stall a fetch), and a connection failure falls back to the next validated address for
+  the same host before giving up.
 - GitHub repo root pages are rewritten to the unauthenticated README API
   (`Accept: application/vnd.github.raw+json`), falling back to the raw README URL
   (`raw.githubusercontent.com`, no API rate limit) and then to the HTML page on failure.
   returning error-page content.
-- Up to 4 redirect hops, each re-validated and re-resolved against the same rules.
+- Up to 5 total requests (initial fetch + up to 4 HTTP redirects or `<meta http-equiv="refresh">` refreshes), each re-validated and re-resolved against the same rules.
 - 512 KiB download cap (10 MiB for PDFs), overall deadline + per-hop socket timeouts, abort-aware
   (`signal` cancels mid-flight). Fetches cut off by a download cap are marked with a trailing
   truncation notice, so a partial page is not mistaken for a complete one.
@@ -124,6 +132,50 @@ Port of Studio's `_fetch_page_text` / `_fetch_url_raw` pipeline:
   deliberately pins the Studio snapshot — seven engines, bing disabled upstream, no vqd,
   no pagination — so engine behavior matches Studio rather than ddgs head.
 
+## When to use alternatives
+
+This package is intentionally a faithful, zero-dependency port of Studio's pipeline. Use it
+when you need deterministic, offline-friendly behavior with strong SSRF guarantees and test
+parity with `unsloth/studio`. For other tradeoffs, prefer:
+
+| Need | Use |
+|---|---|
+| Browser-like TLS/HTTP fingerprinting to unblock bot-defended pages | `pi-smart-fetch` (`wreq-js` `chrome_145`) |
+| Headless Chrome for JS-rendered SPAs/YouTube/Reddit threads | `georgebashi/pi-web-fetch` (puppeteer + trafilatura) |
+| Hosted search with semantic ranking and no scraping | `Brave Search API` / `Tavily` / `Exa` via `pi-ollama-web-search` |
+| Prompt-focused page distillation to save context | `pi-web-fetch` `prompt` -> sub-agent or Claude Code `WebFetch(url,prompt)` |
+| Batch fetching many URLs concurrently | `pi-smart-fetch` `batch_web_fetch` or call `web_fetch` in parallel |
+
+Mixing is supported: `pi install npm:pi-unsloth-webtools npm:pi-smart-fetch` lets the model
+choose the best tool per URL. No need to fork this package to add those features.
+
+## Configuration
+
+Optional settings in `~/.pi/agent/settings.json` or `.pi/settings.json` (project overrides global):
+
+```json
+{
+  "unslothWebTools": {
+    "maxResults": 5,
+    "maxChars": 50000,
+    "timeoutMs": 60000
+  },
+  "webFetch": {
+    "maxChars": 50000,
+    "timeoutMs": 15000
+  }
+}
+```
+
+| Key | Default | Description |
+|---|---|---|
+| `unslothWebTools.maxResults` | `5` | Default `maxResults` for `web_search` (clamped 1-20) |
+| `unslothWebTools.maxChars` / `webFetch.maxChars` / `smartFetchDefaultMaxChars` | tool param | Default `maxChars` for `web_fetch` and `web_search` url mode |
+| `unslothWebTools.timeoutMs` / `webFetch.timeoutMs` / `smartFetchDefaultTimeoutMs` | `60000` | Default `timeoutMs` (>=1000) |
+| `webSearch.maxResults` / `smartWebSearch.resultsPerQuery` | same as above | Legacy aliases for `maxResults` |
+
+Tool params always win over file defaults.
+
 ## Development
 
 ```sh
@@ -158,7 +210,8 @@ The suite ports Unsloth Studio's own tests for these tools:
 - `test/entities.test.ts`: `decodeHtmlEntities` parity with CPython `html.unescape`,
   legacy refs, longest-prefix rule, Windows-1252 numeric mappings, invalid codepoints
 - `test/smoke.test.ts`: live network checks against real hosts, including a per-engine
-  result-health sweep (at least three engines must return well-formed results)
+  result-health sweep (at least two engines must return well-formed results; engines
+  that block or reset connections from datacenter IPs count as unhealthy, not failures)
 
 The seams (`seams.resolve` / `seams.request` / `rawFetch`) replace the network stack
 with fakes, mirroring how the Studio suite monkeypatches `_validate_and_resolve_host`

package/agent-dir.ts

@@ -0,0 +1,19 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+function expandAgentDir(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed === "~") return homedir();
+  if (trimmed.startsWith("~/") || trimmed.startsWith("~\\")) return join(homedir(), trimmed.slice(2));
+  return trimmed;
+}
+
+export function agentDir(): string {
+  const env = (process.env.PI_CODING_AGENT_DIR ?? process.env.PI_AGENT_DIR)?.trim();
+  if (env) return expandAgentDir(env);
+  try {
+    const home = homedir();
+    if (home) return join(home, ".pi", "agent");
+  } catch {}
+  return "";
+}

package/cache.ts

@@ -0,0 +1,154 @@
+import { createHash, randomBytes } from "node:crypto";
+import { mkdir, readdir, readFile, rename, stat, unlink, writeFile, chmod } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { agentDir } from "./agent-dir.ts";
+import { canonicalizeHref } from "./engines.ts";
+
+const CACHE_TTL_MS = 60 * 60 * 1000;
+const MAX_ENTRIES = 128;
+const MAX_BYTES = 128 * 1024 * 1024;
+const STALE_MAX_AGE_MS = 24 * CACHE_TTL_MS;
+
+function cacheDir(): string {
+  const env = process.env.PI_UNSLOTH_CACHE_DIR?.trim();
+  if (env) return env;
+  const base = agentDir();
+  if (base) return join(base, "pi-unsloth-cache");
+  return join(tmpdir(), "pi-unsloth-cache");
+}
+
+function cacheKey(url: string): string {
+  const canon = canonicalizeHref(url) || url;
+  return createHash("sha256").update(canon).digest("hex");
+}
+
+function cachePath(key: string): string {
+  return join(cacheDir(), `${key}.json`);
+}
+
+async function secureChmod(path: string, mode: number): Promise<void> {
+  if (process.platform === "win32") return;
+  try {
+    await chmod(path, mode);
+  } catch {}
+}
+
+async function unlinkSafe(path: string): Promise<boolean> {
+  try {
+    await unlink(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function ensureDir(): Promise<void> {
+  const dir = cacheDir();
+  try {
+    await mkdir(dir, { recursive: true, mode: 0o700 });
+    await secureChmod(dir, 0o700);
+  } catch {}
+}
+
+export interface CacheEntry {
+  url: string;
+  body: string;
+  contentType: string;
+  timestamp: number;
+}
+
+export async function getCached(url: string): Promise<CacheEntry | null> {
+  const key = cacheKey(url);
+  const path = cachePath(key);
+  try {
+    const raw = await readFile(path, "utf8");
+    const entry = JSON.parse(raw) as CacheEntry;
+    if (!entry || typeof entry.body !== "string" || typeof entry.contentType !== "string" || typeof entry.timestamp !== "number") return null;
+    if (!Number.isFinite(entry.timestamp)) return null;
+    return entry;
+  } catch {
+    return null;
+  }
+}
+
+export async function setCached(url: string, body: string, contentType: string): Promise<void> {
+  await ensureDir();
+  const key = cacheKey(url);
+  const path = cachePath(key);
+  const entry: CacheEntry = { url: canonicalizeHref(url) || url, body, contentType, timestamp: Date.now() };
+  const tmpPath = `${path}.tmp.${process.pid}.${Date.now()}.${randomBytes(4).toString("hex")}`;
+  try {
+    await writeFile(tmpPath, JSON.stringify(entry), { mode: 0o600 });
+    await secureChmod(tmpPath, 0o600);
+    await rename(tmpPath, path);
+    await secureChmod(path, 0o600);
+  } catch {
+    await unlinkSafe(tmpPath);
+    return;
+  }
+  await prune();
+}
+
+async function prune(): Promise<void> {
+  const dir = cacheDir();
+  let files: string[];
+  try {
+    files = await readdir(dir);
+  } catch {
+    return;
+  }
+  const jsonFiles = files.filter((f) => f.endsWith(".json"));
+  if (!jsonFiles.length) return;
+  const now = Date.now();
+  const entries: { file: string; size: number; timestamp: number }[] = [];
+  let total = 0;
+  for (const file of jsonFiles) {
+    const full = join(dir, file);
+    try {
+      const s = await stat(full);
+      let timestamp = s.mtimeMs;
+      try {
+        const raw = await readFile(full, "utf8");
+        const entry = JSON.parse(raw) as CacheEntry;
+        if (entry && typeof entry.timestamp === "number" && Number.isFinite(entry.timestamp)) timestamp = entry.timestamp;
+      } catch {}
+      if (now - timestamp >= STALE_MAX_AGE_MS) {
+        await unlinkSafe(full);
+        continue;
+      }
+      entries.push({ file: full, size: s.size, timestamp });
+      total += s.size;
+    } catch {}
+  }
+  const expired = entries.filter((e) => now - e.timestamp >= CACHE_TTL_MS).sort((a, b) => a.timestamp - b.timestamp);
+  for (const e of expired) {
+    if (entries.length <= MAX_ENTRIES && total <= MAX_BYTES) break;
+    if (await unlinkSafe(e.file)) {
+      total -= e.size;
+      const idx = entries.indexOf(e);
+      if (idx !== -1) entries.splice(idx, 1);
+    }
+  }
+  if (entries.length <= MAX_ENTRIES && total <= MAX_BYTES) return;
+  entries.sort((a, b) => a.timestamp - b.timestamp);
+  let idx = 0;
+  while (idx < entries.length && (entries.length > MAX_ENTRIES || total > MAX_BYTES)) {
+    const e = entries[idx];
+    if (await unlinkSafe(e.file)) {
+      total -= e.size;
+      entries.splice(idx, 1);
+    } else {
+      idx++;
+    }
+  }
+}
+
+export function isFresh(entry: CacheEntry, now = Date.now()): boolean {
+  return now - entry.timestamp < CACHE_TTL_MS;
+}
+
+export function staleNotice(entry: CacheEntry): string {
+  const date = new Date(entry.timestamp).toISOString().slice(0, 10);
+  return `\n\n*STALE cache from ${date} — network fetch failed, serving cached copy*`;
+}

package/engines.ts

@@ -1,7 +1,11 @@
 import { randomBytes } from "node:crypto";
+import { appendFile, chmod, mkdir } from "node:fs/promises";
+import { dirname, isAbsolute, join } from "node:path";
 import { collapseWhitespace, decodeHtmlEntities, feedHtml } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { randomUserAgent } from "./user-agents.ts";
+import { agentDir } from "./agent-dir.ts";
+import { MAX_SIGNAL_TIMEOUT_MS } from "./web-access.ts";
 export class EmptySweepError extends Error {
   constructor() {
     super("No results found");
@@ -9,8 +13,10 @@ export class EmptySweepError extends Error {
 }
 
 export class SearchTimeoutError extends Error {
-  constructor() {
+  providers: string[];
+  constructor(providers: string[] = []) {
     super("timed out");
+    this.providers = providers;
   }
 }
 
@@ -556,7 +562,8 @@ async function httpFetch(
         .join("; ")
     : null;
   if (cookie) headers["Cookie"] = cookie;
-  const signals: AbortSignal[] = [AbortSignal.timeout(options.timeoutMs)];
+  const timeoutMs = Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, options.timeoutMs));
+  const signals: AbortSignal[] = [AbortSignal.timeout(timeoutMs)];
   if (options.signal) signals.push(options.signal);
   let response: Response;
   try {
@@ -840,6 +847,40 @@ export function rankResults(docs: SearchResult[], query: string): SearchResult[]
   }
   return [...wiki, ...both, ...titleOnly, ...bodyOnly, ...neither];
 }
+async function recordSweepStats(query: string, maxResults: number, started: number, timedOutProviders: string[], resultCount: number): Promise<void> {
+  const flag = process.env.PI_UNSLOTH_WEBTOOLS_STATS?.trim();
+  if (!flag) return;
+  const lower = flag.toLowerCase();
+  if (lower === "0" || lower === "false" || lower === "no" || lower === "off") return;
+  const base = agentDir();
+  if (!base) return;
+  const isDefaultFlag = lower === "1" || lower === "true" || lower === "yes" || lower === "on";
+  const statsPath = isDefaultFlag ? join(base, "pi-unsloth-webtools-stats.jsonl") : isAbsolute(flag) ? flag : join(base, flag);
+  try {
+    const dir = dirname(statsPath);
+    await mkdir(dir, { recursive: true, mode: 0o700 });
+    if (process.platform !== "win32") {
+      try {
+        await chmod(dir, 0o700);
+      } catch {}
+    }
+    const sortedProviders = [...timedOutProviders].sort();
+    const entry = JSON.stringify({
+      ts: new Date().toISOString(),
+      query,
+      maxResults,
+      durationMs: Date.now() - started,
+      timedOutProviders: sortedProviders,
+      resultCount,
+    });
+    await appendFile(statsPath, entry + "\n", { mode: 0o600 });
+    if (process.platform !== "win32") {
+      try {
+        await chmod(statsPath, 0o600);
+      } catch {}
+    }
+  } catch {}
+}
 
 function shuffledEngines(): Engine[] {
   const shuffled = [...TEXT_ENGINES];
@@ -858,12 +899,22 @@ export async function autoTextSearch(
   timeoutMs: number,
   signal?: AbortSignal,
 ): Promise<SearchResult[]> {
+  const started = Date.now();
   const engines = shuffledEngines();
-  const deadline = Date.now() + timeoutMs;
+  const deadline = started + timeoutMs;
   const seenProviders = new Set<string>();
   const aggregator = new ResultsAggregator();
   const ctx: EngineContext = { region: "us-en", safesearch: "moderate" };
-  let timedOut = false;
+  const controller = new AbortController();
+  let onAbort: (() => void) | undefined;
+  if (signal) {
+    if (signal.aborted) controller.abort();
+    else {
+      onAbort = () => controller.abort();
+      signal.addEventListener("abort", onAbort, { once: true });
+    }
+  }
+  const timedOutProviders = new Set<string>();
   let cancelled = false;
   const uniqueProviders = new Set(engines.map((e) => e.provider)).size;
   const maxWorkers = Math.min(uniqueProviders, Math.ceil(maxResults / 10) + 1);
@@ -872,19 +923,27 @@ export async function autoTextSearch(
   const run = async (engine: Engine) => {
     let results: SearchResult[] | null = null;
     for (let attempt = 0; attempt < 2 && results === null; attempt++) {
+      if (controller.signal.aborted) {
+        if (signal?.aborted) cancelled = true;
+        return;
+      }
       const budgetLeft = deadline - Date.now();
       if (budgetLeft <= 0) return;
       if (attempt > 0 && budgetLeft < ENGINE_RETRY_BACKOFF_MS) return;
       const remaining = Math.max(1, budgetLeft);
+      const engineSignal = signal
+        ? AbortSignal.any([signal, controller.signal])
+        : controller.signal;
       try {
-        results = await engine.search(query, ctx, remaining, signal);
+        results = await engine.search(query, ctx, remaining, engineSignal);
       } catch (e) {
         if (e instanceof SearchCancelled) {
+          if (controller.signal.aborted && !signal?.aborted) return;
           cancelled = true;
           return;
         }
         if (e instanceof SearchTimeoutError) {
-          timedOut = true;
+          timedOutProviders.add(engine.name);
           return;
         }
       }
@@ -895,15 +954,20 @@ export async function autoTextSearch(
           cancelled = true;
           return;
         }
+        if (controller.signal.aborted) return;
       }
     }
     if (results && results.length) {
       aggregator.extend(results);
       seenProviders.add(engine.provider);
+      if (aggregator.size >= maxResults) controller.abort();
     }
   };
   while (i < engines.length) {
-    if (aggregator.size >= maxResults || cancelled) break;
+    if (aggregator.size >= maxResults || cancelled) {
+      controller.abort();
+      break;
+    }
     const engine = engines[i++];
     if (seenProviders.has(engine.provider)) continue;
     pending.push(run(engine));
@@ -913,9 +977,18 @@ export async function autoTextSearch(
     }
   }
   await Promise.allSettled(pending);
+  if (onAbort && signal) signal.removeEventListener("abort", onAbort);
   if (cancelled) throw new SearchCancelled();
   const results = rankResults(aggregator.extractDicts(), query);
-  if (results.length) return results.slice(0, maxResults);
-  if (timedOut) throw new SearchTimeoutError();
+  if (results.length) {
+    void recordSweepStats(query, maxResults, started, [...timedOutProviders], results.length);
+    return results.slice(0, maxResults);
+  }
+  if (timedOutProviders.size) {
+    const sorted = [...timedOutProviders].sort();
+    void recordSweepStats(query, maxResults, started, sorted, 0);
+    throw new SearchTimeoutError(sorted);
+  }
+  void recordSweepStats(query, maxResults, started, [], 0);
   throw new EmptySweepError();
 }

package/html-to-md.ts

@@ -228,8 +228,11 @@ interface HtmlHandlers {
   handleCharRef(name: string): void;
 }
 
-const START_TAG_NAME_RE = /^[a-zA-Z][^\s/>]*/;
-const ATTR_NAME_RE = /^[^\s=/>]+/;
+const START_TAG_NAME_RE = /[a-zA-Z][^\s/>]*/y;
+const ATTR_NAME_RE = /[^\s=/>]+/y;
+const ENTITY_NAMED_RE = /&([A-Za-z][A-Za-z0-9.-]*);/y;
+const ENTITY_NUMERIC_RE = /&#([xX][0-9a-fA-F]+|[0-9]+);?/y;
+const ENTITY_LEGACY_RE = /&([A-Za-z][A-Za-z0-9.-]*)(?=[^A-Za-z0-9]|$)/y;
 
 const RAW_TEXT_TAGS = [
   "script",
@@ -242,9 +245,23 @@ const RAW_TEXT_TAGS = [
   "noframes",
 ];
 
-const RAW_TEXT_CLOSERS: Record<string, RegExp> = Object.fromEntries(
-  RAW_TEXT_TAGS.map((name) => [name, new RegExp(`</${name}\\s*>`, "i")]),
-);
+function findRawTextClose(
+  input: string,
+  lowerInput: string,
+  from: number,
+  name: string,
+): { start: number; end: number } | null {
+  const needle = `</${name}`;
+  let pos = from;
+  while (true) {
+    const hit = lowerInput.indexOf(needle, pos);
+    if (hit === -1) return null;
+    let j = hit + needle.length;
+    while (j < input.length && /\s/.test(input[j])) j++;
+    if (j < input.length && input[j] === ">") return { start: hit, end: j + 1 };
+    pos = hit + 1;
+  }
+}
 
 function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, boolean] {
   const attrs: AttrDict = {};
@@ -257,10 +274,11 @@ function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, bo
       pos++;
       continue;
     }
-    const nameMatch = ATTR_NAME_RE.exec(input.slice(pos));
+    ATTR_NAME_RE.lastIndex = pos;
+    const nameMatch = ATTR_NAME_RE.exec(input);
     if (!nameMatch) return [attrs, -1, false];
     const name = nameMatch[0].toLowerCase();
-    pos += nameMatch[0].length;
+    pos = nameMatch.index + nameMatch[0].length;
     while (pos < input.length && /\s/.test(input[pos])) pos++;
     let value: string | null = null;
     if (pos < input.length && input[pos] === "=") {
@@ -285,64 +303,69 @@ function parseAttrsUntilClose(input: string, pos: number): [AttrDict, number, bo
 }
 
 function scanTag(
-  html: string,
+  input: string,
   i: number,
 ): { end: number; kind: "comment" | "decl" | "end" | "start" | "startend"; name?: string; attrs?: AttrDict } | null {
-  const rest = html.slice(i + 1);
-  if (rest.startsWith("!--")) {
-    const close = html.indexOf("-->", i + 4);
+  if (input.startsWith("!--", i + 1)) {
+    const close = input.indexOf("-->", i + 4);
     if (close === -1) return null;
     return { end: close + 3, kind: "decl" };
   }
-  if (rest.startsWith("!") || rest.startsWith("?")) {
+  const next = input[i + 1];
+  if (next === "!" || next === "?") {
     let j = i + 2;
-    while (j < html.length && html[j] !== ">") j++;
-    if (j >= html.length) return null;
+    while (j < input.length && input[j] !== ">") j++;
+    if (j >= input.length) return null;
     return { end: j + 1, kind: "decl" };
   }
-  if (rest.startsWith("/")) {
+  if (next === "/") {
     let j = i + 2;
-    while (j < html.length && /[\s>]/.test(html[j]) === false) j++;
-    const name = html.slice(i + 2, j).toLowerCase();
+    while (j < input.length && !/[\s>]/.test(input[j])) j++;
+    const name = input.slice(i + 2, j).toLowerCase();
     if (!name) return null;
-    while (j < html.length && html[j] !== ">") j++;
-    if (j >= html.length) return null;
+    while (j < input.length && input[j] !== ">") j++;
+    if (j >= input.length) return null;
     return { end: j + 1, kind: "end", name };
   }
-  const nameMatch = START_TAG_NAME_RE.exec(rest);
+  START_TAG_NAME_RE.lastIndex = i + 1;
+  const nameMatch = START_TAG_NAME_RE.exec(input);
   if (!nameMatch) return null;
   const name = nameMatch[0].toLowerCase();
-  const [attrs, next, selfClosing] = parseAttrsUntilClose(html, i + 1 + nameMatch[0].length);
-  if (next === -1) return null;
-  return { end: next, kind: selfClosing ? "startend" : "start", name, attrs };
+  const [attrs, nextPos, selfClosing] = parseAttrsUntilClose(input, i + 1 + nameMatch[0].length);
+  if (nextPos === -1) return null;
+  return { end: nextPos, kind: selfClosing ? "startend" : "start", name, attrs };
 }
 
 
 export function feedHtml(input: string, handlers: HtmlHandlers): void {
-  const emitText = (text: string) => {
-    if (!text) return;
-    let pos = 0;
-    while (pos < text.length) {
-      const amp = text.indexOf("&", pos);
-      if (amp === -1) {
-        handlers.handleData(text.slice(pos));
+  let lowerInput: string | null = null;
+  const emitTextRange = (start: number, end: number) => {
+    if (end <= start) return;
+    let pos = start;
+    while (pos < end) {
+      const amp = input.indexOf("&", pos);
+      if (amp === -1 || amp >= end) {
+        handlers.handleData(input.slice(pos, end));
         return;
       }
-      if (amp > pos) handlers.handleData(text.slice(pos, amp));
-      const named = /^&([A-Za-z][A-Za-z0-9.-]*);/.exec(text.slice(amp));
-      if (named) {
+      if (amp > pos) handlers.handleData(input.slice(pos, amp));
+      ENTITY_NAMED_RE.lastIndex = amp;
+      const named = ENTITY_NAMED_RE.exec(input);
+      if (named && named.index === amp && amp + named[0].length <= end) {
         handlers.handleEntityRef(named[1]);
         pos = amp + named[0].length;
         continue;
       }
-      const numeric = /^&#([xX][0-9a-fA-F]+|[0-9]+);?/.exec(text.slice(amp));
-      if (numeric) {
+      ENTITY_NUMERIC_RE.lastIndex = amp;
+      const numeric = ENTITY_NUMERIC_RE.exec(input);
+      if (numeric && numeric.index === amp && amp + numeric[0].length <= end) {
         handlers.handleCharRef(numeric[1]);
         pos = amp + numeric[0].length;
         continue;
       }
-      const legacy = /^&([A-Za-z][A-Za-z0-9.-]*)(?=[^A-Za-z0-9]|$)/.exec(text.slice(amp));
-      if (legacy) {
+      ENTITY_LEGACY_RE.lastIndex = amp;
+      const legacy = ENTITY_LEGACY_RE.exec(input);
+      if (legacy && legacy.index === amp && amp + legacy[0].length <= end) {
         handlers.handleEntityRef(legacy[1]);
         pos = amp + legacy[0].length;
         continue;
@@ -364,28 +387,28 @@ export function feedHtml(input: string, handlers: HtmlHandlers): void {
       i++;
       continue;
     }
-    emitText(input.slice(textStart, i));
+    emitTextRange(textStart, i);
     if (tag.kind === "start") {
-      const rawCloser = RAW_TEXT_CLOSERS[tag.name!];
-      if (rawCloser) {
-        const after = input.slice(tag.end);
-        const closeMatch = rawCloser.exec(after);
-        if (closeMatch) {
-          handlers.handleStartTag(tag.name!, tag.attrs!);
-          emitText(after.slice(0, closeMatch.index));
-          handlers.handleEndTag(tag.name!);
-          textStart = tag.end + closeMatch.index + closeMatch[0].length;
+      const rawName = tag.name!;
+      if (RAW_TEXT_TAGS.includes(rawName)) {
+        lowerInput ??= input.toLowerCase();
+        const close = findRawTextClose(input, lowerInput, tag.end, rawName);
+        if (close) {
+          handlers.handleStartTag(rawName, tag.attrs!);
+          emitTextRange(tag.end, close.start);
+          handlers.handleEndTag(rawName);
+          textStart = close.end;
           i = textStart;
           continue;
         }
       }
-      handlers.handleStartTag(tag.name!, tag.attrs!);
+      handlers.handleStartTag(rawName, tag.attrs!);
     } else if (tag.kind === "startend") handlers.handleStartEndTag(tag.name!, tag.attrs!);
     else if (tag.kind === "end") handlers.handleEndTag(tag.name!);
     textStart = tag.end;
     i = tag.end;
   }
-  emitText(input.slice(textStart));
+  emitTextRange(textStart, input.length);
 }
 
 function popMarksAbove(marks: number[], index: number): void {
@@ -1063,8 +1086,13 @@ function visibleLineChars(line: string): number {
   let openBracket = false;
   while (i < n) {
     if (line[i] === "\\") {
-      total += 2;
-      i += 2;
+      if (i + 1 < n) {
+        total += 2;
+        i += 2;
+      } else {
+        total += 1;
+        i += 1;
+      }
       continue;
     }
     if (line[i] === "[") openBracket = true;
@@ -1074,7 +1102,11 @@ function visibleLineChars(line: string): number {
       while (j < n && depth) {
         const char = line[j];
         if (char === "\\") {
-          j += 2;
+          if (j + 1 < n) {
+            j += 2;
+          } else {
+            j += 1;
+          }
           continue;
         }
         depth += (char === "(" ? 1 : 0) - (char === ")" ? 1 : 0);