pi-unsloth-webtools 0.7.6 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-unsloth-webtools",
-  "version": "0.7.6",
+  "version": "0.9.0",
   "type": "module",
   "description": "Pi extension: web_search and web_fetch tools that began as a port of the Unsloth Studio codebase and now diverge from it (multi-engine search, opt-in SSRF guard, HTML-to-Markdown extraction)",
   "main": "index.ts",
@@ -23,11 +23,10 @@
   "license": "AGPL-3.0",
   "files": [
     "index.ts",
-    "config.ts",
-    "config-ui.ts",
     "web-search.ts",
     "web-fetch.ts",
-    "web-render.ts",
+    "tls-fetch.ts",
+    "lightpanda.ts",
     "web-access.ts",
     "html-to-md.ts",
     "engines.ts",
@@ -39,7 +38,8 @@
     "settings.ts",
     "agent-dir.ts",
     "README.md",
-    "LICENSE"
+    "LICENSE",
+    "scripts/install-lightpanda.sh"
   ],
   "pi": {
     "extensions": [
@@ -58,6 +58,11 @@
     "test": "vitest run",
     "test:unit": "vitest run --exclude **/smoke.test.ts",
     "test:smoke": "vitest run test/smoke.test.ts",
+    "test:lightpanda": "vitest run test/lightpanda-live.test.ts",
+    "compare:fetch": "node scripts/fetch-compare.ts",
+    "compare:browsers": "node scripts/compare-browsers.ts",
+    "camoufox:warmup": "node scripts/camoufox-warmup.ts",
+    "stealth:matrix": "node scripts/stealth-matrix.ts",
     "typecheck": "tsc --noEmit",
     "check:package": "node -e \"const fs=require('fs');const pkg=require('./package.json');const listed=new Set(pkg.files);const bad=[];for(const f of pkg.files){if(!fs.existsSync(f))bad.push('missing file: '+f)}for(const f of fs.readdirSync('.').filter(f=>f.endsWith('.ts'))){if(!listed.has(f))bad.push('unlisted source: '+f)}if(bad.length){console.error(bad.join('\\n'));process.exit(1)}\"",
     "prepublishOnly": "npm run typecheck && npm run lint && npm run test:unit && npm run check:package && npm run check:publint",
@@ -65,6 +70,9 @@
     "check:publint": "publint",
     "check:attw": "attw --pack"
   },
+  "dependencies": {
+    "wreq-js": "^3.2.0"
+  },
   "devDependencies": {
     "@earendil-works/pi-coding-agent": "^0.87.0",
     "@earendil-works/pi-tui": "^0.87.0",

package/proxy.ts

@@ -3,6 +3,7 @@ import https from "node:https";
 import net from "node:net";
 import tls from "node:tls";
 import type { Duplex } from "node:stream";
+import { stripIpv6Brackets } from "./web-access.ts";
 
 export interface SocksProxy {
   host: string;
@@ -124,7 +125,7 @@ export function socksProxyForUrl(url: URL): SocksProxy | null {
   const proxy = parseSocksProxy(firstEnv(url.protocol === "https:" ? HTTPS_PROXY_VARS : HTTP_PROXY_VARS));
   if (!proxy) return null;
   const port = url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80;
-  if (bypassesProxy(url.hostname, port)) return null;
+  if (bypassesProxy(stripIpv6Brackets(url.hostname), port)) return null;
   return proxy;
 }
 

package/scripts/install-lightpanda.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="lightpanda-io/browser"
+DEFAULT_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/pi-unsloth-webtools/lightpanda"
+PINNED_FALLBACK_VERSION="1.0.0"
+
+DIR="$DEFAULT_DIR"
+VERSION=""
+FORCE=0
+ALLOW_SHIM=1
+PRINT_PATH=0
+
+usage() {
+  cat <<'EOF'
+Install the Lightpanda browser used by this package's local rendering tier.
+
+Usage: bash scripts/install-lightpanda.sh [options]
+
+  --version=X    Release tag to install (default: the newest stable release, e.g. 1.0.0;
+                 pass --version=nightly for the nightly build)
+  --dir=PATH     Install directory (default: $XDG_DATA_HOME/pi-unsloth-webtools/lightpanda)
+  --force        Reinstall even when a working binary is already present
+  --no-shim      Fail instead of bundling a newer libc when the system glibc is too old
+  --print-path   Print only the launcher path (for scripts and CI)
+  -h, --help     Show this message
+
+On Linux, if the release needs a newer glibc than the system provides, the script
+downloads Debian's libc6 for the current stable suite, extracts it into the install
+directory, and writes a launcher shim that runs the binary with that libc. Point
+`webRender.lightpandaPath` (settings.json) or `PI_LIGHTPANDA_BIN` at the printed path.
+EOF
+}
+
+for arg in "$@"; do
+  case "$arg" in
+    --version=*) VERSION="${arg#*=}" ;;
+    --dir=*) DIR="${arg#*=}" ;;
+    --force) FORCE=1 ;;
+    --no-shim) ALLOW_SHIM=0 ;;
+    --print-path) PRINT_PATH=1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown option: $arg" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+need() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "missing required command: $1" >&2
+    exit 1
+  fi
+}
+
+need curl
+need awk
+need sed
+
+os="$(uname -s)"
+arch="$(uname -m)"
+deb_arch=""
+case "$os/$arch" in
+  Linux/x86_64) asset="lightpanda-x86_64-linux"; deb_arch="amd64" ;;
+  Linux/aarch64|Linux/arm64) asset="lightpanda-aarch64-linux"; deb_arch="arm64" ;;
+  Darwin/x86_64) asset="lightpanda-x86_64-macos" ;;
+  Darwin/arm64) asset="lightpanda-aarch64-macos" ;;
+  MINGW*|MSYS*|CYGWIN*)
+    cat >&2 <<'EOF'
+Lightpanda ships no native Windows build. Run it inside WSL2 instead:
+
+  1. in WSL:   bash scripts/install-lightpanda.sh
+               (it prints a path such as /home/<you>/.local/share/pi-unsloth-webtools/lightpanda/lightpanda-run)
+
+  2. in Windows settings.json, drive that binary through wsl.exe:
+
+     {
+       "webRender": {
+         "lightpandaCommand": ["wsl.exe", "-e", "/home/<you>/.local/share/pi-unsloth-webtools/lightpanda/lightpanda-run"]
+       }
+     }
+
+Everything else in this package (search, the browser-fingerprint transport) runs natively on Windows.
+EOF
+    exit 1 ;;
+  *) echo "unsupported platform: $os/$arch" >&2; exit 1 ;;
+esac
+
+if [ -z "$VERSION" ]; then
+  VERSION="$(curl -fsSL "https://api.github.com/repos/$REPO/releases?per_page=30" 2>/dev/null \
+    | grep -o '"tag_name":[[:space:]]*"[^"]*"' \
+    | sed -n 's/.*"\([^"]*\)"$/\1/p' \
+    | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \
+    | head -1 || true)"
+  if [ -z "$VERSION" ]; then
+    echo "could not resolve the newest release; falling back to $PINNED_FALLBACK_VERSION" >&2
+    VERSION="$PINNED_FALLBACK_VERSION"
+  fi
+fi
+
+mkdir -p "$DIR"
+binary="$DIR/lightpanda"
+shim="$DIR/lightpanda-run"
+
+report_path() {
+  if [ "$PRINT_PATH" = 1 ]; then
+    printf '%s\n' "$1"
+  else
+    echo "already installed and working: $1" >&2
+  fi
+  exit 0
+}
+
+if [ "$FORCE" = 0 ] && [ -x "$shim" ] && "$shim" version >/dev/null 2>&1; then
+  report_path "$shim"
+fi
+if [ "$FORCE" = 0 ] && [ -x "$binary" ] && "$binary" version >/dev/null 2>&1; then
+  report_path "$binary"
+fi
+
+url="https://github.com/$REPO/releases/download/$VERSION/$asset"
+if [ "$PRINT_PATH" = 1 ]; then
+  echo "downloading $VERSION ($asset)" >&2
+  curl -fL --silent --show-error "$url" -o "$binary.part"
+else
+  echo "downloading $VERSION ($asset)"
+  curl -fL --progress-bar "$url" -o "$binary.part"
+fi
+chmod +x "$binary.part"
+mv -f "$binary.part" "$binary"
+rm -f "$shim"
+
+run_path="$binary"
+if ! "$binary" version >/dev/null 2>&1; then
+  run_path=""
+  if [ "$ALLOW_SHIM" = 0 ] || [ "$os" != "Linux" ]; then
+    echo "the $VERSION binary cannot run on this system:" >&2
+    "$binary" version 2>&1 | head -3 >&2
+    exit 1
+  fi
+  required="$({ "$binary" version 2>&1 || true; } | sed -n "s/.*GLIBC_\([0-9][0-9.]*\).*/\1/p" | sort -V | tail -1)"
+  if [ -z "$required" ]; then
+    echo "the $VERSION binary failed to start and the cause is not a glibc version:" >&2
+    "$binary" version 2>&1 | head -3 >&2
+    exit 1
+  fi
+  system_glibc="$(ldd --version 2>/dev/null | head -1 | sed -n 's/.* \([0-9][0-9.]*\)$/\1/p')"
+  echo "binary needs glibc $required, system has ${system_glibc:-unknown}: bundling a newer libc"
+  need dpkg-deb
+  glibc_dir="$DIR/glibc"
+  rm -rf "$glibc_dir"
+  for suite in trixie stable sid; do
+    index="https://deb.debian.org/debian/dists/$suite/main/binary-$deb_arch/Packages.gz"
+    deb_path="$(curl -fsSL "$index" 2>/dev/null | gzip -dc 2>/dev/null \
+      | awk '/^Package: libc6$/{found=1} found && /^Filename:/{print $2; exit}' || true)"
+    [ -n "$deb_path" ] || continue
+    echo "fetching libc6 from debian/$suite"
+    curl -fsSL "https://deb.debian.org/debian/$deb_path" -o "$DIR/libc6.deb" || continue
+    mkdir -p "$glibc_dir"
+    dpkg-deb -x "$DIR/libc6.deb" "$glibc_dir"
+    rm -f "$DIR/libc6.deb"
+    loader="$(find "$glibc_dir" -path "*-linux-gnu/ld-linux-*.so.1" | head -1)"
+    [ -n "$loader" ] || continue
+    bundled_glibc="$("$loader" --version 2>/dev/null | head -1 | sed -n 's/.* \([0-9][0-9.]*\)$/\1/p')"
+    if [ -z "$bundled_glibc" ]; then
+      rm -rf "$glibc_dir"
+      continue
+    fi
+    newest="$(printf '%s\n%s\n' "$required" "$bundled_glibc" | sort -V | tail -1)"
+    if [ "$newest" != "$bundled_glibc" ]; then
+      echo "bundled glibc $bundled_glibc is older than required $required, trying the next suite" >&2
+      rm -rf "$glibc_dir"
+      continue
+    fi
+    libdir="$(dirname "$loader")"
+    {
+      echo "#!/bin/sh"
+      echo "exec \"$loader\" --library-path \"$libdir\" \"$binary\" \"\$@\""
+    } > "$shim"
+    chmod +x "$shim"
+    if "$shim" version >/dev/null 2>&1; then
+      run_path="$shim"
+      echo "bundled glibc $bundled_glibc at $glibc_dir"
+      break
+    fi
+    rm -rf "$glibc_dir" "$shim"
+  done
+  if [ -z "$run_path" ]; then
+    echo "could not make $VERSION runnable on this system." >&2
+    echo "Options: upgrade the distribution, run Lightpanda in a container, or install an older" >&2
+    echo "release that targets this glibc (for example 0.2.9)." >&2
+    exit 1
+  fi
+fi
+
+installed_version="$("$run_path" version 2>/dev/null | head -1)"
+if [ "$PRINT_PATH" = 1 ]; then
+  printf '%s\n' "$run_path"
+  exit 0
+fi
+
+echo
+echo "Lightpanda $installed_version installed"
+echo "  run this:          $run_path"
+echo "  configure one of:"
+echo "    export PI_LIGHTPANDA_BIN=$run_path"
+echo "    settings.json:   { \"webRender\": { \"lightpandaPath\": \"$run_path\" } }"
+echo "  verify:            $run_path fetch --dump markdown --wait-until networkidle https://example.com/"

package/settings.ts

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { agentDir } from "./agent-dir.ts";
+import type { FetchTransport } from "./web-fetch.ts";
 
 async function readJson(file: string): Promise<Record<string, unknown> | undefined> {
   try {
@@ -49,6 +50,23 @@ function pickBoolean(data: Record<string, unknown>, paths: string[][]): boolean
   return undefined;
 }
 
+function pickStringArray(data: Record<string, unknown>, paths: string[][]): string[] | undefined {
+  for (const path of paths) {
+    let cur: unknown = data;
+    for (const key of path) {
+      if (cur && typeof cur === "object" && !Array.isArray(cur)) cur = (cur as Record<string, unknown>)[key];
+      else {
+        cur = undefined;
+        break;
+      }
+    }
+    if (!Array.isArray(cur)) continue;
+    const values = cur.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0);
+    if (values.length) return values;
+  }
+  return undefined;
+}
+
 function pickString(data: Record<string, unknown>, paths: string[][]): string | undefined {
   for (const path of paths) {
     let cur: unknown = data;
@@ -94,11 +112,27 @@ const ALLOW_LOCAL_FILES_PATHS: string[][] = [
   ["webFetch", "allowLocalFiles"],
 ];
 
-const JINA_API_KEY_PATHS: string[][] = [
-  ["unslothWebTools", "jinaApiKey"],
-  ["webRender", "jinaApiKey"],
+const TRANSPORT_PATHS: string[][] = [
+  ["unslothWebTools", "transport"],
+  ["webFetch", "transport"],
+];
+
+const TRANSPORT_VALUES: ReadonlySet<string> = new Set(["tls-first", "direct-first", "off"]);
+
+const LIGHTPANDA_ENABLED_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaEnabled"],
+  ["webRender", "lightpandaEnabled"],
+];
+
+const LIGHTPANDA_PATH_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaPath"],
+  ["webRender", "lightpandaPath"],
 ];
 
+const LIGHTPANDA_COMMAND_PATHS: string[][] = [
+  ["unslothWebTools", "lightpandaCommand"],
+  ["webRender", "lightpandaCommand"],
+];
 function clampMaxResults(value: number): number {
   return Math.min(20, Math.max(1, value));
 }
@@ -151,11 +185,13 @@ export async function loadDefaultFetchSettings(cwd?: string): Promise<{
   timeoutMs?: number;
   allowPrivateAddresses: boolean;
   allowLocalFiles: boolean;
+  transport: FetchTransport;
 }> {
   let maxChars: number | undefined;
   let timeoutMs: number | undefined;
   let allowPrivateAddresses = true;
   let allowLocalFiles = true;
+  let transport: FetchTransport = "tls-first";
   for (const data of await settingsEntries(cwd)) {
     const c = pickNumber(data, FETCH_MAX_CHARS_PATHS);
     if (c !== undefined && c > 0) maxChars = c;
@@ -165,17 +201,27 @@ export async function loadDefaultFetchSettings(cwd?: string): Promise<{
     if (p !== undefined) allowPrivateAddresses = p;
     const l = pickBoolean(data, ALLOW_LOCAL_FILES_PATHS);
     if (l !== undefined) allowLocalFiles = l;
+    const mode = pickString(data, TRANSPORT_PATHS);
+    if (mode !== undefined && TRANSPORT_VALUES.has(mode)) transport = mode as FetchTransport;
   }
-  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles };
+  return { maxChars, timeoutMs, allowPrivateAddresses, allowLocalFiles, transport };
 }
 
-export async function loadJinaApiKey(cwd?: string): Promise<string | undefined> {
-  let result: string | undefined;
+export async function loadLightpandaSettings(cwd?: string): Promise<{
+  enabled: boolean;
+  binaryPath: string | null;
+  command: string[] | null;
+}> {
+  let enabled = true;
+  let binaryPath: string | null = null;
+  let command: string[] | null = null;
   for (const data of await settingsEntries(cwd)) {
-    const candidate = pickString(data, JINA_API_KEY_PATHS);
-    if (candidate !== undefined) result = candidate;
+    const e = pickBoolean(data, LIGHTPANDA_ENABLED_PATHS);
+    if (e !== undefined) enabled = e;
+    const p = pickString(data, LIGHTPANDA_PATH_PATHS);
+    if (p !== undefined) binaryPath = p;
+    const c = pickStringArray(data, LIGHTPANDA_COMMAND_PATHS);
+    if (c !== undefined) command = c;
   }
-  if (result !== undefined) return result;
-  const env = process.env.JINA_API_KEY?.trim();
-  return env ? env : undefined;
+  return { enabled, binaryPath, command };
 }

package/tls-fetch.ts

@@ -0,0 +1,205 @@
+import { MAX_SIGNAL_TIMEOUT_MS } from "./web-access.ts";
+
+const MODULE_NAME = "wreq-js";
+const DEFAULT_PROFILE = "chrome_145";
+const DEFAULT_OS = "windows";
+const TRANSPORT_CACHE_LIMIT = 8;
+
+const HOP_HEADER_EXCLUSIONS = new Set(["host", "user-agent", "accept-encoding", "content-length"]);
+const PDF_MAGIC = Buffer.from("%PDF-");
+const MAGIC_HEAD_BYTES = 1024;
+const SKIPPED_MAGIC_PREFIX_BYTES = new Set([0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20]);
+const BOM_PREFIXES = [
+  Buffer.from([0xff, 0xfe, 0x00, 0x00]),
+  Buffer.from([0x00, 0x00, 0xfe, 0xff]),
+  Buffer.from([0xff, 0xfe]),
+  Buffer.from([0xfe, 0xff]),
+  Buffer.from([0xef, 0xbb, 0xbf]),
+];
+
+function looksLikePdf(head: Buffer): boolean {
+  let start = 0;
+  while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
+  for (const bom of BOM_PREFIXES) {
+    if (head.subarray(start).length >= bom.length && head.subarray(start, start + bom.length).equals(bom)) {
+      start += bom.length;
+      while (start < head.length && SKIPPED_MAGIC_PREFIX_BYTES.has(head[start])) start++;
+      break;
+    }
+  }
+  const magic = head.subarray(start);
+  return magic.length >= PDF_MAGIC.length && magic.subarray(0, PDF_MAGIC.length).equals(PDF_MAGIC);
+}
+
+interface ImpersonationResponse {
+  status: number;
+  url: string;
+  headers: { get(name: string): string | null };
+  body: AsyncIterable<Uint8Array> | null;
+}
+
+interface ImpersonationTransport {
+  close?(): Promise<void>;
+}
+
+interface ImpersonationModule {
+  createTransport(options: Record<string, unknown>): Promise<ImpersonationTransport>;
+  fetch(input: string, init: Record<string, unknown>): Promise<ImpersonationResponse>;
+}
+
+export type ImpersonationLoader = () => Promise<ImpersonationModule | null>;
+
+export interface TlsHopOptions {
+  url: URL;
+  pinnedIp: string;
+  family: number;
+  timeoutMs: number;
+  signal?: AbortSignal;
+  maxBytes: number;
+  maxPdfBytes?: number;
+  profile?: string;
+  extraHeaders?: Record<string, string>;
+  loader?: ImpersonationLoader;
+}
+
+export interface TlsHopResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: Buffer;
+  truncated: boolean;
+}
+
+let cachedModule: Promise<ImpersonationModule | null> | null = null;
+
+async function loadImpersonationModule(): Promise<ImpersonationModule | null> {
+  cachedModule ??= (async () => {
+    try {
+      return (await import(MODULE_NAME)) as unknown as ImpersonationModule;
+    } catch {
+      return null;
+    }
+  })();
+  return cachedModule;
+}
+
+const transports = new Map<string, Promise<ImpersonationTransport>>();
+
+function transportKey(options: TlsHopOptions): string {
+  return `${options.profile ?? DEFAULT_PROFILE}|${options.url.hostname}|${options.pinnedIp}|${options.family}`;
+}
+
+function evictTransports(): void {
+  while (transports.size > TRANSPORT_CACHE_LIMIT) {
+    const oldestKey = transports.keys().next().value as string | undefined;
+    if (oldestKey === undefined) return;
+    const oldest = transports.get(oldestKey);
+    transports.delete(oldestKey);
+    void oldest
+      ?.then((transport) => transport.close?.())
+      .catch(() => {});
+  }
+}
+
+async function transportFor(
+  module: ImpersonationModule,
+  options: TlsHopOptions,
+): Promise<ImpersonationTransport> {
+  const key = transportKey(options);
+  const existing = transports.get(key);
+  if (existing) return existing;
+  const created = module
+    .createTransport({
+      resolve: { [options.url.hostname]: options.pinnedIp },
+      browser: options.profile ?? DEFAULT_PROFILE,
+      os: DEFAULT_OS,
+    })
+    .catch((error: unknown) => {
+      transports.delete(key);
+      throw error;
+    });
+  transports.set(key, created);
+  evictTransports();
+  return created;
+}
+
+async function readCapped(
+  body: AsyncIterable<Uint8Array> | null,
+  maxBytes: number,
+  maxPdfBytes: number,
+): Promise<{ buffer: Buffer; truncated: boolean }> {
+  if (!body) return { buffer: Buffer.alloc(0), truncated: false };
+  const chunks: Buffer[] = [];
+  let total = 0;
+  let truncated = false;
+  let head = Buffer.alloc(0);
+  let limit = maxBytes;
+  for await (const chunk of body) {
+    const buffer = Buffer.from(chunk);
+    if (head.length < MAGIC_HEAD_BYTES) {
+      head = Buffer.concat([head, buffer.subarray(0, MAGIC_HEAD_BYTES - head.length)]);
+      if (limit === maxBytes && looksLikePdf(head)) limit = maxPdfBytes;
+    }
+    if (total + buffer.length > limit) {
+      const take = buffer.subarray(0, limit - total);
+      chunks.push(take);
+      total += take.length;
+      truncated = true;
+      break;
+    }
+    chunks.push(buffer);
+    total += buffer.length;
+  }
+  return { buffer: Buffer.concat(chunks), truncated };
+}
+
+function failureMessage(error: unknown, caller: AbortSignal | undefined, attempt: AbortSignal): string {
+  if (caller?.aborted) return "cancelled";
+  if (attempt.aborted) return "timed out";
+  if (error instanceof Error && error.name === "TimeoutError") return "timed out";
+  if (error instanceof Error && error.name === "AbortError") return "cancelled";
+  return error instanceof Error ? error.message : String(error);
+}
+
+function hopHeaders(extraHeaders: Record<string, string> | undefined): Record<string, string> {
+  const headers: Record<string, string> = {};
+  for (const [key, value] of Object.entries(extraHeaders ?? {})) {
+    if (HOP_HEADER_EXCLUSIONS.has(key.toLowerCase())) continue;
+    headers[key] = value;
+  }
+  return headers;
+}
+
+export async function impersonatedRequest(options: TlsHopOptions): Promise<TlsHopResponse | null> {
+  const load = options.loader ?? loadImpersonationModule;
+  const module = await load();
+  if (!module) return null;
+  const transport = await transportFor(module, options);
+  const timeoutMs = Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, Math.floor(options.timeoutMs)));
+  const timeoutSignal = AbortSignal.timeout(timeoutMs);
+  const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
+  let response: ImpersonationResponse;
+  try {
+    response = await module.fetch(options.url.toString(), {
+      transport,
+      redirect: "manual",
+      signal,
+      headers: hopHeaders(options.extraHeaders),
+    });
+  } catch (error) {
+    throw new Error(failureMessage(error, options.signal, signal));
+  }
+  if (response.status < 200) return null;
+  const { buffer, truncated } = await readCapped(
+    response.body,
+    options.maxBytes,
+    options.maxPdfBytes ?? options.maxBytes,
+  );
+  const headers: Record<string, string> = {};
+  const contentType = response.headers.get("content-type");
+  if (contentType) headers["content-type"] = contentType;
+  if (response.status >= 300 && response.status < 400) {
+    const location = response.headers.get("location");
+    if (location) headers.location = location;
+  }
+  return { status: response.status, headers, body: buffer, truncated };
+}

package/web-access.ts

@@ -70,6 +70,10 @@ export function normalizeDomain(value: unknown): string {
   return asciiDomain;
 }
 
+export function stripIpv6Brackets(hostname: string): string {
+  return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+}
+
 function compressIpv6(ip: string): string {
   const segments = ip.toLowerCase().split("::");
   if (segments.length > 2) throw new Error("invalid ipv6");