pi-unsloth-webtools 0.3.0 → 0.4.0

package/web-fetch.ts

@@ -1,20 +1,25 @@
 import { lookup as dnsLookup } from "node:dns/promises";
+import type { LookupAllOptions } from "node:dns";
 import http from "node:http";
 import https from "node:https";
+
 import { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from "node:zlib";
 import type { IncomingMessage } from "node:http";
 import type { Transform } from "node:stream";
 import {
   checkUrlAccess,
+  githubRawContentUrl,
   githubRepoRawReadmeUrl,
   githubRepoReadmeApiUrl,
   isPublicIp,
+  MAX_SIGNAL_TIMEOUT_MS,
   normalizeUrlScheme,
   type WebsitePolicy,
 } from "./web-access.ts";
 import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown } from "./html-to-md.ts";
 import type { AttrDict } from "./html-to-md.ts";
 import { INVALID_CHARREFS } from "./entities.ts";
+import { getCached, isFresh, setCached, staleNotice } from "./cache.ts";
 import { extractPdfText } from "./pdf.ts";
 import { randomUserAgent } from "./user-agents.ts";
 
@@ -23,8 +28,23 @@ const MAX_DECOMPRESSED_BYTES = 64 * 1024 * 1024;
 const META_VALUE_MAX_CHARS = 300;
 const MAX_PDF_FETCH_BYTES = 10 * 1024 * 1024;
 const MAX_REQUESTS = 5;
-const MAX_SIGNAL_TIMEOUT_MS = 2 ** 31 - 1;
 export const DEFAULT_FETCH_TIMEOUT_MS = 60_000;
+export { loadDefaultFetchMaxChars, loadDefaultFetchTimeoutMs, loadDefaultFetchSettings } from "./settings.ts";
+function remainingMs(deadline: number, now: () => number): number {
+  return Math.max(1, deadline - now());
+}
+
+function clampedRemainingMs(deadline: number, now: () => number): number {
+  return Math.min(MAX_SIGNAL_TIMEOUT_MS, remainingMs(deadline, now));
+}
+
+function withTruncation(text: string, truncated: boolean): string {
+  return truncated ? text + TRUNCATED_BODY_NOTICE : text;
+}
+
+function preserveTruncationNotice(original: string, converted: string): string {
+  return original.endsWith(TRUNCATED_BODY_SUFFIX) && !converted.endsWith(TRUNCATED_BODY_SUFFIX) ? converted + TRUNCATED_BODY_NOTICE : converted;
+}
 
 const UTF32_LE_BOM = Buffer.from([0xff, 0xfe, 0x00, 0x00]);
 const UTF32_BE_BOM = Buffer.from([0x00, 0x00, 0xfe, 0xff]);
@@ -119,8 +139,8 @@ export class FetchTimeoutError extends Error {
 }
 const FETCH_CANCELLED_MESSAGE = "Failed to fetch URL: cancelled.";
 const FETCH_TIMEOUT_MESSAGE = "Failed to fetch URL: timed out.";
-const TRUNCATED_BODY_NOTICE = "\n\n... (page truncated at the download limit)";
 const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
+const TRUNCATED_BODY_NOTICE = "\n\n" + TRUNCATED_BODY_SUFFIX;
 
 function fetchErrorMessage(err: unknown): string {
   if (err instanceof FetchCancelledError) return FETCH_CANCELLED_MESSAGE;
@@ -131,13 +151,13 @@ function fetchErrorMessage(err: unknown): string {
   return `Failed to fetch URL: ${message}`;
 }
 
+function emptyResult(error: string, contentType = ""): RawFetchResult {
+  return { error, body: "", contentType };
+}
+
 function statusErrorResult(status: number): RawFetchResult {
   const reason = http.STATUS_CODES[status] ?? "";
-  return {
-    error: `Failed to fetch URL: HTTP ${status}${reason ? ` ${reason}` : ""}`,
-    body: "",
-    contentType: "",
-  };
+  return emptyResult(`Failed to fetch URL: HTTP ${status}${reason ? ` ${reason}` : ""}`);
 }
 
 export interface FetchPageOptions {
@@ -165,6 +185,7 @@ export interface ResolvedHost {
   reason: string;
   ip: string;
   family: number;
+  alternates?: { ip: string; family: number }[];
 }
 
 export interface FetchSeams {
@@ -204,14 +225,25 @@ export interface RawFetchResult {
 }
 
 function htmlProbe(body: string, re: RegExp): boolean {
-  let probe = body;
-  while (true) {
-    probe = probe.replace(/^[ \t\n\r\f\v]+/, "");
-    const stripped = probe.replace(/^(?:<!--[\s\S]*?-->|<\?[\s\S]*?\?>)/, "");
-    if (stripped === probe) break;
-    probe = stripped;
+  let i = 0;
+  const n = body.length;
+  while (i < n) {
+    while (i < n && /[ \t\n\r\f\v]/.test(body[i])) i++;
+    if (body.startsWith("<!--", i)) {
+      const close = body.indexOf("-->", i + 4);
+      if (close === -1) break;
+      i = close + 3;
+      continue;
+    }
+    if (body.startsWith("<?", i)) {
+      const close = body.indexOf("?>", i + 2);
+      if (close === -1) break;
+      i = close + 2;
+      continue;
+    }
+    break;
   }
-  return re.test(probe.slice(0, 256).toLowerCase());
+  return re.test(body.slice(i, i + 256).toLowerCase());
 }
 
 export function looksLikeHtml(body: string): boolean {
@@ -361,6 +393,41 @@ function sniffMetaCharsetForHtml(bytes: Buffer, contentType: string): string | n
   }
   return sniffMetaCharset(bytes);
 }
+function extractMetaRefresh(html: string): string | null {
+  let refresh: string | null = null;
+  const handle = (name: string, attrs: AttrDict) => {
+    if (refresh !== null) return;
+    if (name !== "meta") return;
+    const equiv = (attrs["http-equiv"] ?? "").toLowerCase();
+    if (equiv !== "refresh") return;
+    const content = attrs["content"] ?? "";
+    const match = /;\s*url\s*=\s*(.+)/i.exec(content);
+    if (!match) return;
+    let url = match[1].trim();
+    if ((url.startsWith('"') && url.endsWith('"')) || (url.startsWith("'") && url.endsWith("'"))) url = url.slice(1, -1);
+    url = url.trim();
+    if (url) refresh = url;
+  };
+  feedHtml(html, {
+    handleStartTag: handle,
+    handleStartEndTag: handle,
+    handleEndTag() {},
+    handleData() {},
+    handleEntityRef() {},
+    handleCharRef() {},
+  });
+  return refresh;
+}
+function metaRefreshUrl(html: string, base: string): string | null {
+  const target = extractMetaRefresh(html);
+  if (!target) return null;
+  try {
+    const next = new URL(target, base).toString();
+    return next !== base ? next : null;
+  } catch {
+    return null;
+  }
+}
 
 function decodeUtf32(bytes: Buffer, littleEndian: boolean): string {
   let out = "";
@@ -455,32 +522,15 @@ function decodeTis620(bytes: Buffer): string {
 }
 
 
-function withAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
-  if (!signal) return promise;
-  return new Promise<T>((resolve, reject) => {
-    const onAbort = () => reject(new DOMException("aborted", "AbortError"));
-    if (signal.aborted) {
-      onAbort();
-      return;
-    }
-    signal.addEventListener("abort", onAbort, { once: true });
-    promise.then(
-      (value) => {
-        signal.removeEventListener("abort", onAbort);
-        resolve(value);
-      },
-      (err) => {
-        signal.removeEventListener("abort", onAbort);
-        reject(err);
-      },
-    );
-  });
-}
-
 async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promise<ResolvedHost> {
   let addresses: { address: string; family: number }[];
   try {
-    addresses = await withAbort(dnsLookup(hostname, { all: true, verbatim: true }), signal);
+    const lookupOptions: LookupAllOptions & { signal?: AbortSignal } = {
+      all: true,
+      verbatim: true,
+      signal,
+    };
+    addresses = await dnsLookup(hostname, lookupOptions);
   } catch (err) {
     return { ok: false, reason: `Failed to resolve host: ${err}`, ip: "", family: 0 };
   }
@@ -492,19 +542,24 @@ async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promi
       return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
     }
   }
+  addresses.sort((a, b) => (a.family === 4 ? 0 : 1) - (b.family === 4 ? 0 : 1));
   const first = addresses[0];
-  return { ok: true, reason: "", ip: first.address, family: first.family };
+  return {
+    ok: true,
+    reason: "",
+    ip: first.address,
+    family: first.family,
+    alternates: addresses.slice(1).map((entry) => ({ ip: entry.address, family: entry.family })),
+  };
 }
-
-
 function budgetExceededResult(
   deadline: number | null,
   signal: AbortSignal | undefined,
   now: () => number = Date.now,
   contentType = "",
 ): RawFetchResult | null {
-  if (signal?.aborted) return { error: FETCH_CANCELLED_MESSAGE, body: "", contentType };
-  if (deadline !== null && now() >= deadline) return { error: FETCH_TIMEOUT_MESSAGE, body: "", contentType };
+  if (signal?.aborted) return emptyResult(FETCH_CANCELLED_MESSAGE, contentType);
+  if (deadline !== null && now() >= deadline) return emptyResult(FETCH_TIMEOUT_MESSAGE, contentType);
   return null;
 }
 
@@ -539,6 +594,8 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
         callback(null, [{ address: opts.pinnedIp, family: opts.family }]),
     };
     let settled = false;
+    let resRef: IncomingMessage | null = null;
+    let decoderRef: Transform | null = null;
     const settle = (action: () => void) => {
       if (settled) return;
       settled = true;
@@ -546,6 +603,7 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
       action();
     };
     const request = transport.request(options, (res: IncomingMessage) => {
+      resRef = res;
       const codec = contentEncodingCodec(res.headers["content-encoding"]);
       const chunks: Buffer[] = [];
       let total = 0;
@@ -603,7 +661,8 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
         if (total >= limit) {
           truncated =
             codec !== null ||
-            !(Number.isFinite(declaredLength) && declaredLength === total);
+            take.length < chunk.length ||
+            (Number.isFinite(declaredLength) && declaredLength > total);
           decoder?.destroy();
           res.destroy();
           finish(null, Buffer.concat(chunks));
@@ -621,6 +680,8 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
         let rawFallbackTried = false;
         let resEnded = false;
         const wire = (d: Transform) => {
+          decoderRef = d;
+          decoder = d;
           d.on("data", (chunk: Buffer) => {
             outputStarted = true;
             acceptData(chunk);
@@ -638,6 +699,7 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
               rawFallbackTried = true;
               d.destroy();
               decoder = createInflateRaw({ maxOutputLength: MAX_DECOMPRESSED_BYTES });
+              decoderRef = decoder;
               wire(decoder);
               for (const buffered of rawBuffer) decoder.write(buffered);
               if (resEnded) decoder.end();
@@ -652,6 +714,7 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
           });
         };
         decoder = createDecodeStream(codec);
+        decoderRef = decoder;
         wire(decoder);
         res.on("data", (chunk: Buffer) => {
           if (settled) return;
@@ -676,7 +739,11 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
         finish(err.message, Buffer.concat(chunks));
       });
     });
-    const onAbort = () => request.destroy(new FetchCancelledError());
+    const onAbort = () => {
+      decoderRef?.destroy();
+      resRef?.destroy();
+      request.destroy(new FetchCancelledError());
+    };
     opts.signal?.addEventListener("abort", onAbort, { once: true });
     request.on("timeout", () => request.destroy(new FetchTimeoutError()));
     request.on("error", (err) => settle(() => reject(err)));
@@ -701,31 +768,36 @@ export async function fetchUrlRaw(
   const resolveHost = seams.resolve ?? resolveAndValidate;
   const performRequest = seams.request ?? requestHop;
   const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
-    const deadlineSignal = AbortSignal.timeout(Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, deadline - now())));
-    const resolveSignal = signal ? AbortSignal.any([signal, deadlineSignal]) : deadlineSignal;
-    const resolved = await resolveHost(hostname, resolveSignal);
-    if (resolved.ok) return resolved;
-    if (signal?.aborted) return { ...resolved, reason: FETCH_CANCELLED_MESSAGE };
-    if (resolveSignal.aborted) return { ...resolved, reason: FETCH_TIMEOUT_MESSAGE };
-    return resolved;
+    const abortController = new AbortController();
+    const timeoutId = setTimeout(() => abortController.abort(), clampedRemainingMs(deadline, now));
+    const resolveSignal = signal ? AbortSignal.any([signal, abortController.signal]) : abortController.signal;
+    try {
+      const resolved = await resolveHost(hostname, resolveSignal);
+      if (resolved.ok) return resolved;
+      if (signal?.aborted) return { ...resolved, reason: FETCH_CANCELLED_MESSAGE };
+      if (resolveSignal.aborted) return { ...resolved, reason: FETCH_TIMEOUT_MESSAGE };
+      return resolved;
+    } finally {
+      clearTimeout(timeoutId);
+    }
   };
-
+  const checkBudget = (contentType = ""): RawFetchResult | null => budgetExceededResult(deadline, signal, now, contentType);
   url = normalizeUrlScheme(url);
   const [allowed, reason, hostname] = checkUrlAccess(url, policy);
-  if (!allowed) return { error: reason, body: "", contentType: "" };
-
-  const budgetResult = budgetExceededResult(deadline, signal, now);
+  if (!allowed) return emptyResult(reason);
+  const budgetResult = checkBudget();
   if (budgetResult !== null) return budgetResult;
   let resolved = await resolveWithBudget(hostname);
-  if (!resolved.ok) return { error: resolved.reason, body: "", contentType: "" };
+  if (!resolved.ok) return emptyResult(resolved.reason);
 
   let currentUrl = url;
   let pinnedIp = resolved.ip;
   let pinnedFamily = resolved.family;
+  let alternates: { ip: string; family: number }[] = resolved.alternates ?? [];
+  let alternateIndex = 0;
   const userAgent = randomUserAgent();
-
   for (let hop = 0; hop < MAX_REQUESTS; hop++) {
-    const budgetResult = budgetExceededResult(deadline, signal, now);
+    const budgetResult = checkBudget();
     if (budgetResult !== null) return budgetResult;
     const parsed = new URL(currentUrl);
     const hostHeader = parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
@@ -736,7 +808,7 @@ export async function fetchUrlRaw(
       "Accept-Encoding": "identity",
     };
     if (options.extraHeaders) Object.assign(headers, options.extraHeaders);
-    const inactivity = Math.max(1, deadline - now());
+    const inactivity = remainingMs(deadline, now);
     let response: HopResponse;
     try {
       response = await performRequest({
@@ -752,7 +824,17 @@ export async function fetchUrlRaw(
         signal,
       });
     } catch (err) {
-      return { error: fetchErrorMessage(err), body: "", contentType: "" };
+      if (
+        !(err instanceof FetchCancelledError) &&
+        !(err instanceof FetchTimeoutError) &&
+        alternateIndex < alternates.length
+      ) {
+        const next = alternates[alternateIndex++];
+        pinnedIp = next.ip;
+        pinnedFamily = next.family;
+        continue;
+      }
+      return emptyResult(fetchErrorMessage(err));
     }
 
     if (response.status >= 300 && response.status < 400) {
@@ -762,30 +844,28 @@ export async function fetchUrlRaw(
       const rawLocation = response.headers.location;
       const location = Array.isArray(rawLocation) ? rawLocation[0] : rawLocation;
       if (!location) {
-        return {
-          error: "Failed to fetch URL: the redirect is missing a Location header.",
-          body: "",
-          contentType: "",
-        };
+        return emptyResult("Failed to fetch URL: the redirect is missing a Location header.");
       }
       try {
         currentUrl = new URL(location, currentUrl).toString();
       } catch {
-        return { error: "Failed to fetch URL: the redirect has an invalid Location.", body: "", contentType: "" };
+        return emptyResult("Failed to fetch URL: the redirect has an invalid Location.");
       }
       const [redirectAllowed, redirectReason, redirectHost] = checkUrlAccess(
         currentUrl,
         policy,
       );
-      if (!redirectAllowed) return { error: redirectReason, body: "", contentType: "" };
+      if (!redirectAllowed) return emptyResult(redirectReason);
       const redirected = await resolveWithBudget(redirectHost);
-      if (!redirected.ok) return { error: redirected.reason, body: "", contentType: "" };
+      if (!redirected.ok) return emptyResult(redirected.reason);
       pinnedIp = redirected.ip;
       pinnedFamily = redirected.family;
+      alternates = redirected.alternates ?? [];
+      alternateIndex = 0;
       continue;
     }
 
-    const postBudgetResult = budgetExceededResult(deadline, signal, now);
+    const postBudgetResult = checkBudget();
     if (postBudgetResult !== null) return postBudgetResult;
 
     if (response.status >= 400) {
@@ -812,46 +892,59 @@ export async function fetchUrlRaw(
       try {
         pdfText = await extractPdfText(response.body);
       } catch {
-        return {
-          error: response.truncated
+        return emptyResult(
+          response.truncated
             ? "(PDF content could not be read as text; the download was truncated at the download limit)"
             : "(PDF content could not be read as text)",
-          body: "",
           contentType,
-        };
+        );
       }
-      const budgetResult = budgetExceededResult(deadline, signal, now, contentType);
+      const budgetResult = checkBudget(contentType);
       if (budgetResult !== null) return budgetResult;
       if (!pdfText) pdfText = "(PDF contains no extractable text)";
-      if (response.truncated) pdfText += TRUNCATED_BODY_NOTICE;
+      pdfText = withTruncation(pdfText, Boolean(response.truncated));
       return { error: null, body: pdfText, contentType: "application/pdf" };
     }
 
     if (!isTextCandidateContentType(contentType)) {
       const safeType = parseContentType(contentType) || "unknown type";
-      return {
-        error: `(non-text content: ${safeType}, ${response.body.length} bytes; not readable as text)`,
-        body: "",
+      return emptyResult(
+        `(non-text content: ${safeType}, ${response.body.length} bytes; not readable as text)`,
         contentType,
-      };
+      );
     }
 
     if (hasBinaryMagic(response.body)) {
-      return {
-        error: `(binary content, ${response.body.length} bytes; not readable as text)`,
-        body: "",
+      return emptyResult(
+        `(binary content, ${response.body.length} bytes; not readable as text)`,
         contentType,
-      };
+      );
     }
 
     const declaredCodec = declaredCharset ? normalizeCharset(declaredCharset) : null;
     const bomCodec = bomCodecFor(response.body);
-    const rawHtml =
+    const rawHtml = withTruncation(
       decodeWithCodec(
         response.body,
         bomCodec ?? declaredCodec ?? sniffMetaCharsetForHtml(response.body, contentType) ?? "utf-8",
-      ) + (response.truncated ? TRUNCATED_BODY_NOTICE : "");
-
+      ),
+      Boolean(response.truncated),
+    );
+    const nextUrl = isHtmlContent(rawHtml, contentType) ? metaRefreshUrl(rawHtml, currentUrl) : null;
+    if (nextUrl) {
+      const [refreshAllowed, refreshReason, refreshHost] = checkUrlAccess(nextUrl, policy);
+      if (!refreshAllowed) return emptyResult(refreshReason);
+      const refreshBudget = checkBudget();
+      if (refreshBudget !== null) return refreshBudget;
+      const redirected = await resolveWithBudget(refreshHost);
+      if (!redirected.ok) return emptyResult(redirected.reason);
+      currentUrl = nextUrl;
+      pinnedIp = redirected.ip;
+      pinnedFamily = redirected.family;
+      alternates = redirected.alternates ?? [];
+      alternateIndex = 0;
+      continue;
+    }
     if (looksBinary(rawHtml)) {
       let alt: string | null = null;
       if (
@@ -862,19 +955,18 @@ export async function fetchUrlRaw(
         if (!looksBinary(candidate)) alt = candidate;
       }
       if (alt !== null) {
-        if (response.truncated) alt += TRUNCATED_BODY_NOTICE;
+        alt = withTruncation(alt, Boolean(response.truncated));
         return { error: null, body: alt, contentType };
       }
-      return {
-        error: `(binary content, ${response.body.length} bytes; not readable as text)`,
-        body: "",
+      return emptyResult(
+        `(binary content, ${response.body.length} bytes; not readable as text)`,
         contentType,
-      };
+      );
     }
 
     return { error: null, body: rawHtml, contentType };
   }
-  return { error: "Failed to fetch URL: too many redirects.", body: "", contentType: "" };
+  return emptyResult("Failed to fetch URL: too many redirects.");
 }
 
 function bomCodecFor(bytes: Buffer): string | null {
@@ -966,8 +1058,14 @@ const META_KEYS: Record<string, keyof PageMeta> = {
 
 function cutAtCharBoundary(text: string, maxChars: number): string {
   const sliced = text.slice(0, maxChars);
+  if (!sliced) return sliced;
   const last = sliced.charCodeAt(sliced.length - 1);
-  return last >= 0xd800 && last <= 0xdbff ? sliced.slice(0, -1) : sliced;
+  if (last >= 0xd800 && last <= 0xdbff) return sliced.slice(0, -1);
+  if (last >= 0xdc00 && last <= 0xdfff) {
+    const prev = sliced.length >= 2 ? sliced.charCodeAt(sliced.length - 2) : 0;
+    if (prev < 0xd800 || prev > 0xdbff) return sliced.slice(0, -1);
+  }
+  return sliced;
 }
 
 function capMetaValue(value: string): string {
@@ -1011,10 +1109,7 @@ function pagePrefixedMarkdown(html: string): string {
   if (meta.site) lines.push(`Site: ${meta.site}`);
   const markdown = htmlToMarkdown(html, true);
   const converted = lines.length ? `${lines.join("\n")}\n\n${markdown}` : markdown;
-  if (html.endsWith(TRUNCATED_BODY_SUFFIX) && !converted.endsWith(TRUNCATED_BODY_SUFFIX)) {
-    return converted + TRUNCATED_BODY_NOTICE;
-  }
-  return converted;
+  return preserveTruncationNotice(html, converted);
 }
 
 function formatReadmeBody(body: string): string {
@@ -1024,7 +1119,62 @@ function formatReadmeBody(body: string): string {
   }
   return body;
 }
-
+function isHtmlContent(body: string, contentType: string): boolean {
+  return contentType.includes("html") || looksLikeHtml(body);
+}
+function renderBody(body: string, contentType: string): string {
+  return isHtmlContent(body, contentType) ? pagePrefixedMarkdown(body) : body.trim();
+}
+async function persistCache(url: string, body: string, contentType: string, useCache: boolean): Promise<void> {
+  if (!useCache) return;
+  try {
+    await setCached(url, body, contentType);
+  } catch {}
+}
+function isHttp404Error(error: string): boolean {
+  return /HTTP 404\b/.test(error);
+}
+async function fetchWaybackSnapshot(
+  url: string,
+  rawFetch: (url: string, options: RawFetchOptions) => Promise<RawFetchResult>,
+  options: RawFetchOptions,
+  deadlineMs: number,
+  now: () => number,
+): Promise<{ body: string; contentType: string; timestamp: string } | null> {
+  if (now() >= deadlineMs) return null;
+  const availUrl = `https://archive.org/wayback/available?url=${encodeURIComponent(url)}`;
+  let availResult: RawFetchResult | null = null;
+  try {
+    availResult = await rawFetch(availUrl, options);
+  } catch {
+    return null;
+  }
+  if (!availResult || availResult.error !== null) return null;
+  let snapshotUrl: string | null = null;
+  let timestamp = "";
+  try {
+    const data = JSON.parse(availResult.body) as {
+      archived_snapshots?: { closest?: { available?: boolean; url?: string; timestamp?: string; status?: string | number } };
+    };
+    const closest = data.archived_snapshots?.closest;
+    if (closest?.available && closest.url && String(closest.status) === "200") {
+      snapshotUrl = closest.url;
+      timestamp = closest.timestamp ?? "";
+    }
+  } catch {
+    return null;
+  }
+  if (!snapshotUrl) return null;
+  if (now() >= deadlineMs) return null;
+  let snapResult: RawFetchResult | null = null;
+  try {
+    snapResult = await rawFetch(snapshotUrl, options);
+  } catch {
+    return null;
+  }
+  if (!snapResult || snapResult.error !== null) return null;
+  return { body: snapResult.body, contentType: snapResult.contentType, timestamp };
+}
 export async function fetchPageText(
   url: string,
   options: FetchPageOptions = {},
@@ -1036,7 +1186,6 @@ export async function fetchPageText(
   const policy = options.websitePolicy ?? null;
   const maxChars = options.maxChars;
   const rawFetch = options.rawFetch ?? fetchUrlRaw;
-
   url = normalizeUrlScheme(url);
   const [allowed, reason] = checkUrlAccess(url, policy);
   if (!allowed) return reason;
@@ -1048,7 +1197,16 @@ export async function fetchPageText(
     maxPdfBytes: options.maxPdfBytes,
     seams: options.seams,
   };
-
+  const useCache = !options.seams && !options.rawFetch;
+  const githubRaw = githubRawContentUrl(url);
+  if (githubRaw) {
+    const rawResult = await rawFetch(githubRaw, rawFetchOptions);
+    if (rawResult.error === null) {
+      const out = renderBody(rawResult.body, rawResult.contentType);
+      await persistCache(url, rawResult.body, rawResult.contentType, useCache);
+      return truncatePageText(out, maxChars);
+    }
+  }
   const readmeApiUrl = githubRepoReadmeApiUrl(url);
   if (readmeApiUrl) {
     const readmeResult = await rawFetch(readmeApiUrl, {
@@ -1060,29 +1218,52 @@ export async function fetchPageText(
     });
     const apiBody = readmeResult.error === null ? formatReadmeBody(readmeResult.body) : "";
     if (apiBody.trim()) {
-      return truncatePageText(
-        `README of ${url} (fetched via the GitHub README API):\n\n` + apiBody,
-        maxChars,
-      );
+      const rendered = `README of ${url} (fetched via the GitHub README API):\n\n` + apiBody;
+      await persistCache(url, rendered, "text/markdown", useCache);
+      return truncatePageText(rendered, maxChars);
     }
     const rawReadmeUrl = githubRepoRawReadmeUrl(url);
     if (rawReadmeUrl) {
       const rawResult = await rawFetch(rawReadmeUrl, rawFetchOptions);
       const rawBody = rawResult.error === null ? formatReadmeBody(rawResult.body) : "";
       if (rawBody.trim()) {
-        return truncatePageText(
-          `README of ${url} (fetched via the GitHub raw README URL):\n\n` + rawBody,
-          maxChars,
-        );
+        const rendered = `README of ${url} (fetched via the GitHub raw README URL):\n\n` + rawBody;
+        await persistCache(url, rendered, "text/markdown", useCache);
+        return truncatePageText(rendered, maxChars);
       }
     }
   }
-
+  const originalUrl = url;
   const result = await rawFetch(url, rawFetchOptions);
-  if (result.error !== null) return result.error;
-
-  const isHtml = result.contentType.includes("html") || looksLikeHtml(result.body);
-  if (!isHtml) return truncatePageText(result.body.trim(), maxChars);
-
-  return truncatePageText(pagePrefixedMarkdown(result.body), maxChars);
+  if (result.error !== null) {
+    if (!result.error.startsWith("Blocked:")) {
+      if (useCache) {
+        try {
+          const cached = await getCached(originalUrl);
+          if (cached) {
+            let cachedOut = renderBody(cached.body, cached.contentType);
+            if (!isFresh(cached, now())) cachedOut += staleNotice(cached);
+            else cachedOut += "\n\n*Served from cache — network fetch failed*";
+            return truncatePageText(cachedOut, maxChars);
+          }
+        } catch {}
+      }
+      if (isHttp404Error(result.error)) {
+        try {
+          const wb = await fetchWaybackSnapshot(originalUrl, rawFetch, rawFetchOptions, deadlineMs, now);
+          if (wb) {
+            let out = renderBody(wb.body, wb.contentType);
+            const ts = wb.timestamp ? `${wb.timestamp.slice(0, 4)}-${wb.timestamp.slice(4, 6)}-${wb.timestamp.slice(6, 8)}` : "unknown date";
+            out = `*Fetched from Wayback Machine snapshot (${ts}) for ${originalUrl}:*\n\n` + out;
+            await persistCache(originalUrl, wb.body, wb.contentType, useCache);
+            return truncatePageText(out, maxChars);
+          }
+        } catch {}
+      }
+    }
+    return result.error;
+  }
+  const finalOut = renderBody(result.body, result.contentType);
+  await persistCache(originalUrl, result.body, result.contentType, useCache);
+  return truncatePageText(finalOut, maxChars);
 }