pi-unsloth-webtools 0.2.5 → 0.3.1

package/web-fetch.ts

@@ -1,22 +1,31 @@
 import { lookup as dnsLookup } from "node:dns/promises";
+import type { LookupAllOptions } from "node:dns";
 import http from "node:http";
 import https from "node:https";
+import { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from "node:zlib";
 import type { IncomingMessage } from "node:http";
+import type { Transform } from "node:stream";
 import {
   checkUrlAccess,
+  githubRepoRawReadmeUrl,
   githubRepoReadmeApiUrl,
   isPublicIp,
   normalizeUrlScheme,
   type WebsitePolicy,
 } from "./web-access.ts";
-import { htmlToMarkdown } from "./html-to-md.ts";
+import { collapseWhitespace, decodeHtmlEntities, feedHtml, htmlToMarkdown } from "./html-to-md.ts";
+import type { AttrDict } from "./html-to-md.ts";
 import { INVALID_CHARREFS } from "./entities.ts";
-import { extractPdfText, PdfParseError } from "./pdf.ts";
+import { extractPdfText } from "./pdf.ts";
 import { randomUserAgent } from "./user-agents.ts";
 
 const MAX_FETCH_BYTES = 512 * 1024;
+const MAX_DECOMPRESSED_BYTES = 64 * 1024 * 1024;
+const META_VALUE_MAX_CHARS = 300;
 const MAX_PDF_FETCH_BYTES = 10 * 1024 * 1024;
 const MAX_REQUESTS = 5;
+const MAX_SIGNAL_TIMEOUT_MS = 2 ** 31 - 1;
+export const DEFAULT_FETCH_TIMEOUT_MS = 60_000;
 
 const UTF32_LE_BOM = Buffer.from([0xff, 0xfe, 0x00, 0x00]);
 const UTF32_BE_BOM = Buffer.from([0x00, 0x00, 0xfe, 0xff]);
@@ -109,6 +118,28 @@ export class FetchTimeoutError extends Error {
     super("timed out");
   }
 }
+const FETCH_CANCELLED_MESSAGE = "Failed to fetch URL: cancelled.";
+const FETCH_TIMEOUT_MESSAGE = "Failed to fetch URL: timed out.";
+const TRUNCATED_BODY_NOTICE = "\n\n... (page truncated at the download limit)";
+const TRUNCATED_BODY_SUFFIX = "... (page truncated at the download limit)";
+
+function fetchErrorMessage(err: unknown): string {
+  if (err instanceof FetchCancelledError) return FETCH_CANCELLED_MESSAGE;
+  if (err instanceof FetchTimeoutError) return FETCH_TIMEOUT_MESSAGE;
+  const message = err instanceof Error ? err.message : String(err);
+  if (message === "cancelled") return FETCH_CANCELLED_MESSAGE;
+  if (message === "timed out") return FETCH_TIMEOUT_MESSAGE;
+  return `Failed to fetch URL: ${message}`;
+}
+
+function statusErrorResult(status: number): RawFetchResult {
+  const reason = http.STATUS_CODES[status] ?? "";
+  return {
+    error: `Failed to fetch URL: HTTP ${status}${reason ? ` ${reason}` : ""}`,
+    body: "",
+    contentType: "",
+  };
+}
 
 export interface FetchPageOptions {
   timeoutMs?: number;
@@ -127,6 +158,7 @@ export interface HopResponse {
   status: number;
   headers: Record<string, string | string[] | undefined>;
   body: Buffer;
+  truncated?: boolean;
 }
 
 export interface ResolvedHost {
@@ -134,6 +166,7 @@ export interface ResolvedHost {
   reason: string;
   ip: string;
   family: number;
+  alternates?: { ip: string; family: number }[];
 }
 
 export interface FetchSeams {
@@ -172,20 +205,43 @@ export interface RawFetchResult {
   contentType: string;
 }
 
+function htmlProbe(body: string, re: RegExp): boolean {
+  let i = 0;
+  const n = body.length;
+  while (i < n) {
+    while (i < n && /[ \t\n\r\f\v]/.test(body[i])) i++;
+    if (body.startsWith("<!--", i)) {
+      const close = body.indexOf("-->", i + 4);
+      if (close === -1) break;
+      i = close + 3;
+      continue;
+    }
+    if (body.startsWith("<?", i)) {
+      const close = body.indexOf("?>", i + 2);
+      if (close === -1) break;
+      i = close + 2;
+      continue;
+    }
+    break;
+  }
+  return re.test(body.slice(i, i + 256).toLowerCase());
+}
+
 export function looksLikeHtml(body: string): boolean {
-  const probe = body.replace(/^[ \t\n\r\f\v]+/, "").slice(0, 256).toLowerCase();
-  return HTML_LEADING_RE.test(probe);
+  return htmlProbe(body, HTML_LEADING_RE);
 }
 
 export function looksLikeHtmlDocument(body: string): boolean {
-  const probe = body.replace(/^[ \t\n\r\f\v]+/, "").slice(0, 256).toLowerCase();
-  return HTML_DOCUMENT_RE.test(probe);
+  return htmlProbe(body, HTML_DOCUMENT_RE);
+}
+
+function parseContentType(value: string | null | undefined): string {
+  return /^[\w.+-]+\/[\w.+-]+/.exec(value ?? "")?.[0] ?? "";
 }
 
 export function isTextCandidateContentType(contentType: string | null): boolean {
-  const match = /^[\w.+-]+\/[\w.+-]+/.exec(contentType ?? "");
-  if (!match) return true;
-  const ct = match[0].toLowerCase();
+  const ct = parseContentType(contentType).toLowerCase();
+  if (!ct) return true;
   if (ct.startsWith("text/")) return true;
   if (ct.startsWith("application/")) {
     const subtype = ct.slice("application/".length);
@@ -321,7 +377,12 @@ function sniffMetaCharsetForHtml(bytes: Buffer, contentType: string): string | n
 
 function decodeUtf32(bytes: Buffer, littleEndian: boolean): string {
   let out = "";
-  for (let i = 0; i + 3 < bytes.length; i += 4) {
+  let i = 0;
+  if (bytes.length >= 4) {
+    const first = littleEndian ? bytes.readUInt32LE(0) : bytes.readUInt32BE(0);
+    if (first === 0xfeff) i = 4;
+  }
+  for (; i + 3 < bytes.length; i += 4) {
     const v = littleEndian ? bytes.readUInt32LE(i) : bytes.readUInt32BE(i);
     if (v === 0 || v > 0x10ffff || (v >= 0xd800 && v <= 0xdfff)) {
       out += "\ufffd";
@@ -334,7 +395,8 @@ function decodeUtf32(bytes: Buffer, littleEndian: boolean): string {
 
 function decodeUtf16Be(bytes: Buffer): string {
   let out = "";
-  for (let i = 0; i + 1 < bytes.length; i += 2) {
+  let i = bytes.length >= 2 && bytes.readUInt16BE(0) === 0xfeff ? 2 : 0;
+  for (; i + 1 < bytes.length; i += 2) {
     out += String.fromCharCode(bytes.readUInt16BE(i));
   }
   return out;
@@ -409,7 +471,12 @@ function decodeTis620(bytes: Buffer): string {
 async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promise<ResolvedHost> {
   let addresses: { address: string; family: number }[];
   try {
-    addresses = await dnsLookup(hostname, { all: true, verbatim: true });
+    const lookupOptions: LookupAllOptions & { signal?: AbortSignal } = {
+      all: true,
+      verbatim: true,
+      signal,
+    };
+    addresses = await dnsLookup(hostname, lookupOptions);
   } catch (err) {
     return { ok: false, reason: `Failed to resolve host: ${err}`, ip: "", family: 0 };
   }
@@ -421,22 +488,44 @@ async function resolveAndValidate(hostname: string, signal?: AbortSignal): Promi
       return { ok: false, reason: `Blocked: refusing to fetch the non-public address ${entry.address}.`, ip: "", family: 0 };
     }
   }
+  addresses.sort((a, b) => (a.family === 4 ? 0 : 1) - (b.family === 4 ? 0 : 1));
   const first = addresses[0];
-  return { ok: true, reason: "", ip: first.address, family: first.family };
+  return {
+    ok: true,
+    reason: "",
+    ip: first.address,
+    family: first.family,
+    alternates: addresses.slice(1).map((entry) => ({ ip: entry.address, family: entry.family })),
+  };
 }
 
 
-function fetchBudgetExceeded(
+function budgetExceededResult(
   deadline: number | null,
   signal: AbortSignal | undefined,
   now: () => number = Date.now,
-): string | null {
-  if (signal?.aborted) return "Failed to fetch URL: cancelled.";
-  if (deadline !== null && now() >= deadline) return "Failed to fetch URL: timed out.";
+  contentType = "",
+): RawFetchResult | null {
+  if (signal?.aborted) return { error: FETCH_CANCELLED_MESSAGE, body: "", contentType };
+  if (deadline !== null && now() >= deadline) return { error: FETCH_TIMEOUT_MESSAGE, body: "", contentType };
   return null;
 }
 
 
+function contentEncodingCodec(value: string | string[] | undefined): string | null {
+  const declared = Array.isArray(value) ? value[0] : value;
+  const codec = (declared ?? "").split(",", 1)[0].trim().toLowerCase();
+  if (codec === "gzip" || codec === "x-gzip" || codec === "deflate" || codec === "br") return codec;
+  return null;
+}
+
+function createDecodeStream(codec: string): Transform {
+  const options = { maxOutputLength: MAX_DECOMPRESSED_BYTES };
+  if (codec === "br") return createBrotliDecompress(options);
+  if (codec === "deflate") return createInflate(options);
+  return createGunzip(options);
+}
+
 export function requestHop(opts: HopOptions): Promise<HopResponse> {
   return new Promise((resolve, reject) => {
     const url = opts.url;
@@ -460,12 +549,18 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
       action();
     };
     const request = transport.request(options, (res: IncomingMessage) => {
+      const codec = contentEncodingCodec(res.headers["content-encoding"]);
       const chunks: Buffer[] = [];
       let total = 0;
       let head = Buffer.alloc(0);
+      let truncated = false;
+      let decoder: Transform | null = null;
       const declaredPdf = String(res.headers["content-type"] ?? "").toLowerCase().includes("pdf");
       let limit = declaredPdf ? opts.maxPdfBytes : opts.maxBytes;
       let extendedForPdf = false;
+      const declaredLengthHeader = res.headers["content-length"];
+      const declaredLength =
+        declaredLengthHeader === undefined ? NaN : Number(declaredLengthHeader);
       const finish = (err: string | null, body: Buffer) => {
         settle(() => {
           if (err) {
@@ -477,44 +572,113 @@ export function requestHop(opts: HopOptions): Promise<HopResponse> {
               status: res.statusCode ?? 0,
               headers: res.headers as Record<string, string | string[] | undefined>,
               body,
+              truncated,
             });
           }
         });
       };
-      res.on("data", (chunk: Buffer) => {
+      const timeoutAndDestroy = () => {
+        decoder?.destroy();
+        res.destroy();
+        finish("timed out", Buffer.concat(chunks));
+      };
+      const acceptData = (chunk: Buffer) => {
         if (settled) return;
         const now = opts.nowMs ?? Date.now;
         if (opts.deadlineMs !== undefined && now() >= opts.deadlineMs) {
-          res.destroy();
-          finish("timed out", Buffer.concat(chunks));
+          timeoutAndDestroy();
           return;
         }
         if (head.length < 1024) {
           const need = 1024 - head.length;
           head = Buffer.concat([head, chunk.subarray(0, Math.min(need, chunk.length))]);
         }
-        if (!declaredPdf && !extendedForPdf && total + chunk.length > opts.maxBytes) {
+        if (!declaredPdf && !extendedForPdf && total + chunk.length > limit) {
           if (hasPdfMagic(head)) {
             limit = opts.maxPdfBytes;
             extendedForPdf = true;
           }
         }
         const space = limit - total;
-        if (space <= 0) {
-          res.destroy();
-          finish(null, Buffer.concat(chunks));
-          return;
-        }
         const take = chunk.subarray(0, Math.min(chunk.length, space));
         chunks.push(take);
         total += take.length;
         if (total >= limit) {
+          truncated =
+            codec !== null ||
+            take.length < chunk.length ||
+            (Number.isFinite(declaredLength) && declaredLength > total);
+          decoder?.destroy();
           res.destroy();
           finish(null, Buffer.concat(chunks));
         }
+      };
+      if (codec === null) {
+        res.on("data", (chunk: Buffer) => {
+          acceptData(chunk);
+        });
+        res.on("end", () => finish(null, Buffer.concat(chunks)));
+      } else {
+        let rawBuffer: Buffer[] = [];
+        let rawBufferBytes = 0;
+        let outputStarted = false;
+        let rawFallbackTried = false;
+        let resEnded = false;
+        const wire = (d: Transform) => {
+          d.on("data", (chunk: Buffer) => {
+            outputStarted = true;
+            acceptData(chunk);
+          });
+          d.on("end", () => finish(null, Buffer.concat(chunks)));
+          d.on("drain", () => res.resume());
+          d.on("error", (err: NodeJS.ErrnoException) => {
+            if (settled) return;
+            if (
+              codec === "deflate" &&
+              !rawFallbackTried &&
+              !outputStarted &&
+              err.code === "Z_DATA_ERROR"
+            ) {
+              rawFallbackTried = true;
+              d.destroy();
+              decoder = createInflateRaw({ maxOutputLength: MAX_DECOMPRESSED_BYTES });
+              wire(decoder);
+              for (const buffered of rawBuffer) decoder.write(buffered);
+              if (resEnded) decoder.end();
+              return;
+            }
+            if (outputStarted) {
+              truncated = true;
+              finish(null, Buffer.concat(chunks));
+              return;
+            }
+            finish(null, Buffer.concat(rawBuffer));
+          });
+        };
+        decoder = createDecodeStream(codec);
+        wire(decoder);
+        res.on("data", (chunk: Buffer) => {
+          if (settled) return;
+          const now = opts.nowMs ?? Date.now;
+          if (opts.deadlineMs !== undefined && now() >= opts.deadlineMs) {
+            timeoutAndDestroy();
+            return;
+          }
+          if (!outputStarted && !rawFallbackTried && rawBufferBytes < limit) {
+            rawBuffer.push(chunk);
+            rawBufferBytes += chunk.length;
+          }
+          if (!decoder!.write(chunk)) res.pause();
+        });
+        res.on("end", () => {
+          resEnded = true;
+          if (!settled) decoder!.end();
+        });
+      }
+      res.on("error", (err) => {
+        decoder?.destroy();
+        finish(err.message, Buffer.concat(chunks));
       });
-      res.on("end", () => finish(null, Buffer.concat(chunks)));
-      res.on("error", (err) => finish(err.message, Buffer.concat(chunks)));
     });
     const onAbort = () => request.destroy(new FetchCancelledError());
     opts.signal?.addEventListener("abort", onAbort, { once: true });
@@ -540,24 +704,34 @@ export async function fetchUrlRaw(
   const seams = options.seams ?? {};
   const resolveHost = seams.resolve ?? resolveAndValidate;
   const performRequest = seams.request ?? requestHop;
+  const resolveWithBudget = async (hostname: string): Promise<ResolvedHost> => {
+    const deadlineSignal = AbortSignal.timeout(Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, deadline - now())));
+    const resolveSignal = signal ? AbortSignal.any([signal, deadlineSignal]) : deadlineSignal;
+    const resolved = await resolveHost(hostname, resolveSignal);
+    if (resolved.ok) return resolved;
+    if (signal?.aborted) return { ...resolved, reason: FETCH_CANCELLED_MESSAGE };
+    if (resolveSignal.aborted) return { ...resolved, reason: FETCH_TIMEOUT_MESSAGE };
+    return resolved;
+  };
 
   url = normalizeUrlScheme(url);
   const [allowed, reason, hostname] = checkUrlAccess(url, policy);
   if (!allowed) return { error: reason, body: "", contentType: "" };
 
-  let budgetError = fetchBudgetExceeded(deadline, signal, now);
-  if (budgetError !== null) return { error: budgetError, body: "", contentType: "" };
-  let resolved = await resolveHost(hostname, signal);
+  const budgetResult = budgetExceededResult(deadline, signal, now);
+  if (budgetResult !== null) return budgetResult;
+  let resolved = await resolveWithBudget(hostname);
   if (!resolved.ok) return { error: resolved.reason, body: "", contentType: "" };
 
   let currentUrl = url;
   let pinnedIp = resolved.ip;
   let pinnedFamily = resolved.family;
+  let alternates: { ip: string; family: number }[] = resolved.alternates ?? [];
+  let alternateIndex = 0;
   const userAgent = randomUserAgent();
-
   for (let hop = 0; hop < MAX_REQUESTS; hop++) {
-    budgetError = fetchBudgetExceeded(deadline, signal, now);
-    if (budgetError !== null) return { error: budgetError, body: "", contentType: "" };
+    const budgetResult = budgetExceededResult(deadline, signal, now);
+    if (budgetResult !== null) return budgetResult;
     const parsed = new URL(currentUrl);
     const hostHeader = parsed.hostname + (parsed.port ? `:${parsed.port}` : "");
     const headers: Record<string, string> = {
@@ -583,26 +757,22 @@ export async function fetchUrlRaw(
         signal,
       });
     } catch (err) {
-      if (err instanceof FetchCancelledError)
-        return { error: "Failed to fetch URL: cancelled.", body: "", contentType: "" };
-      if (err instanceof FetchTimeoutError)
-        return { error: "Failed to fetch URL: timed out.", body: "", contentType: "" };
-      const message = err instanceof Error ? err.message : String(err);
-      if (message === "cancelled")
-        return { error: "Failed to fetch URL: cancelled.", body: "", contentType: "" };
-      if (message === "timed out")
-        return { error: "Failed to fetch URL: timed out.", body: "", contentType: "" };
-      return { error: `Failed to fetch URL: ${message}`, body: "", contentType: "" };
+      if (
+        !(err instanceof FetchCancelledError) &&
+        !(err instanceof FetchTimeoutError) &&
+        alternateIndex < alternates.length
+      ) {
+        const next = alternates[alternateIndex++];
+        pinnedIp = next.ip;
+        pinnedFamily = next.family;
+        continue;
+      }
+      return { error: fetchErrorMessage(err), body: "", contentType: "" };
     }
 
     if (response.status >= 300 && response.status < 400) {
       if (![301, 302, 303, 307, 308].includes(response.status)) {
-        const reason = http.STATUS_CODES[response.status] ?? "";
-        return {
-          error: `Failed to fetch URL: HTTP ${response.status}${reason ? ` ${reason}` : ""}`,
-          body: "",
-          contentType: "",
-        };
+        return statusErrorResult(response.status);
       }
       const rawLocation = response.headers.location;
       const location = Array.isArray(rawLocation) ? rawLocation[0] : rawLocation;
@@ -623,29 +793,24 @@ export async function fetchUrlRaw(
         policy,
       );
       if (!redirectAllowed) return { error: redirectReason, body: "", contentType: "" };
-      const redirected = await resolveHost(redirectHost, signal);
+      const redirected = await resolveWithBudget(redirectHost);
       if (!redirected.ok) return { error: redirected.reason, body: "", contentType: "" };
       pinnedIp = redirected.ip;
       pinnedFamily = redirected.family;
+      alternates = redirected.alternates ?? [];
+      alternateIndex = 0;
       continue;
     }
 
-    budgetError = fetchBudgetExceeded(deadline, signal, now);
-    if (budgetError !== null) return { error: budgetError, body: "", contentType: "" };
+    const postBudgetResult = budgetExceededResult(deadline, signal, now);
+    if (postBudgetResult !== null) return postBudgetResult;
 
     if (response.status >= 400) {
-      const reason = http.STATUS_CODES[response.status] ?? "";
-      return {
-        error: `Failed to fetch URL: HTTP ${response.status}${reason ? ` ${reason}` : ""}`,
-        body: "",
-        contentType: "",
-      };
+      return statusErrorResult(response.status);
     }
 
     const contentTypeHeader = response.headers["content-type"];
-    const contentType = contentTypeHeader
-      ? (/^[\w.+-]+\/[\w.+-]+/.exec(String(contentTypeHeader).toLowerCase()) ?? [""])[0]
-      : "";
+    const contentType = parseContentType(contentTypeHeader ? String(contentTypeHeader).toLowerCase() : null);
     const declaredCharset = contentTypeHeader
       ? (/charset=([^;\s]+)/i.exec(String(contentTypeHeader))?.[1] ?? null)
       : null;
@@ -664,16 +829,23 @@ export async function fetchUrlRaw(
       try {
         pdfText = await extractPdfText(response.body);
       } catch {
-        return { error: "(PDF content could not be read as text)", body: "", contentType };
+        return {
+          error: response.truncated
+            ? "(PDF content could not be read as text; the download was truncated at the download limit)"
+            : "(PDF content could not be read as text)",
+          body: "",
+          contentType,
+        };
       }
-      budgetError = fetchBudgetExceeded(deadline, signal, now);
-      if (budgetError !== null) return { error: budgetError, body: "", contentType };
+      const budgetResult = budgetExceededResult(deadline, signal, now, contentType);
+      if (budgetResult !== null) return budgetResult;
       if (!pdfText) pdfText = "(PDF contains no extractable text)";
+      if (response.truncated) pdfText += TRUNCATED_BODY_NOTICE;
       return { error: null, body: pdfText, contentType: "application/pdf" };
     }
 
     if (!isTextCandidateContentType(contentType)) {
-      const safeType = /^[\w.+-]+\/[\w.+-]+/.exec(contentType ?? "")?.[0] ?? "unknown type";
+      const safeType = parseContentType(contentType) || "unknown type";
       return {
         error: `(non-text content: ${safeType}, ${response.body.length} bytes; not readable as text)`,
         body: "",
@@ -691,10 +863,11 @@ export async function fetchUrlRaw(
 
     const declaredCodec = declaredCharset ? normalizeCharset(declaredCharset) : null;
     const bomCodec = bomCodecFor(response.body);
-    const rawHtml = decodeWithCodec(
-      response.body,
-      bomCodec ?? declaredCodec ?? sniffMetaCharsetForHtml(response.body, contentType) ?? "utf-8",
-    );
+    const rawHtml =
+      decodeWithCodec(
+        response.body,
+        bomCodec ?? declaredCodec ?? sniffMetaCharsetForHtml(response.body, contentType) ?? "utf-8",
+      ) + (response.truncated ? TRUNCATED_BODY_NOTICE : "");
 
     if (looksBinary(rawHtml)) {
       let alt: string | null = null;
@@ -706,6 +879,7 @@ export async function fetchUrlRaw(
         if (!looksBinary(candidate)) alt = candidate;
       }
       if (alt !== null) {
+        if (response.truncated) alt += TRUNCATED_BODY_NOTICE;
         return { error: null, body: alt, contentType };
       }
       return {
@@ -732,16 +906,147 @@ function bomCodecFor(bytes: Buffer): string | null {
 export function truncatePageText(text: string, maxChars?: number): string {
   if (!text) return "(page returned no readable text)";
   if (typeof maxChars === "number" && maxChars > 0 && text.length > maxChars) {
-    return text.slice(0, maxChars) + `\n\n... (truncated, ${text.length} chars total)`;
+    const hadCapNotice = text.endsWith(TRUNCATED_BODY_SUFFIX);
+    const core = (hadCapNotice ? text.slice(0, -TRUNCATED_BODY_SUFFIX.length) : text).trimEnd();
+    return (
+      cutAtCharBoundary(core, maxChars) +
+      `\n\n... (truncated, ${text.length} chars total)` +
+      (hadCapNotice ? TRUNCATED_BODY_NOTICE : "")
+    );
   }
   return text;
 }
 
+const NON_DOCUMENT_TITLE_TAGS = new Set(["math", "noscript", "svg", "template"]);
+
+function extractPageTitle(html: string): string {
+  let inTitle = false;
+  let done = false;
+  let skipDepth = 0;
+  const parts: string[] = [];
+  feedHtml(html, {
+    handleStartTag(name: string) {
+      if (NON_DOCUMENT_TITLE_TAGS.has(name)) {
+        skipDepth++;
+        return;
+      }
+      if (skipDepth) return;
+      if (name === "title" && !done) {
+        inTitle = true;
+      } else if (inTitle) {
+        inTitle = false;
+      }
+    },
+    handleStartEndTag() {},
+    handleEndTag(name: string) {
+      if (NON_DOCUMENT_TITLE_TAGS.has(name)) {
+        skipDepth = Math.max(0, skipDepth - 1);
+        return;
+      }
+      if (skipDepth) return;
+      if (name === "title") {
+        inTitle = false;
+        done = true;
+      }
+    },
+    handleData(text: string) {
+      if (inTitle) parts.push(text);
+    },
+    handleEntityRef(name: string) {
+      if (inTitle) parts.push(decodeHtmlEntities(`&${name};`));
+    },
+    handleCharRef(name: string) {
+      if (inTitle) parts.push(decodeHtmlEntities(`&#${name};`));
+    },
+  });
+  return collapseWhitespace(parts.join(""));
+}
+
+interface PageMeta {
+  title: string;
+  author: string;
+  date: string;
+  site: string;
+}
+
+const META_KEYS: Record<string, keyof PageMeta> = {
+  author: "author",
+  "article:author": "author",
+  "dc.creator": "author",
+  "article:published_time": "date",
+  date: "date",
+  "dc.date": "date",
+  datepublished: "date",
+  "og:site_name": "site",
+  "application-name": "site",
+};
+
+function cutAtCharBoundary(text: string, maxChars: number): string {
+  const sliced = text.slice(0, maxChars);
+  const last = sliced.charCodeAt(sliced.length - 1);
+  return last >= 0xd800 && last <= 0xdbff ? sliced.slice(0, -1) : sliced;
+}
+
+function capMetaValue(value: string): string {
+  if (value.length <= META_VALUE_MAX_CHARS) return value;
+  return cutAtCharBoundary(value, META_VALUE_MAX_CHARS);
+}
+
+function extractPageMeta(html: string): PageMeta {
+  const meta: PageMeta = { title: extractPageTitle(html), author: "", date: "", site: "" };
+  const seen = new Set<string>();
+  const record = (name: string, attrs: AttrDict) => {
+    if (name !== "meta") return;
+    const key = (attrs["property"] ?? attrs["name"] ?? "").toLowerCase();
+    const content = collapseWhitespace(attrs["content"] ?? "");
+    if (!key || !content || seen.has(key)) return;
+    seen.add(key);
+    const field = META_KEYS[key];
+    if (field && !meta[field]) meta[field] = capMetaValue(content);
+  };
+  feedHtml(html, {
+    handleStartTag(name: string, attrs: AttrDict) {
+      record(name, attrs);
+    },
+    handleStartEndTag(name: string, attrs: AttrDict) {
+      record(name, attrs);
+    },
+    handleEndTag() {},
+    handleData() {},
+    handleEntityRef() {},
+    handleCharRef() {},
+  });
+  return meta;
+}
+
+function pagePrefixedMarkdown(html: string): string {
+  const meta = extractPageMeta(html);
+  const lines: string[] = [];
+  if (meta.title) lines.push(`Title: ${meta.title}`);
+  if (meta.author) lines.push(`Author: ${meta.author}`);
+  if (meta.date) lines.push(`Date: ${meta.date}`);
+  if (meta.site) lines.push(`Site: ${meta.site}`);
+  const markdown = htmlToMarkdown(html, true);
+  const converted = lines.length ? `${lines.join("\n")}\n\n${markdown}` : markdown;
+  if (html.endsWith(TRUNCATED_BODY_SUFFIX) && !converted.endsWith(TRUNCATED_BODY_SUFFIX)) {
+    return converted + TRUNCATED_BODY_NOTICE;
+  }
+  return converted;
+}
+
+function formatReadmeBody(body: string): string {
+  if (looksLikeHtmlDocument(body)) {
+    const converted = pagePrefixedMarkdown(body);
+    if (converted.trim()) return converted;
+  }
+  return body;
+}
+
 export async function fetchPageText(
   url: string,
   options: FetchPageOptions = {},
 ): Promise<string> {
-  const timeoutMs = options.timeoutMs ?? 60_000;
+  const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
   const now = options.nowMs ?? Date.now;
   const deadlineMs = options.deadlineMs ?? now() + timeoutMs;
   const signal = options.signal;
@@ -752,48 +1057,49 @@ export async function fetchPageText(
   url = normalizeUrlScheme(url);
   const [allowed, reason] = checkUrlAccess(url, policy);
   if (!allowed) return reason;
+  const rawFetchOptions = {
+    deadlineMs,
+    signal,
+    websitePolicy: policy,
+    maxBytes: options.maxBytes,
+    maxPdfBytes: options.maxPdfBytes,
+    seams: options.seams,
+  };
 
   const readmeApiUrl = githubRepoReadmeApiUrl(url);
   if (readmeApiUrl) {
     const readmeResult = await rawFetch(readmeApiUrl, {
-      deadlineMs,
-      signal,
-      websitePolicy: policy,
-      maxBytes: options.maxBytes,
-      maxPdfBytes: options.maxPdfBytes,
-      seams: options.seams,
+      ...rawFetchOptions,
       extraHeaders: {
         Accept: "application/vnd.github.raw+json",
         "X-GitHub-Api-Version": "2022-11-28",
       },
     });
-    if (readmeResult.error === null && readmeResult.body.trim()) {
-      let readmeBody = readmeResult.body;
-      if (looksLikeHtmlDocument(readmeBody)) {
-        const converted = htmlToMarkdown(readmeBody, true);
-        if (converted.trim()) readmeBody = converted;
-      }
-      if (readmeBody.trim()) {
+    const apiBody = readmeResult.error === null ? formatReadmeBody(readmeResult.body) : "";
+    if (apiBody.trim()) {
+      return truncatePageText(
+        `README of ${url} (fetched via the GitHub README API):\n\n` + apiBody,
+        maxChars,
+      );
+    }
+    const rawReadmeUrl = githubRepoRawReadmeUrl(url);
+    if (rawReadmeUrl) {
+      const rawResult = await rawFetch(rawReadmeUrl, rawFetchOptions);
+      const rawBody = rawResult.error === null ? formatReadmeBody(rawResult.body) : "";
+      if (rawBody.trim()) {
         return truncatePageText(
-          `README of ${url} (fetched via the GitHub README API):\n\n` + readmeBody,
+          `README of ${url} (fetched via the GitHub raw README URL):\n\n` + rawBody,
           maxChars,
         );
       }
     }
   }
 
-  const result = await rawFetch(url, {
-    deadlineMs,
-    signal,
-    websitePolicy: policy,
-    maxBytes: options.maxBytes,
-    maxPdfBytes: options.maxPdfBytes,
-    seams: options.seams,
-  });
+  const result = await rawFetch(url, rawFetchOptions);
   if (result.error !== null) return result.error;
 
   const isHtml = result.contentType.includes("html") || looksLikeHtml(result.body);
   if (!isHtml) return truncatePageText(result.body.trim(), maxChars);
 
-  return truncatePageText(htmlToMarkdown(result.body, true), maxChars);
+  return truncatePageText(pagePrefixedMarkdown(result.body), maxChars);
 }