pi-memory-stone 0.1.4 → 0.1.5

package/src/vault/capture.ts

@@ -0,0 +1,268 @@
+/** URL capture for memory vault source pages. */
+
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import { redactSecrets } from "../privacy/index.js";
+import { initVault, getVaultStatus, type VaultRegistry, type VaultRegistryPage } from "./index.js";
+import { sanitizeSlug } from "./markdown.js";
+import { resolveSourcePacketPath, resolveVaultPath, type VaultScope } from "./paths.js";
+import { extractArticle, type ExtractedArticle } from "./extract.js";
+import { assessCaptureQuality, type CaptureQuality, type CaptureQualityReport } from "./quality.js";
+import { fetchCandidate, type CaptureFetchAttempt, type CaptureFetchOptions, type FetchedCandidate } from "./fetch.js";
+import { resolveCaptureTargets, type CaptureCandidate } from "./url-resolvers.js";
+
+const MAX_EXTRACTED_CHARS = 200_000;
+
+export interface CaptureUrlOptions extends CaptureFetchOptions {}
+
+export interface CaptureUrlResult {
+  vaultPath: string;
+  pagePath: string;
+  sourcePacketPath: string;
+  title: string;
+  url: string;
+  finalUrl: string;
+  initialized: boolean;
+  quality: CaptureQuality;
+  qualityScore: number;
+  warnings: string[];
+}
+
+export async function captureUrlToVault(
+  scope: VaultScope,
+  projectId: string | null,
+  cwd: string,
+  url: string,
+  options: CaptureUrlOptions = {},
+): Promise<CaptureUrlResult> {
+  const targets = resolveCaptureTargets(url);
+  const vaultPath = resolveVaultPath(scope, projectId, cwd);
+  const wasInitialized = getVaultStatus(scope, projectId, cwd).initialized;
+  if (!wasInitialized) {
+    initVault(scope, projectId, cwd);
+  }
+
+  const selected = await fetchAndExtractBest(targets.candidates, options);
+  const title = selected.extracted.title || new URL(selected.fetched.finalUrl).hostname;
+  const slug = sanitizeSlug(title).slice(0, 70) || "captured-page";
+  const captureId = `SRC-${new Date().toISOString().slice(0, 10)}-${sha256(targets.originalUrl).slice(0, 8)}`;
+  const packetPath = resolveSourcePacketPath(scope, projectId, cwd, captureId);
+  const packetRelPath = normalizePath(relative(vaultPath, packetPath));
+  const sourcePageRelPath = join("sources", `${slug}-${sha256(targets.originalUrl).slice(0, 8)}.md`);
+  const sourcePagePath = join(vaultPath, sourcePageRelPath);
+
+  mkdirSync(join(packetPath, "original"), { recursive: true, mode: 0o700 });
+  mkdirSync(join(packetPath, "attachments"), { recursive: true, mode: 0o700 });
+  mkdirSync(join(vaultPath, "sources"), { recursive: true, mode: 0o700 });
+
+  const capturedAt = new Date().toISOString();
+  const originalName = originalArtifactName(selected.fetched.contentType, selected.fetched.finalUrl, selected.extracted.extractor);
+  const extractedMarkdown = unescapeRedactionMarkers(redactSecrets(selected.extracted.markdown)).slice(0, MAX_EXTRACTED_CHARS);
+  const redactedRaw = redactSecrets(selected.fetched.raw);
+  const contentHash = sha256(extractedMarkdown);
+
+  const manifest = {
+    id: captureId,
+    url: targets.originalUrl,
+    canonical_url: selected.extracted.canonicalUrl ?? selected.fetched.finalUrl,
+    final_url: selected.fetched.finalUrl,
+    title,
+    byline: selected.extracted.byline,
+    site_name: selected.extracted.siteName,
+    excerpt: selected.extracted.excerpt,
+    published_at: selected.extracted.publishedAt,
+    content_type: selected.fetched.contentType,
+    captured_at: capturedAt,
+    original: `original/${originalName}`,
+    extracted: "extracted.md",
+    metadata: "metadata.json",
+    attempts: selected.attempts,
+    extraction: {
+      extractor: selected.extracted.extractor,
+      strategy: selected.fetched.candidate.strategy,
+      candidate_kind: selected.fetched.candidate.kind,
+    },
+    quality: selected.quality,
+    content_hash: contentHash,
+  };
+
+  const metadata = {
+    title,
+    byline: selected.extracted.byline,
+    site_name: selected.extracted.siteName,
+    excerpt: selected.extracted.excerpt,
+    published_at: selected.extracted.publishedAt,
+    source_url: targets.originalUrl,
+    canonical_url: selected.extracted.canonicalUrl ?? selected.fetched.finalUrl,
+    final_url: selected.fetched.finalUrl,
+    content_hash: contentHash,
+    extractor: selected.extracted.extractor,
+    fetch_strategy: selected.fetched.candidate.strategy,
+    quality: selected.quality,
+  };
+
+  writeFileSync(join(packetPath, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 });
+  writeFileSync(join(packetPath, "metadata.json"), JSON.stringify(metadata, null, 2) + "\n", { mode: 0o600 });
+  writeFileSync(join(packetPath, "original", originalName), redactedRaw, { mode: 0o600 });
+  writeFileSync(join(packetPath, "extracted.md"), extractedMarkdown, { mode: 0o600 });
+
+  const pageMarkdown = renderSourcePage({
+    title,
+    url: targets.originalUrl,
+    canonicalUrl: selected.extracted.canonicalUrl ?? selected.fetched.finalUrl,
+    capturedAt,
+    captureId,
+    packetRelPath,
+    extractedMarkdown,
+    quality: selected.quality,
+    warnings: selected.quality.warnings,
+  });
+  writeFileSync(sourcePagePath, pageMarkdown, { mode: 0o600 });
+
+  updateRegistry(vaultPath, {
+    path: normalizePath(sourcePageRelPath),
+    title,
+    kind: "web_source",
+    source_url: targets.originalUrl,
+    source_packet: packetRelPath,
+    content_hash: sha256(pageMarkdown),
+    generated: true,
+    created_at: capturedAt,
+    updated_at: capturedAt,
+  });
+
+  return {
+    vaultPath,
+    pagePath: sourcePagePath,
+    sourcePacketPath: packetPath,
+    title,
+    url: targets.originalUrl,
+    finalUrl: selected.fetched.finalUrl,
+    initialized: !wasInitialized,
+    quality: selected.quality.quality,
+    qualityScore: selected.quality.score,
+    warnings: selected.quality.warnings,
+  };
+}
+
+interface ExtractedCandidate {
+  fetched: FetchedCandidate;
+  extracted: ExtractedArticle;
+  quality: CaptureQualityReport;
+  attempts: CaptureFetchAttempt[];
+}
+
+async function fetchAndExtractBest(candidates: CaptureCandidate[], options: CaptureUrlOptions): Promise<ExtractedCandidate> {
+  const allAttempts: CaptureFetchAttempt[] = [];
+  let best: ExtractedCandidate | null = null;
+  const errors: string[] = [];
+
+  for (const candidate of candidates) {
+    try {
+      const fetched = await fetchCandidate(candidate, options);
+      allAttempts.push(...fetched.attempts);
+      const redactedRaw = redactSecrets(fetched.raw);
+      const extracted = extractArticle({
+        raw: redactedRaw,
+        contentType: fetched.contentType,
+        url: fetched.finalUrl,
+        candidateKind: candidate.kind,
+      });
+      const quality = assessCaptureQuality({
+        title: extracted.title,
+        markdown: extracted.markdown,
+        extractor: extracted.extractor,
+      });
+      const current: ExtractedCandidate = { fetched, extracted, quality, attempts: [...allAttempts] };
+      if (!best || current.quality.score > best.quality.score) best = current;
+      if (quality.quality === "good") return current;
+    } catch (error) {
+      const attempts = (error as Error & { attempts?: CaptureFetchAttempt[] }).attempts;
+      if (attempts) allAttempts.push(...attempts);
+      errors.push(`${candidate.strategy}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+
+  if (best) return { ...best, attempts: allAttempts };
+  throw new Error(`Unable to fetch article. Attempts failed: ${errors.join("; ")}`);
+}
+
+function renderSourcePage(input: {
+  title: string;
+  url: string;
+  canonicalUrl: string;
+  capturedAt: string;
+  captureId: string;
+  packetRelPath: string;
+  extractedMarkdown: string;
+  quality: CaptureQualityReport;
+  warnings: string[];
+}): string {
+  const warningLines = input.warnings.length > 0
+    ? ["", "Warnings:", ...input.warnings.map((warning) => `- ${warning}`)]
+    : [];
+
+  return [
+    "---",
+    `title: ${JSON.stringify(input.title)}`,
+    "kind: web_source",
+    `source_url: ${JSON.stringify(input.url)}`,
+    `canonical_url: ${JSON.stringify(input.canonicalUrl)}`,
+    `source_packet: ${JSON.stringify(input.packetRelPath)}`,
+    `captured_at: ${JSON.stringify(input.capturedAt)}`,
+    `capture_id: ${JSON.stringify(input.captureId)}`,
+    `quality: ${JSON.stringify(input.quality.quality)}`,
+    `quality_score: ${input.quality.score}`,
+    "generated: true",
+    "source: pi-memory-stone",
+    "---",
+    "",
+    `# ${input.title.replace(/[\r\n]+/g, " ").trim()}`,
+    "",
+    `Source: ${input.url}`,
+    `Canonical: ${input.canonicalUrl}`,
+    `Captured: ${input.capturedAt}`,
+    `Quality: ${input.quality.quality} (${input.quality.score})`,
+    `Source packet: ${input.captureId} (stored outside vault: ${input.packetRelPath})`,
+    ...warningLines,
+    "",
+    "## Extracted text",
+    "",
+    input.extractedMarkdown.trim() || "_No text extracted._",
+    "",
+  ].join("\n");
+}
+
+function updateRegistry(vaultPath: string, page: VaultRegistryPage & {
+  source_url: string;
+  source_packet: string;
+}): void {
+  const registryPath = join(vaultPath, "meta", "registry.json");
+  const registry = JSON.parse(readFileSync(registryPath, "utf8")) as VaultRegistry;
+  const pages = registry.pages.filter((existing) => existing.path !== page.path);
+  pages.push(page);
+  pages.sort((a, b) => a.path.localeCompare(b.path));
+  registry.pages = pages;
+  registry.generated_at = new Date().toISOString();
+  writeFileSync(registryPath, JSON.stringify(registry, null, 2) + "\n", { mode: 0o600 });
+}
+
+function originalArtifactName(contentType: string, finalUrl: string, extractor: string): string {
+  if (contentType.includes("html") || extractor.startsWith("html")) return "response.html";
+  if (contentType.includes("markdown") || finalUrl.toLowerCase().match(/\.(md|markdown|mdx)(?:$|[?#])/) || extractor === "markdown") return "response.md";
+  if (contentType.includes("pdf") || finalUrl.toLowerCase().match(/\.pdf(?:$|[?#])/)) return "response.pdf.txt";
+  return "response.txt";
+}
+
+function normalizePath(path: string): string {
+  return path.split(/[\\/]+/).join("/");
+}
+
+function unescapeRedactionMarkers(markdown: string): string {
+  return markdown.replace(/\\\[REDACTED:([a-z-]+)\\\]/g, "[REDACTED:$1]");
+}
+
+function sha256(content: string): string {
+  return createHash("sha256").update(content).digest("hex");
+}

package/src/vault/extract.ts

@@ -0,0 +1,259 @@
+/** Content-type aware article extraction for vault source capture. */
+
+import { JSDOM } from "jsdom";
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+import type { CaptureCandidateKind } from "./url-resolvers.js";
+
+export type ExtractionKind = "html-readability" | "html-main" | "markdown" | "text" | "pdf-unsupported";
+
+export interface ExtractedArticle {
+  title: string;
+  markdown: string;
+  byline?: string;
+  siteName?: string;
+  excerpt?: string;
+  publishedAt?: string;
+  canonicalUrl?: string;
+  extractor: ExtractionKind;
+}
+
+export function extractArticle(input: {
+  raw: string;
+  contentType: string;
+  url: string;
+  candidateKind: CaptureCandidateKind;
+}): ExtractedArticle {
+  if (isPdf(input.contentType, input.url, input.candidateKind)) {
+    return {
+      title: titleFromUrl(input.url),
+      markdown: "_PDF capture is not yet supported. Raw response was stored for future extraction._",
+      canonicalUrl: input.url,
+      extractor: "pdf-unsupported",
+    };
+  }
+
+  if (isHtml(input.raw, input.contentType, input.candidateKind)) {
+    return extractHtml(input.raw, input.url);
+  }
+
+  if (isMarkdown(input.contentType, input.url, input.candidateKind)) {
+    return extractMarkdown(input.raw, input.url);
+  }
+
+  return extractText(input.raw, input.url);
+}
+
+function extractHtml(html: string, url: string): ExtractedArticle {
+  const dom = new JSDOM(html, { url });
+  const document = dom.window.document;
+  const metadata = htmlMetadata(document, url);
+
+  const reader = new Readability(document.cloneNode(true) as Document);
+  const article = reader.parse();
+  if (article?.content && (article.textContent ?? "").trim().length > 0) {
+    const markdown = htmlFragmentToMarkdown(article.content);
+    return {
+      title: normalizeWhitespace(article.title || metadata.title || titleFromUrl(url)),
+      markdown,
+      byline: article.byline || metadata.byline,
+      siteName: article.siteName || metadata.siteName,
+      excerpt: article.excerpt || metadata.excerpt,
+      publishedAt: metadata.publishedAt,
+      canonicalUrl: metadata.canonicalUrl,
+      extractor: "html-readability",
+    };
+  }
+
+  const fallbackElement = document.querySelector("article") ?? document.querySelector("main") ?? document.body;
+  return {
+    title: metadata.title || titleFromUrl(url),
+    markdown: htmlFragmentToMarkdown(fallbackElement?.innerHTML ?? html),
+    byline: metadata.byline,
+    siteName: metadata.siteName,
+    excerpt: metadata.excerpt,
+    publishedAt: metadata.publishedAt,
+    canonicalUrl: metadata.canonicalUrl,
+    extractor: "html-main",
+  };
+}
+
+function extractMarkdown(markdown: string, url: string): ExtractedArticle {
+  const title = markdown.match(/^#\s+(.+)$/m)?.[1]
+    ?? markdown.match(/^title:\s*["']?(.+?)["']?\s*$/im)?.[1]
+    ?? titleFromUrl(url);
+  return {
+    title: normalizeWhitespace(stripMarkdown(title)),
+    markdown: markdown.trim(),
+    canonicalUrl: url,
+    extractor: "markdown",
+  };
+}
+
+function extractText(text: string, url: string): ExtractedArticle {
+  const lines = text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+  return {
+    title: normalizeWhitespace(lines[0] || titleFromUrl(url)),
+    markdown: text.trim(),
+    canonicalUrl: url,
+    extractor: "text",
+  };
+}
+
+function htmlMetadata(document: Document, url: string): Omit<ExtractedArticle, "markdown" | "extractor"> {
+  const jsonLd = parseJsonLd(document);
+  const title = meta(document, "property", "og:title")
+    ?? meta(document, "name", "twitter:title")
+    ?? document.querySelector("title")?.textContent
+    ?? document.querySelector("h1")?.textContent
+    ?? jsonLd?.headline
+    ?? titleFromUrl(url);
+
+  return {
+    title: normalizeWhitespace(title),
+    byline: meta(document, "name", "author") ?? jsonLd?.author,
+    siteName: meta(document, "property", "og:site_name"),
+    excerpt: meta(document, "name", "description") ?? meta(document, "property", "og:description"),
+    publishedAt: meta(document, "property", "article:published_time") ?? jsonLd?.datePublished,
+    canonicalUrl: document.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href
+      ?? meta(document, "property", "og:url")
+      ?? url,
+  };
+}
+
+function parseJsonLd(document: Document): { headline?: string; author?: string; datePublished?: string } | null {
+  for (const script of [...document.querySelectorAll<HTMLScriptElement>('script[type="application/ld+json"]')]) {
+    try {
+      const parsed = JSON.parse(script.textContent || "null") as unknown;
+      const objects = Array.isArray(parsed) ? parsed : [parsed];
+      for (const object of objects) {
+        if (!object || typeof object !== "object") continue;
+        const record = object as Record<string, unknown>;
+        const author = record.author;
+        return {
+          headline: stringValue(record.headline ?? record.name),
+          author: typeof author === "string"
+            ? author
+            : Array.isArray(author)
+              ? stringValue((author[0] as Record<string, unknown> | undefined)?.name)
+              : stringValue((author as Record<string, unknown> | undefined)?.name),
+          datePublished: stringValue(record.datePublished),
+        };
+      }
+    } catch {
+      // Ignore malformed embedded metadata.
+    }
+  }
+  return null;
+}
+
+function meta(document: Document, attr: "name" | "property", value: string): string | undefined {
+  return normalizeWhitespace(document.querySelector<HTMLMetaElement>(`meta[${attr}="${value}"]`)?.content ?? "") || undefined;
+}
+
+function htmlFragmentToMarkdown(html: string): string {
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-",
+  });
+  turndown.remove(["script", "style", "noscript"]);
+  const markdown = turndown.turndown(html)
+    .split(/\r?\n/)
+    .map((line) => line.replace(/[ \t]+$/g, ""))
+    .join("\n")
+    .replace(/\n{4,}/g, "\n\n\n")
+    .trim();
+  return normalizeObsidianMarkdown(markdown);
+}
+
+function normalizeObsidianMarkdown(markdown: string): string {
+  return normalizeBlockLinks(normalizeLinkedImages(markdown));
+}
+
+function normalizeLinkedImages(markdown: string): string {
+  // Turndown can emit image-only links as `[![alt](src)](href)` for linked
+  // article images. Obsidian renders multiline variants as stray `[` / `](href)`
+  // text around the image, so keep the image and drop the redundant outer link.
+  return markdown.replace(/\[\s*(!\[[^\]]*\]\([^)]+\))\s*\]\([^)]+\)/g, "$1");
+}
+
+function normalizeBlockLinks(markdown: string): string {
+  const lines = markdown.split("\n");
+  const output: string[] = [];
+
+  for (let i = 0; i < lines.length; i += 1) {
+    if (lines[i]?.trim() !== "[") {
+      output.push(lines[i] ?? "");
+      continue;
+    }
+
+    const closeIndex = lines.findIndex((line, index) => index > i && /^\]\(([^)]+)\)$/.test(line.trim()));
+    if (closeIndex === -1) {
+      output.push(lines[i] ?? "");
+      continue;
+    }
+
+    const href = lines[closeIndex]?.trim().match(/^\]\(([^)]+)\)$/)?.[1] ?? "";
+    const inner = trimBlankLines(lines.slice(i + 1, closeIndex));
+    output.push(...inner);
+    if (!isImageOnlyBlock(inner) && href) {
+      if (inner.length > 0) output.push("");
+      output.push(`[Link](${href})`);
+    }
+    i = closeIndex;
+  }
+
+  return output.join("\n").replace(/\n{4,}/g, "\n\n\n").trim();
+}
+
+function trimBlankLines(lines: string[]): string[] {
+  let start = 0;
+  let end = lines.length;
+  while (start < end && lines[start]?.trim() === "") start += 1;
+  while (end > start && lines[end - 1]?.trim() === "") end -= 1;
+  return lines.slice(start, end);
+}
+
+function isImageOnlyBlock(lines: string[]): boolean {
+  const nonBlank = lines.map((line) => line.trim()).filter(Boolean);
+  return nonBlank.length === 1 && /^!\[[^\]]*\]\([^)]+\)$/.test(nonBlank[0] ?? "");
+}
+
+function isPdf(contentType: string, url: string, kind: CaptureCandidateKind): boolean {
+  return kind === "pdf" || /application\/pdf/i.test(contentType) || new URL(url).pathname.toLowerCase().endsWith(".pdf");
+}
+
+function isHtml(raw: string, contentType: string, kind: CaptureCandidateKind): boolean {
+  return kind === "html"
+    || /text\/html|application\/xhtml\+xml/i.test(contentType)
+    || /^\s*<!doctype html/i.test(raw)
+    || /^\s*<html[\s>]/i.test(raw);
+}
+
+function isMarkdown(contentType: string, url: string, kind: CaptureCandidateKind): boolean {
+  const path = new URL(url).pathname.toLowerCase();
+  return kind === "markdown"
+    || /text\/(markdown|x-markdown)|application\/markdown/i.test(contentType)
+    || path.endsWith(".md")
+    || path.endsWith(".markdown")
+    || path.endsWith(".mdx");
+}
+
+function titleFromUrl(url: string): string {
+  const parsed = new URL(url);
+  const last = parsed.pathname.split("/").filter(Boolean).pop();
+  return normalizeWhitespace(decodeURIComponent(last || parsed.hostname).replace(/[-_]+/g, " ")) || parsed.hostname;
+}
+
+function stripMarkdown(value: string): string {
+  return value.replace(/^[#>*\-\s]+/, "").replace(/[`*_~\[\]()]/g, "");
+}
+
+function stringValue(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? normalizeWhitespace(value) : undefined;
+}
+
+function normalizeWhitespace(value: string): string {
+  return value.replace(/\s+/g, " ").trim();
+}

package/src/vault/fetch.ts

@@ -0,0 +1,155 @@
+/** Robust bounded fetch helpers for vault source capture. */
+
+import type { CaptureCandidate } from "./url-resolvers.js";
+
+export const MAX_CAPTURE_BYTES = 5 * 1024 * 1024;
+export const DEFAULT_TIMEOUT_MS = 15_000;
+const DEFAULT_RETRIES = 2;
+
+export interface CaptureFetchOptions {
+  timeoutMs?: number;
+  signal?: AbortSignal;
+  maxBytes?: number;
+  retries?: number;
+}
+
+export interface CaptureFetchAttempt {
+  url: string;
+  final_url?: string;
+  status?: number;
+  status_text?: string;
+  content_type?: string;
+  bytes?: number;
+  strategy: string;
+  error?: string;
+}
+
+export interface FetchedCandidate {
+  candidate: CaptureCandidate;
+  finalUrl: string;
+  status: number;
+  statusText: string;
+  contentType: string;
+  headers: Record<string, string>;
+  raw: string;
+  bytes: number;
+  attempts: CaptureFetchAttempt[];
+}
+
+export async function fetchCandidate(candidate: CaptureCandidate, options: CaptureFetchOptions = {}): Promise<FetchedCandidate> {
+  const attempts: CaptureFetchAttempt[] = [];
+  const retries = options.retries ?? DEFAULT_RETRIES;
+  let lastError: unknown;
+
+  for (let attemptNumber = 0; attemptNumber <= retries; attemptNumber += 1) {
+    try {
+      const fetched = await fetchOnce(candidate, options);
+      attempts.push(...fetched.attempts);
+      return { ...fetched, attempts };
+    } catch (error) {
+      lastError = error;
+      const attemptFromError = (error as Error & { attempt?: CaptureFetchAttempt }).attempt;
+      attempts.push({
+        ...(attemptFromError ?? { url: candidate.url, strategy: candidate.strategy }),
+        error: error instanceof Error ? error.message : String(error),
+      });
+
+      if (attemptNumber >= retries || !isRetryableError(error)) break;
+      await sleep(150 * 2 ** attemptNumber);
+    }
+  }
+
+  const message = lastError instanceof Error ? lastError.message : String(lastError);
+  const error = new Error(message || `Failed to fetch ${candidate.url}`) as Error & { attempts?: CaptureFetchAttempt[] };
+  error.attempts = attempts;
+  throw error;
+}
+
+async function fetchOnce(candidate: CaptureCandidate, options: CaptureFetchOptions): Promise<FetchedCandidate> {
+  const maxBytes = options.maxBytes ?? MAX_CAPTURE_BYTES;
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+  const abortFromParent = () => controller.abort();
+  if (options.signal) {
+    if (options.signal.aborted) controller.abort();
+    options.signal.addEventListener("abort", abortFromParent, { once: true });
+  }
+
+  try {
+    const response = await fetch(candidate.url, {
+      redirect: "follow",
+      signal: controller.signal,
+      headers: {
+        "user-agent": "Mozilla/5.0 (compatible; pi-memory-stone/0.1; +https://github.com/nikolasp/pi-memory-stone)",
+        "accept": "text/html, text/markdown, text/plain, application/xhtml+xml, application/pdf, */*;q=0.5",
+      },
+    });
+
+    const contentType = response.headers.get("content-type") ?? "text/plain";
+    const attempt: CaptureFetchAttempt = {
+      url: candidate.url,
+      final_url: response.url,
+      status: response.status,
+      status_text: response.statusText,
+      content_type: contentType,
+      strategy: candidate.strategy,
+    };
+
+    if (!response.ok) {
+      throw httpError(response, attempt);
+    }
+
+    const contentLength = Number(response.headers.get("content-length") ?? "0");
+    if (contentLength > maxBytes) {
+      throw new Error(`Response is too large (${contentLength} bytes; max ${maxBytes})`);
+    }
+
+    const buffer = await response.arrayBuffer();
+    attempt.bytes = buffer.byteLength;
+    if (buffer.byteLength > maxBytes) {
+      throw new Error(`Response is too large (${buffer.byteLength} bytes; max ${maxBytes})`);
+    }
+
+    const raw = new TextDecoder(detectCharset(contentType)).decode(buffer);
+    const headers = Object.fromEntries(response.headers.entries());
+
+    return {
+      candidate,
+      finalUrl: response.url,
+      status: response.status,
+      statusText: response.statusText,
+      contentType,
+      headers,
+      raw,
+      bytes: buffer.byteLength,
+      attempts: [attempt],
+    };
+  } finally {
+    clearTimeout(timeout);
+    options.signal?.removeEventListener("abort", abortFromParent);
+  }
+}
+
+function httpError(response: Response, attempt: CaptureFetchAttempt): Error & { retryable?: boolean; attempt?: CaptureFetchAttempt } {
+  const error = new Error(`HTTP ${response.status} ${response.statusText}`.trim()) as Error & { retryable?: boolean; attempt?: CaptureFetchAttempt };
+  error.retryable = response.status === 429 || response.status >= 500;
+  error.attempt = attempt;
+  return error;
+}
+
+function isRetryableError(error: unknown): boolean {
+  if (error && typeof error === "object" && "retryable" in error) {
+    return Boolean((error as { retryable?: boolean }).retryable);
+  }
+  const message = error instanceof Error ? error.message : String(error);
+  return /fetch failed|network|ECONNRESET|ETIMEDOUT|aborted/i.test(message);
+}
+
+function detectCharset(contentType: string): string {
+  const charset = contentType.match(/charset=([^;]+)/i)?.[1]?.trim();
+  return charset || "utf-8";
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}