pi-read-page 0.1.0

package/src/browser/confidence.ts

@@ -0,0 +1,167 @@
+import type {
+  ConfidenceReport,
+  ExtractedPage,
+  HandoffReason,
+  UserActionDecision,
+} from "../types";
+
+const CAPTCHA_RE =
+  /captcha|recaptcha|hcaptcha|verify you are human|human verification|are you human|请完成安全验证|人机验证/i;
+const BLOCK_RE =
+  /cloudflare|attention required|checking your browser|just a moment|access denied|temporarily blocked|访问受限|安全检查/i;
+const LOGIN_WALL_RE =
+  /sign in to continue|log in to continue|login to continue|please sign in|please log in|subscribe to continue|sign up to continue|join to view|登录后(查看|继续|阅读)|请登录(后)?(查看|继续|阅读)?/i;
+const LOGIN_URL_RE = /\/(login|signin|sign-in|auth|oauth|session)(\/|$|[?#])/i;
+const NAV_ONLY_RE =
+  /home\s+about\s+contact|privacy policy|terms of service|all rights reserved/i;
+
+export function assessConfidence(
+  page: Omit<ExtractedPage, "confidence">,
+): ConfidenceReport {
+  const markdown = page.markdown.trim();
+  const contentHtml = page.contentHtml.trim();
+  const title = page.title.trim();
+  const sample =
+    `${page.url}\n${title}\n${markdown.slice(0, 10_000)}`.toLowerCase();
+  const reasons: string[] = [];
+
+  const wordCount = page.metadata.wordCount || estimateWordCount(markdown);
+  const markdownLength = markdown.length;
+  let score = 100;
+
+  if (CAPTCHA_RE.test(sample)) {
+    score -= 70;
+    reasons.push("captcha_or_human_verification_detected");
+  }
+
+  if (BLOCK_RE.test(sample)) {
+    score -= 70;
+    reasons.push("anti_bot_or_access_block_detected");
+  }
+
+  if (isLoginWall(page, markdownLength, wordCount)) {
+    score -= 65;
+    reasons.push("login_wall_detected");
+  }
+
+  if (!title) {
+    score -= 15;
+    reasons.push("missing_title");
+  }
+
+  if (!contentHtml) {
+    score -= 45;
+    reasons.push("missing_extracted_html");
+  }
+
+  if (markdownLength < 100) {
+    score -= 60;
+    reasons.push("markdown_too_short");
+  } else if (markdownLength < 500) {
+    score -= 25;
+    reasons.push("markdown_short");
+  } else if (markdownLength < 1_500) {
+    score -= 12;
+    reasons.push("markdown_somewhat_short");
+  }
+
+  if (wordCount < 10) {
+    score -= 45;
+    reasons.push("word_count_too_low");
+  } else if (wordCount < 30) {
+    score -= 15;
+    reasons.push("word_count_low");
+  } else if (wordCount < 200) {
+    score -= 8;
+    reasons.push("word_count_somewhat_low");
+  }
+
+  if (NAV_ONLY_RE.test(sample) && wordCount < 250) {
+    score -= 30;
+    reasons.push("looks_like_navigation_or_boilerplate");
+  }
+
+  if (
+    !page.metadata.description &&
+    !page.metadata.published &&
+    !page.metadata.author &&
+    !page.metadata.site
+  ) {
+    score -= 5;
+    reasons.push("little_metadata");
+  }
+
+  const boundedScore = Math.max(0, score);
+  const level =
+    boundedScore >= 70 ? "high" : boundedScore >= 40 ? "medium" : "low";
+  return { level, score: boundedScore, reasons };
+}
+
+export function decideUserAction(extracted: ExtractedPage): UserActionDecision {
+  const handoff = detectActionableHandoff(extracted);
+  return {
+    required: handoff !== undefined,
+    reason: handoff?.reason,
+    message: handoff?.message,
+    confidence: extracted.confidence,
+  };
+}
+
+function detectActionableHandoff(
+  page: ExtractedPage,
+): { reason: HandoffReason; message: string } | undefined {
+  const markdown = page.markdown.trim();
+  const sample =
+    `${page.url}\n${page.title}\n${markdown.slice(0, 10_000)}`.toLowerCase();
+  const wordCount = page.metadata.wordCount || estimateWordCount(markdown);
+  // Captcha/anti-bot/login handoff is only actionable when the extracted body is
+  // thin. A long article that merely discusses these topics is not user-actionable.
+  const contentIsThin = wordCount < 120 || markdown.length < 1_200;
+
+  if (contentIsThin && CAPTCHA_RE.test(sample)) {
+    return {
+      reason: "captcha",
+      message: "The page appears to require captcha or human verification.",
+    };
+  }
+
+  if (contentIsThin && BLOCK_RE.test(sample)) {
+    return {
+      reason: "blocked",
+      message:
+        "The page appears to be blocked by an anti-bot or permission interstitial.",
+    };
+  }
+
+  if (isLoginWall(page, markdown.length, wordCount)) {
+    return {
+      reason: "login_required",
+      message:
+        "The page appears to require login, subscription access, or manual navigation.",
+    };
+  }
+
+  return undefined;
+}
+
+function isLoginWall(
+  page: Pick<ExtractedPage, "url" | "title" | "markdown" | "contentHtml">,
+  markdownLength: number,
+  wordCount: number,
+): boolean {
+  const sample = `${page.url}\n${page.title}\n${page.markdown.slice(0, 10_000)}`;
+  const html = page.contentHtml.toLowerCase();
+  const hasPasswordInput = /<input\b[^>]*type=["']?password/i.test(html);
+  const hasLoginWallText = LOGIN_WALL_RE.test(sample);
+  const loginUrl = LOGIN_URL_RE.test(page.url);
+  const contentIsThin = wordCount < 120 || markdownLength < 1_200;
+
+  return contentIsThin && (hasPasswordInput || hasLoginWallText || loginUrl);
+}
+
+function estimateWordCount(markdown: string): number {
+  const asciiWords =
+    markdown.match(/[A-Za-z0-9_]+(?:[-'][A-Za-z0-9_]+)*/g)?.length || 0;
+  const cjkChars = markdown.match(/[\u3400-\u9fff]/g)?.length || 0;
+  return asciiWords + Math.ceil(cjkChars / 2);
+}

package/src/browser/dom-preparer.ts

@@ -0,0 +1,150 @@
+import { parseHTML } from "linkedom";
+import type { Page } from "playwright-core";
+
+export async function flattenOpenShadowRoots(page: Page): Promise<void> {
+  await page
+    .evaluate(() => {
+      document.querySelectorAll("*").forEach((element) => {
+        element.removeAttribute("data-defuddle-shadow");
+        const shadowRoot = element.shadowRoot;
+        if (shadowRoot?.innerHTML) {
+          element.setAttribute("data-defuddle-shadow", shadowRoot.innerHTML);
+        }
+      });
+    })
+    .catch(() => undefined);
+}
+
+export function prepareHtmlForExtraction(
+  html: string,
+  url: string,
+): {
+  extractionHtml: string;
+  sanitizedHtml: string;
+  fallbackDocument: Document;
+} {
+  const { document: extractionDocument, baseUrl } = parseDocument(html, url);
+  inlineCapturedShadowRoots(extractionDocument);
+  absolutizeUrls(extractionDocument, baseUrl);
+  const extractionHtml = extractionDocument.documentElement?.outerHTML || html;
+
+  const { document: fallbackDocument, baseUrl: fallbackBaseUrl } =
+    parseDocument(extractionHtml, url);
+  sanitizeDocumentForOutput(fallbackDocument);
+  absolutizeUrls(fallbackDocument, fallbackBaseUrl);
+
+  return {
+    extractionHtml,
+    sanitizedHtml:
+      fallbackDocument.documentElement?.outerHTML || extractionHtml,
+    fallbackDocument,
+  };
+}
+
+function inlineCapturedShadowRoots(document: Document): void {
+  document.querySelectorAll("[data-defuddle-shadow]").forEach((host) => {
+    const shadowHtml = host.getAttribute("data-defuddle-shadow");
+    host.removeAttribute("data-defuddle-shadow");
+    if (!shadowHtml) return;
+
+    const { document: shadowDocument } = parseHTML(
+      `<html><body>${shadowHtml}</body></html>`,
+    );
+    const fragment = document.createDocumentFragment();
+    for (const node of Array.from(shadowDocument.body?.childNodes ?? [])) {
+      fragment.appendChild(document.importNode(node, true));
+    }
+
+    if (host.tagName.includes("-") && host.parentNode) {
+      const replacement = document.createElement("div");
+      replacement.appendChild(fragment);
+      host.parentNode.replaceChild(replacement, host);
+      return;
+    }
+
+    host.textContent = "";
+    host.appendChild(fragment);
+  });
+}
+
+function sanitizeDocumentForOutput(document: Document): void {
+  document.querySelectorAll("script, style, noscript").forEach((element) => {
+    element.remove();
+  });
+  document.querySelectorAll("*").forEach((element) => {
+    element.removeAttribute("style");
+  });
+}
+
+function parseDocument(
+  html: string,
+  url: string,
+): { document: Document; baseUrl: string } {
+  const { document } = parseHTML(html);
+  const base = document.querySelector("base[href]");
+  const baseUrl = base?.getAttribute("href")
+    ? new URL(base.getAttribute("href") || url, url).href
+    : url;
+
+  return { document: document as unknown as Document, baseUrl };
+}
+
+function absolutizeUrls(document: Document, baseUrl: string): void {
+  document.querySelectorAll("[src], [href], [srcset]").forEach((element) => {
+    absolutizeAttribute(element, "src", baseUrl);
+    absolutizeAttribute(element, "href", baseUrl);
+    absolutizeSrcset(element, baseUrl);
+  });
+}
+
+function absolutizeAttribute(
+  element: Element,
+  attr: "src" | "href",
+  baseUrl: string,
+): void {
+  const value = element.getAttribute(attr);
+  if (!value) return;
+  if (
+    value.startsWith("http://") ||
+    value.startsWith("https://") ||
+    value.startsWith("data:") ||
+    value.startsWith("#")
+  )
+    return;
+  if (value.startsWith("//")) {
+    const protocol = new URL(baseUrl).protocol;
+    element.setAttribute(attr, `${protocol}${value}`);
+    return;
+  }
+
+  try {
+    element.setAttribute(attr, new URL(value, baseUrl).href);
+  } catch {
+    // Keep the original value if URL parsing fails.
+  }
+}
+
+function absolutizeSrcset(element: Element, baseUrl: string): void {
+  const value = element.getAttribute("srcset");
+  if (!value) return;
+
+  const next = value
+    .split(",")
+    .map((candidate) => {
+      const parts = candidate.trim().split(/\s+/);
+      const src = parts.shift();
+      if (!src) return candidate;
+
+      try {
+        const absolute = src.startsWith("data:")
+          ? src
+          : new URL(src, baseUrl).href;
+        return [absolute, ...parts].join(" ");
+      } catch {
+        return candidate;
+      }
+    })
+    .join(", ");
+
+  element.setAttribute("srcset", next);
+}

package/src/browser/extractor.ts

@@ -0,0 +1,222 @@
+import type { DefuddleOptions, DefuddleResponse } from "defuddle/full";
+import type { Page } from "playwright-core";
+import { createPolicyFetch } from "../security/url-policy";
+import type { ExtractedPage, PageMetadata, UserActionDecision } from "../types";
+import {
+  assessConfidence,
+  decideUserAction as decideUserActionFromConfidence,
+} from "./confidence";
+import {
+  flattenOpenShadowRoots,
+  prepareHtmlForExtraction,
+} from "./dom-preparer";
+
+const DEFAULT_PARSE_TIMEOUT_MS = 8_000;
+
+type ParseMode = ExtractedPage["parseMode"];
+type DefuddleNodeModule = typeof import("defuddle/node");
+
+let defuddleNodeModulePromise: Promise<DefuddleNodeModule> | undefined;
+
+async function getDefuddleNodeModule(): Promise<DefuddleNodeModule> {
+  defuddleNodeModulePromise ??= import("defuddle/node");
+  return defuddleNodeModulePromise;
+}
+
+export async function extractMarkdown(page: Page): Promise<ExtractedPage> {
+  await flattenOpenShadowRoots(page);
+
+  const url = page.url();
+  const rawHtml = await page.content();
+  const { fallbackDocument, extractionHtml, sanitizedHtml } =
+    prepareHtmlForExtraction(rawHtml, url);
+  const { result, parseMode } = await parseWithDefuddle(extractionHtml, url);
+
+  const contentHtml = result.content || fallbackDocument.body?.innerHTML || "";
+  const markdown = (
+    result.contentMarkdown || structuredTextFallback(contentHtml, url)
+  ).trim();
+  const textLength = cleanText(markdown).length;
+  const metadata = buildMetadata(result);
+  const title = cleanText(result.title || document.title || url);
+
+  const withoutConfidence = {
+    url,
+    title,
+    markdown,
+    contentHtml,
+    fullHtml: sanitizedHtml,
+    textLength,
+    capturedAt: new Date().toISOString(),
+    extractor: "defuddle" as const,
+    extraction: result.extractorType || result.debug?.contentSelector || "auto",
+    parseMode,
+    metadata,
+    warnings: buildWarnings(result, markdown, textLength),
+    debug: result.debug,
+  };
+
+  const confidence = assessConfidence(withoutConfidence);
+
+  return {
+    ...withoutConfidence,
+    confidence,
+  };
+}
+
+export function decideUserAction(extracted: ExtractedPage): UserActionDecision {
+  return decideUserActionFromConfidence(extracted);
+}
+
+async function parseWithDefuddle(
+  html: string,
+  url: string,
+): Promise<{ result: DefuddleResponse; parseMode: ParseMode }> {
+  const allowThirdPartyAsync = process.env.READ_PAGE_DEFUDDLE_ASYNC === "1";
+  const timeoutMs =
+    Number.parseInt(process.env.READ_PAGE_PARSE_TIMEOUT_MS || "", 10) ||
+    DEFAULT_PARSE_TIMEOUT_MS;
+  const options: DefuddleOptions = {
+    url,
+    debug: process.env.READ_PAGE_DEFUDDLE_DEBUG === "1",
+    useAsync: allowThirdPartyAsync,
+    includeReplies: "extractors",
+    separateMarkdown: true,
+    markdown: false,
+    fetch: createPolicyFetch(),
+  };
+
+  const { Defuddle } = await getDefuddleNodeModule();
+
+  try {
+    const result = await withTimeout(Defuddle(html, url, options), timeoutMs);
+    return { result, parseMode: allowThirdPartyAsync ? "async" : "sync" };
+  } catch {
+    const result = await Defuddle(html, url, {
+      ...options,
+      useAsync: false,
+      separateMarkdown: true,
+    });
+    return { result, parseMode: "sync-fallback" };
+  }
+}
+
+function buildMetadata(result: DefuddleResponse): PageMetadata {
+  return {
+    title: cleanText(result.title || ""),
+    author: cleanText(result.author || ""),
+    description: cleanText(result.description || ""),
+    domain: result.domain || "",
+    favicon: result.favicon || "",
+    image: result.image || "",
+    published: result.published || "",
+    site: cleanText(result.site || ""),
+    language: result.language || "",
+    wordCount: result.wordCount || 0,
+    parseTime: result.parseTime || 0,
+    schemaOrgData: result.schemaOrgData,
+    metaTags: result.metaTags || [],
+    variables: result.variables || {},
+  };
+}
+
+function buildWarnings(
+  result: DefuddleResponse,
+  markdown: string,
+  textLength: number,
+): string[] {
+  const warnings: string[] = [];
+
+  if (!result.content)
+    warnings.push(
+      "Defuddle did not return extracted HTML; body fallback may have been used internally.",
+    );
+  if (!result.contentMarkdown)
+    warnings.push(
+      "Defuddle did not return Markdown; used structured plain-text fallback, so some formatting may be lost.",
+    );
+  if (!markdown.trim()) warnings.push("No Markdown content extracted.");
+  if (textLength < 500)
+    warnings.push(
+      "Extracted text is short; the page may require login, captcha, or manual navigation.",
+    );
+  if (!result.title) warnings.push("No title extracted.");
+
+  return warnings;
+}
+
+function structuredTextFallback(html: string, url: string): string {
+  const { fallbackDocument } = prepareHtmlForExtraction(
+    `<body>${html}</body>`,
+    url,
+  );
+  const blocks = Array.from(
+    fallbackDocument.querySelectorAll("h1,h2,h3,h4,h5,h6,p,li,pre,blockquote"),
+  );
+
+  if (blocks.length === 0) {
+    return cleanTextPreservingLines(
+      fallbackDocument.body?.textContent ||
+        fallbackDocument.documentElement?.textContent ||
+        fallbackDocument.textContent ||
+        "",
+    );
+  }
+
+  return blocks
+    .map((element) => formatBlock(element))
+    .filter(Boolean)
+    .join("\n\n")
+    .trim();
+}
+
+function formatBlock(element: Element): string {
+  const tag = element.tagName.toLowerCase();
+  const text = cleanTextPreservingLines(element.textContent || "");
+  if (!text) return "";
+
+  if (/^h[1-6]$/.test(tag))
+    return `${"#".repeat(Number(tag.slice(1)))} ${text}`;
+  if (tag === "li") return `- ${text}`;
+  if (tag === "blockquote")
+    return text
+      .split("\n")
+      .map((line) => `> ${line}`)
+      .join("\n");
+  if (tag === "pre") return `\`\`\`\n${text}\n\`\`\``;
+  return text;
+}
+
+function cleanText(value: string): string {
+  return value.replace(/\s+/g, " ").trim();
+}
+
+function cleanTextPreservingLines(value: string): string {
+  return value
+    .replace(/\r\n/g, "\n")
+    .replace(/[ \t\f\v]+/g, " ")
+    .replace(/\n{3,}/g, "\n\n")
+    .split("\n")
+    .map((line) => line.trim())
+    .join("\n")
+    .trim();
+}
+
+function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const timeout = setTimeout(
+      () => reject(new Error(`Timed out after ${timeoutMs}ms`)),
+      timeoutMs,
+    );
+    promise.then(
+      (value) => {
+        clearTimeout(timeout);
+        resolve(value);
+      },
+      (error) => {
+        clearTimeout(timeout);
+        reject(error);
+      },
+    );
+  });
+}

package/src/browser/user-action.ts

@@ -0,0 +1,43 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { HandoffReason } from "../types";
+
+export async function waitForUserAction(
+  ctx: ExtensionContext,
+  url: string,
+  reason: HandoffReason,
+  message: string,
+  signal?: AbortSignal,
+): Promise<boolean> {
+  if (!ctx.hasUI) {
+    throw new Error(
+      `No interactive UI available for required user action: ${reason}`,
+    );
+  }
+
+  const statusKey = "read-page";
+  ctx.ui.setStatus(statusKey, "Waiting for browser action");
+  ctx.ui.setWidget(statusKey, [
+    "read-page needs user action.",
+    `Reason: ${reason}`,
+    `URL: ${url}`,
+    "Finish the action in the opened browser, then confirm here.",
+  ]);
+
+  try {
+    return await ctx.ui.confirm(
+      "read-page needs user action",
+      [
+        message,
+        "",
+        `URL: ${url}`,
+        "",
+        "Complete login / captcha / manual navigation in the opened browser.",
+        "When the page is ready, return here and confirm.",
+      ].join("\n"),
+      { signal, timeout: 15 * 60 * 1000 },
+    );
+  } finally {
+    ctx.ui.setStatus(statusKey, undefined);
+    ctx.ui.setWidget(statusKey, []);
+  }
+}