webforai 2.1.0 → 3.0.0

package/dist/bin.js

@@ -1,12 +1,12 @@
 #!/usr/bin/env node
 
 // src/cli/bin.ts
-import { program } from "commander";
+import { CommanderError, program } from "commander";
 
 // package.json
 var package_default = {
   name: "webforai",
-  version: "2.1.0",
+  version: "3.0.0",
   description: "A library that provides a web interface for AI",
   author: "inaridiy",
   license: "Apache-2.0",
@@ -52,6 +52,11 @@ var package_default = {
       import: "./dist/index.js",
       require: "./dist/cjs/index.js"
     },
+    "./platform": {
+      types: "./dist/types/platform/index.d.ts",
+      import: "./dist/platform/index.js",
+      require: "./dist/cjs/platform/index.js"
+    },
     "./loaders/playwright": {
       types: "./dist/types/loaders/playwright.d.ts",
       import: "./dist/loaders/playwright.js",
@@ -89,6 +94,9 @@ var package_default = {
       ],
       "loaders/puppeteer": [
         "./dist/types/loaders/puppeteer.d.ts"
+      ],
+      platform: [
+        "./dist/types/platform/index.d.ts"
       ]
     }
   },
@@ -124,8 +132,7 @@ var package_default = {
     "mdast-util-to-markdown": "^2.1.0",
     picocolors: "^1.0.1",
     "trim-trailing-lines": "^2.1.0",
-    "unist-util-filter": "^5.0.1",
-    zx: "^8.1.5"
+    "unist-util-filter": "^5.0.1"
   },
   devDependencies: {
     "@cloudflare/puppeteer": "^0.0.6",
@@ -147,12 +154,520 @@ var package_default = {
   }
 };
 
-// src/cli/commands/webforai/index.ts
-import fs5 from "node:fs/promises";
-import path2 from "node:path";
-import { intro, log as log2, outro, spinner } from "@clack/prompts";
+// src/cli/commands/skill/index.ts
+import { spawn } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+// src/cli/commands/skill/content.ts
+var renderSkillMarkdown = (version) => `---
+name: webforai
+description: Convert web pages or local HTML files to clean, LLM-ready Markdown with the webforai CLI. Use when a task needs page content as Markdown - reading documentation or articles, scraping, summarising URLs, building text corpora - including JavaScript-heavy or bot-protected pages via the hosted platform.
+---
+
+# webforai CLI${version ? ` (v${version})` : ""}
+
+Convert a URL or local HTML file to Markdown. Non-interactive by default: the Markdown goes
+to stdout, every log goes to stderr, so output can be piped or captured directly.
+
+## Basic usage
+
+\`\`\`bash
+npx webforai https://example.com/article            # Markdown to stdout
+npx webforai https://example.com/article -o out.md  # write to a file instead
+npx webforai ./page.html                            # convert a local HTML file
+npx webforai https://example.com --json             # machine-readable envelope (see below)
+\`\`\`
+
+## Flags
+
+- \`-o, --output <path>\` \u2014 write Markdown to a file instead of stdout.
+- \`-l, --loader <fetch|playwright|platform>\` \u2014 how HTML is acquired. Default: \`fetch\` for
+  URLs, automatic \`local\` for file paths. \`playwright\` renders JavaScript locally (needs
+  \`npx playwright install chromium\`). \`platform\` uses the hosted API (below).
+- \`-m, --mode <default|ai>\` \u2014 \`ai\` strips links/tables/images down to plain text
+  (local conversion only).
+- \`--extractor <auto|takumi|minimal|none>\` \u2014 main-content extraction preset
+  (\`none\` converts the whole page).
+- \`--frontmatter\` \u2014 prepend YAML front matter (title, author, canonical URL, ...).
+- \`--json\` \u2014 print a JSON envelope to stdout instead of raw Markdown.
+- \`--engine <fetch|browser|proxy-fetch|proxy-browser>\` \u2014 platform acquisition engine
+  (implies \`-l platform\`). \`browser\`/\`proxy-browser\` render JavaScript; \`proxy-*\` engines
+  egress through rotating proxies for bot-protected sites.
+- \`--region <auto|us|eu|uk|jp|asia>\` \u2014 platform proxy egress region (implies \`-l platform\`).
+- \`--screenshot\` \u2014 platform browser engines only; the envelope carries an expiring
+  \`screenshotUrl\`.
+- \`--api-key <key>\` / \`--platform-url <url>\` \u2014 platform credentials and host; usually set
+  via env vars instead.
+- \`-i, --interactive\` \u2014 guided prompt flow (for humans; do not use from an agent).
+- \`-d, --debug\` \u2014 verbose logs on stderr.
+
+## JSON envelope (\`--json\`)
+
+\`\`\`json
+{
+  "source": "https://example.com",
+  "loader": "fetch",
+  "url": "https://example.com",
+  "markdown": "# ...",
+  "metadata": { "title": "..." },
+  "engine": "browser",
+  "region": "auto",
+  "credits": 5,
+  "screenshotUrl": "https://...",
+  "output": "out.md"
+}
+\`\`\`
+
+\`engine\`/\`region\`/\`credits\`/\`screenshotUrl\` appear only for the platform loader; \`output\`
+only when \`-o\` also wrote a file.
+
+## Hosted platform
+
+For JavaScript-heavy or bot-protected pages, the hosted platform fetches server-side:
+
+\`\`\`bash
+export WEBFORAI_API_KEY=wfa_...       # from https://platform.webforai.dev/dashboard
+npx webforai https://example.com --engine browser --json
+\`\`\`
+
+\`WEBFORAI_PLATFORM_URL\` overrides the host for self-hosted deployments. Requests are metered
+in credits (fetch 1, proxy-fetch 2, browser/proxy-browser 5); failures are never billed.
+
+## Exit codes
+
+- \`0\` \u2014 success.
+- \`1\` \u2014 runtime failure (network, conversion, platform error; message on stderr).
+- \`2\` \u2014 usage error (unknown flag or invalid value; message on stderr).
+
+## Errors worth handling
+
+- \`payment_required\` on stderr \u2192 platform credits exhausted.
+- \`rate_limited\` \u2192 back off (a \`retryAfter\` seconds hint is included when known).
+- Playwright loader errors explain the exact \`npx playwright install\` command to run.
+`;
+
+// src/cli/commands/skill/index.ts
+var buildSkillsAddArgs = (source, flags) => {
+  const args = ["-y", "skills", "add", source];
+  if (flags.global) {
+    args.push("--global");
+  }
+  if (flags.agent) {
+    args.push("--agent", flags.agent);
+  }
+  if (flags.yes) {
+    args.push("-y");
+  }
+  if (flags.copy) {
+    args.push("--copy");
+  }
+  if (flags.all) {
+    args.push("--all");
+  }
+  return args;
+};
+var runSkillsAdd = (args) => new Promise((resolve, reject) => {
+  const child = spawn("npx", args, { stdio: ["inherit", "inherit", "inherit"] });
+  child.on("error", reject);
+  child.on("close", (code) => resolve(code ?? 1));
+});
+var skillCommand = async (flags) => {
+  const markdown = renderSkillMarkdown(package_default.version);
+  if (!(flags.install || flags.dir)) {
+    process.stdout.write(markdown);
+    return;
+  }
+  if (flags.dir) {
+    const target = path.join(flags.dir, "webforai", "SKILL.md");
+    await fs.mkdir(path.dirname(target), { recursive: true });
+    await fs.writeFile(target, markdown);
+    console.error(`[webforai] Agent Skill written to ${target}`);
+    return;
+  }
+  const stage = await fs.mkdtemp(path.join(os.tmpdir(), "webforai-skill-"));
+  try {
+    await fs.mkdir(path.join(stage, "webforai"), { recursive: true });
+    await fs.writeFile(path.join(stage, "webforai", "SKILL.md"), markdown);
+    const args = buildSkillsAddArgs(stage, flags);
+    console.error(`[webforai] running: npx ${args.join(" ")}`);
+    const code = await runSkillsAdd(args).catch((error) => {
+      console.error(
+        `[webforai] could not launch the skills CLI (${error instanceof Error ? error.message : String(error)}).`
+      );
+      console.error("[webforai] fallback: `webforai skill --dir .claude/skills` writes the file directly.");
+      return 1;
+    });
+    if (code !== 0) {
+      process.exitCode = 1;
+    }
+  } finally {
+    await fs.rm(stage, { recursive: true, force: true });
+  }
+};
+
+// src/cli/commands/webforai/interactive.ts
+import { intro, log, outro, select as select7, spinner, text as text5 } from "@clack/prompts";
 import pc2 from "picocolors";
 
+// src/platform/error.ts
+var PlatformApiError = class extends Error {
+  constructor(code, message, status, retryAfter) {
+    super(message);
+    this.name = "PlatformApiError";
+    this.code = code;
+    this.status = status;
+    this.retryAfter = retryAfter;
+  }
+};
+
+// src/platform/fetch.ts
+var resolveFetch = (custom) => {
+  if (custom) {
+    return custom;
+  }
+  const globalFetch = globalThis.fetch;
+  if (typeof globalFetch !== "function") {
+    throw new Error("No global fetch in this runtime \u2014 pass a fetch implementation to createPlatformClient({ fetch })");
+  }
+  return globalFetch.bind(globalThis);
+};
+
+// src/platform/types.ts
+var ENGINES = ["fetch", "browser", "proxy-fetch", "proxy-browser"];
+var REGIONS = ["auto", "us", "eu", "uk", "jp", "asia"];
+var isStoredPageStub = (item) => item.status === "ok" && "resultUrl" in item;
+
+// src/platform/client.ts
+var DEFAULT_BASE_URL = "https://platform.webforai.dev";
+var TERMINAL_STATES = /* @__PURE__ */ new Set(["completed", "failed"]);
+var sleep = (ms, signal) => new Promise((resolve, reject) => {
+  if (signal?.aborted) {
+    reject(signal.reason instanceof Error ? signal.reason : new Error("aborted"));
+    return;
+  }
+  const timer = setTimeout(() => {
+    signal?.removeEventListener("abort", onAbort);
+    resolve();
+  }, ms);
+  const onAbort = () => {
+    clearTimeout(timer);
+    reject(signal?.reason instanceof Error ? signal.reason : new Error("aborted"));
+  };
+  signal?.addEventListener("abort", onAbort, { once: true });
+});
+var parseRetryAfter = (body, response) => {
+  const fromBody = body?.error?.retryAfter;
+  if (typeof fromBody === "number") {
+    return fromBody;
+  }
+  const header = Number(response.headers.get("retry-after"));
+  return Number.isFinite(header) && header > 0 ? header : void 0;
+};
+var toApiError = (response, body) => {
+  const envelope = body;
+  const code = typeof envelope?.error?.code === "string" ? envelope.error.code : "invalid_response";
+  const message = typeof envelope?.error?.message === "string" ? envelope.error.message : `unexpected response (HTTP ${response.status})`;
+  return new PlatformApiError(code, message, response.status, parseRetryAfter(body, response));
+};
+var createPlatformClient = (options = {}) => {
+  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  const fetchImpl = resolveFetch(options.fetch);
+  const request = async (path4, init) => {
+    const headers = { accept: "application/json" };
+    if (init.body !== void 0) {
+      headers["content-type"] = "application/json";
+    }
+    if (init.auth) {
+      if (!options.apiKey) {
+        throw new PlatformApiError(
+          "missing_api_key",
+          "This endpoint needs an API key: pass `apiKey` to createPlatformClient (create one on the platform dashboard).",
+          0
+        );
+      }
+      headers.authorization = `Bearer ${options.apiKey}`;
+    }
+    const response = await fetchImpl(`${baseUrl}${path4}`, {
+      method: init.method,
+      headers,
+      body: init.body === void 0 ? void 0 : JSON.stringify(init.body)
+    });
+    const body = await response.json().catch(() => void 0);
+    if (!response.ok) {
+      throw toApiError(response, body);
+    }
+    if (body === void 0) {
+      throw new PlatformApiError("invalid_response", "response body is not JSON", response.status);
+    }
+    return body;
+  };
+  const getJob = (jobId) => request(`/v1/jobs/${encodeURIComponent(jobId)}`, { method: "GET", auth: true });
+  const getJobResults = (jobId, opts) => {
+    const query = opts?.cursor ? `?cursor=${encodeURIComponent(opts.cursor)}` : "";
+    return request(`/v1/jobs/${encodeURIComponent(jobId)}/results${query}`, { method: "GET", auth: true });
+  };
+  const resolveStored = async (resultUrl) => {
+    const response = await fetchImpl(resultUrl, { method: "GET" });
+    if (!response.ok) {
+      throw new PlatformApiError(
+        "artifact_fetch_failed",
+        `stored result fetch failed (HTTP ${response.status}) \u2014 result URLs expire after ~24h`,
+        response.status
+      );
+    }
+    return await response.json();
+  };
+  return {
+    scrape: (opts) => request("/v1/scrape", { method: "POST", body: opts, auth: true }),
+    scrapeAsync: (opts) => request("/v1/scrape", { method: "POST", body: { ...opts, async: true }, auth: true }),
+    batch: (opts) => request("/v1/batch", { method: "POST", body: opts, auth: true }),
+    crawl: (opts) => request("/v1/crawl", { method: "POST", body: opts, auth: true }),
+    getJob,
+    getJobResults,
+    waitForJob: async (jobId, opts = {}) => {
+      const interval = opts.pollIntervalMs ?? 2e3;
+      const timeout = opts.timeoutMs ?? 10 * 60 * 1e3;
+      const startedAt = Date.now();
+      for (; ; ) {
+        const status = await getJob(jobId);
+        opts.onStatus?.(status);
+        if (TERMINAL_STATES.has(status.status)) {
+          return status;
+        }
+        if (Date.now() - startedAt + interval > timeout) {
+          throw new PlatformApiError("poll_timeout", `job ${jobId} still ${status.status} after ${timeout}ms`, 0);
+        }
+        await sleep(interval, opts.signal);
+      }
+    },
+    jobResults: async function* (jobId) {
+      let cursor;
+      do {
+        const page = await getJobResults(jobId, { cursor });
+        for (const item of page.results) {
+          yield isStoredPageStub(item) ? await resolveStored(item.resultUrl) : item;
+        }
+        cursor = page.cursor;
+      } while (cursor);
+    },
+    demoScrape: (opts) => request("/v1/demo/scrape", { method: "POST", body: opts, auth: false })
+  };
+};
+
+// src/cli/constants.ts
+var DEFAULT_PATH = "https://example.com";
+var LOADERS = ["fetch", "playwright", "platform"];
+var MODES = ["default", "ai"];
+var EXTRACTORS = ["auto", "takumi", "minimal", "none"];
+var API_KEY_ENV = "WEBFORAI_API_KEY";
+var PLATFORM_URL_ENV = "WEBFORAI_PLATFORM_URL";
+
+// src/cli/helpers/assertContinue.ts
+import { cancel, isCancel } from "@clack/prompts";
+function assertContinue(message, cancelMessage = "Canceled.") {
+  if (isCancel(message)) {
+    cancel(cancelMessage);
+    process.exit(1);
+  }
+}
+
+// src/cli/helpers/inputOutputPath.ts
+import fs3 from "node:fs";
+import { confirm, text } from "@clack/prompts";
+
+// src/cli/utils.ts
+import fs2 from "node:fs";
+import path2 from "node:path";
+var isUrl = (maybeUrl) => {
+  try {
+    new URL(maybeUrl);
+    return true;
+  } catch {
+    return false;
+  }
+};
+function changeFileExtension(filePath, newExtension) {
+  const parsedPath = filePath.split("/");
+  const fileName = parsedPath[parsedPath.length - 1];
+  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  if (fileName.startsWith(".")) {
+    const parts = fileName.split(".");
+    if (parts.length === 2) {
+      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
+    }
+    parts[parts.length - 1] = newExtension.replace(/^\./, "");
+    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+  }
+  const lastDotIndex = fileName.lastIndexOf(".");
+  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
+  const newFileName = `${baseName}${formattedNewExtension}`;
+  parsedPath[parsedPath.length - 1] = newFileName;
+  return parsedPath.join("/");
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
+      if (index === 0) {
+        return acc;
+      }
+      if (acc.length >= 2) {
+        return acc;
+      }
+      if (part === "www") {
+        return acc;
+      }
+      return [part, ...acc];
+    }, []);
+    const domainString = domainParts.reverse().join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2);
+    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
+    let filename = [domainString, pathString].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+var sourcePathToOutputPath = (sourcePath) => {
+  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
+};
+function getNextAvailableFilePath(filePath) {
+  const parsedPath = path2.parse(filePath);
+  const directory = parsedPath.dir;
+  const fullName = parsedPath.base;
+  const [firstPart, ...restParts] = fullName.split(".");
+  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
+  const baseName = firstPart.replace(/_\d+$/, "");
+  let counter = 1;
+  let nextFilePath = filePath;
+  while (fs2.existsSync(nextFilePath)) {
+    const match = firstPart.match(/_(\d+)$/);
+    if (match) {
+      counter = Number.parseInt(match[1], 10) + 1;
+    }
+    const newName = `${baseName}_${counter}${restName}`;
+    nextFilePath = path2.join(directory, newName);
+    counter++;
+  }
+  return nextFilePath;
+}
+
+// src/cli/helpers/inputOutputPath.ts
+var inputOutputPath = async (sourcePath) => {
+  const outputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: sourcePathToOutputPath(sourcePath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
+      }
+      if (fs3.existsSync(value) && fs3.statSync(value).isDirectory()) {
+        return "No directory can be specified.";
+      }
+      if (!fs3.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
+      }
+    }
+  });
+  assertContinue(outputPath);
+  if (!fs3.existsSync(outputPath)) {
+    return outputPath;
+  }
+  const isOutputFileOverwrite = await confirm({
+    message: "The file already exists. Overwrite?",
+    initialValue: false
+  });
+  assertContinue(isOutputFileOverwrite);
+  if (isOutputFileOverwrite) {
+    return outputPath;
+  }
+  const escapedOutputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: getNextAvailableFilePath(outputPath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
+      }
+      if (fs3.existsSync(value)) {
+        return "The file already exists";
+      }
+      if (!fs3.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
+      }
+    }
+  });
+  assertContinue(escapedOutputPath);
+  return escapedOutputPath;
+};
+
+// src/cli/helpers/inputSourcePath.ts
+import fs4 from "node:fs";
+import { text as text2 } from "@clack/prompts";
+var inputSourcePath = async () => {
+  const result = await text2({
+    message: "Enter the URL or html path to be converted to markdown:",
+    placeholder: DEFAULT_PATH,
+    initialValue: "",
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Source is required";
+      }
+      if (!isUrl(value)) {
+        if (!fs4.existsSync(value)) {
+          return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
+        }
+        if (fs4.statSync(value).isDirectory()) {
+          return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
+        }
+      }
+    }
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/helpers/selectExtractMode.ts
+import { select } from "@clack/prompts";
+var selectExtractMode = async () => {
+  const result = await select({
+    message: "Select processing mode:",
+    options: MODES.map((mode) => ({ value: mode, label: mode })),
+    initialValue: MODES[0]
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/helpers/selectLoader.ts
+import { select as select2 } from "@clack/prompts";
+var loadersHint = {
+  fetch: "Fetch HTML content from the given URL",
+  playwright: "Render the page in local headless Chromium first; Playwright must be installed",
+  platform: "Convert via the hosted webforai platform API (needs an API key)"
+};
+var selectLoader = async () => {
+  const result = await select2({
+    message: "Select loader:",
+    initialValue: "fetch",
+    options: LOADERS.map((loader) => ({ value: loader, label: loader, hint: loadersHint[loader] }))
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/commands/webforai/run.ts
+import fs6 from "node:fs/promises";
+import path3 from "node:path";
+
+// src/html-to-markdown.ts
+import { fromHtml as fromHtml2 } from "hast-util-from-html";
+
 // src/html-to-mdast.ts
 import { fromHtml } from "hast-util-from-html";
 import { toMdast } from "hast-util-to-mdast";
@@ -179,192 +694,1941 @@ var extractMdast = (node) => {
   return extracted;
 };
 
-// src/extractors/presets/takumi.ts
-import { select, selectAll } from "hast-util-select";
-import { toString as hastToString } from "hast-util-to-string";
-import { filter as filter2 } from "unist-util-filter";
+// src/utils/hast-fast.ts
+var EMPTY_METRICS = { text: 0, link: 0, commas: 0, paragraphs: 0, elements: 0 };
+var COMMA_REGEX = /[,、，،]/g;
+var PARAGRAPH_LIKE = /* @__PURE__ */ new Set(["p", "pre", "li", "blockquote", "figcaption", "dd", "dt"]);
+var NON_PROSE_TAGS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "object"]);
+var isElement = (node) => typeof node === "object" && node !== null && node.type === "element";
+var isParent = (node) => typeof node === "object" && node !== null && Array.isArray(node.children);
+var classList = (element3) => {
+  const className = element3.properties?.className;
+  if (Array.isArray(className)) {
+    return className;
+  }
+  if (typeof className === "string") {
+    return className.split(/\s+/).filter(Boolean);
+  }
+  return [];
+};
+var matchString = (element3) => {
+  const id = element3.properties?.id;
+  const classes = classList(element3);
+  if (!id && classes.length === 0) {
+    return element3.tagName;
+  }
+  return `${element3.tagName} ${typeof id === "string" ? id : ""} ${classes.join(" ")}`;
+};
+var MetricsCollector = class {
+  #cache = /* @__PURE__ */ new WeakMap();
+  metrics(node) {
+    if (node.type === "text") {
+      const value = node.value;
+      if (value.trim().length === 0) {
+        return EMPTY_METRICS;
+      }
+      return {
+        text: value.length,
+        link: 0,
+        commas: (value.match(COMMA_REGEX) ?? []).length,
+        paragraphs: 0,
+        elements: 0
+      };
+    }
+    if (!isParent(node)) {
+      return EMPTY_METRICS;
+    }
+    const cached = this.#cache.get(node);
+    if (cached) {
+      return cached;
+    }
+    if (isElement(node) && NON_PROSE_TAGS.has(node.tagName)) {
+      this.#cache.set(node, EMPTY_METRICS);
+      return EMPTY_METRICS;
+    }
+    let text6 = 0;
+    let link = 0;
+    let commas = 0;
+    let paragraphs = 0;
+    let elements = 0;
+    for (const child of node.children) {
+      const childMetrics = this.metrics(child);
+      text6 += childMetrics.text;
+      link += childMetrics.link;
+      commas += childMetrics.commas;
+      paragraphs += childMetrics.paragraphs;
+      elements += childMetrics.elements;
+      if (isElement(child)) {
+        elements += 1;
+        if (PARAGRAPH_LIKE.has(child.tagName)) {
+          paragraphs += 1;
+        }
+      }
+    }
+    if (isElement(node) && node.tagName === "a") {
+      link = text6;
+    }
+    const metrics = { text: text6, link, commas, paragraphs, elements };
+    this.#cache.set(node, metrics);
+    return metrics;
+  }
+  textLength(node) {
+    return this.metrics(node).text;
+  }
+  /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
+  linkDensity(node) {
+    const { text: text6, link } = this.metrics(node);
+    if (text6 === 0) {
+      return 0;
+    }
+    return Math.min(1, link / text6);
+  }
+};
+var pruneInPlace = (tree, keep) => {
+  if (!isParent(tree)) {
+    return tree;
+  }
+  const stack = [tree];
+  while (stack.length > 0) {
+    const parent = stack.pop();
+    const children = parent.children;
+    let write = 0;
+    for (let read = 0; read < children.length; read++) {
+      const child = children[read];
+      if (!keep(child)) {
+        continue;
+      }
+      children[write++] = child;
+      if (isParent(child)) {
+        stack.push(child);
+      }
+    }
+    children.length = write;
+  }
+  return tree;
+};
+var walk = (tree, visit) => {
+  const stack = [{ node: tree, parent: void 0 }];
+  while (stack.length > 0) {
+    const { node, parent } = stack.pop();
+    if (visit(node, parent) === false) {
+      continue;
+    }
+    if (isParent(node)) {
+      for (let i = node.children.length - 1; i >= 0; i--) {
+        stack.push({ node: node.children[i], parent: node });
+      }
+    }
+  }
+};
+var collectElements = (tree, predicate) => {
+  const found = [];
+  walk(tree, (node) => {
+    if (isElement(node) && predicate(node)) {
+      found.push(node);
+    }
+  });
+  return found;
+};
+var findElement = (tree, predicate) => {
+  let found;
+  walk(tree, (node) => {
+    if (found) {
+      return false;
+    }
+    if (isElement(node) && predicate(node)) {
+      found = node;
+      return false;
+    }
+  });
+  return found;
+};
+var cloneHast = (tree) => {
+  try {
+    return structuredClone(tree);
+  } catch {
+    return manualClone(tree);
+  }
+};
+var manualClone = (node) => {
+  if (Array.isArray(node)) {
+    return node.map(manualClone);
+  }
+  if (typeof node !== "object" || node === null) {
+    return node;
+  }
+  const copy = {};
+  for (const [key, value] of Object.entries(node)) {
+    copy[key] = manualClone(value);
+  }
+  return copy;
+};
+var property = (element3, key) => {
+  const properties = element3.properties;
+  if (!properties) {
+    return void 0;
+  }
+  const direct = properties[key];
+  if (direct !== void 0) {
+    return direct;
+  }
+  if (!(key.includes("-") || key.includes(":"))) {
+    return void 0;
+  }
+  return properties[camelCaseAttribute(key)];
+};
+var camelCaseAttribute = (key) => key.replace(/[-:]([a-z])/g, (_, character) => character.toUpperCase());
+var numericProperty = (element3, key) => {
+  const raw = property(element3, key);
+  if (typeof raw === "number") {
+    return Number.isFinite(raw) ? raw : void 0;
+  }
+  if (typeof raw === "string" && raw.length > 0) {
+    const parsed = Number.parseFloat(raw);
+    return Number.isFinite(parsed) ? parsed : void 0;
+  }
+  return void 0;
+};
+var stringProperty = (element3, key) => {
+  const raw = property(element3, key);
+  if (typeof raw === "string") {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return raw.join(" ");
+  }
+  if (raw === true) {
+    return "";
+  }
+  return void 0;
+};
+var isTruthyAttribute = (element3, key) => {
+  const raw = property(element3, key);
+  if (raw === void 0 || raw === null || raw === false) {
+    return false;
+  }
+  if (raw === true) {
+    return true;
+  }
+  if (typeof raw === "string") {
+    return raw.toLowerCase() !== "false";
+  }
+  return true;
+};
+
+// src/extractors/lib/constants.ts
+var REGEXPS = {
+  /**
+   * Containers that almost never hold the article body.
+   *
+   * Note the words that are deliberately *absent*. `header` and `footer` match the heading
+   * anchors and article furniture that documentation generators emit (`<a class="header">`,
+   * `class="post-header"`), so matching them here deletes the page's headings. The landmark
+   * versions of those elements are handled structurally instead.
+   */
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumb|combx|comment(?!ary)|community|cover-wrap|disqus|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|site-index|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar|subscribe|newsletter|paywall|consent|cookie|toolbar|masthead|promo|share|sharing|widget|tooltip|dropdown|modal|overlay|backdrop|carousel|announcement|notification|toast|skip-link|screen-reader|visually-hidden|sr-only/i,
+  /** Rescues containers whose class merely happens to contain an unlikely substring. */
+  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code|post|entry|story|markdown|prose|doc/i,
+  /** Very high-confidence boilerplate, not rescued by the clause above. */
+  stronglyUnlikely: /^(nav|navbar|breadcrumbs?|cookie-?(banner|consent|notice)|consent-?(banner|manager)|skip-?link|site-?header|site-?footer|global-?(header|footer)|social-?(share|links?)|share-?(bar|buttons?)|related-?(posts?|articles?)|newsletter-?(signup|form)|comment-?(form|list)|toc-?sidebar)$/i,
+  /** Class/id fragments that mark real article content. */
+  positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story|markdown|prose|documentation|doc-?content|readme/i,
+  /** Class/id fragments that mark furniture. */
+  negative: /-ad-|hidden|^hid$| hid$|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget|toolbar|nav|menu|cookie|consent|subscribe|newsletter|popup|modal|overlay|breadcrumb|pagination|social/i,
+  /** Placeholder imagery that should never reach the output. */
+  hidden: /fallback-image/i,
+  byline: /byline|author|dateline|writtenby|p-author|c-author|post-meta|entry-meta/i,
+  /** Wiki/CMS chrome that generic scoring alone tends to keep. */
+  specialUnlikelyCandidates: /frb-|uls-menu|language-link|mw-editsection|noprint|navbox|metadata|catlinks|printfooter/i
+};
+var NON_CONTENT_TAGS = /* @__PURE__ */ new Set([
+  "script",
+  "style",
+  "link",
+  "meta",
+  "noscript",
+  "template",
+  "base",
+  "param",
+  "source",
+  "track",
+  "title"
+]);
+var CHROME_TAGS = /* @__PURE__ */ new Set(["nav", "aside", "dialog", "menu"]);
+var CONTEXTUAL_LANDMARK_TAGS = /* @__PURE__ */ new Set(["header", "footer"]);
+var HIDDEN_CLASS_NAMES = /* @__PURE__ */ new Set([
+  "hidden",
+  "invisible",
+  "is-hidden",
+  "d-none",
+  "display-none",
+  "hide",
+  "is-invisible"
+]);
+var RESPONSIVE_DISPLAY_OVERRIDE = /^(sm|md|lg|xl|2xl|print):(block|flex|grid|inline|inline-block|inline-flex|table|contents|list-item)$/;
+var UNLIKELY_ROLES = /* @__PURE__ */ new Set([
+  "menu",
+  "menubar",
+  "menuitem",
+  "complementary",
+  "navigation",
+  "alert",
+  "alertdialog",
+  "dialog",
+  "banner",
+  "contentinfo",
+  "search",
+  "searchbox",
+  "toolbar",
+  "tablist",
+  "tooltip"
+  // `presentation` is deliberately absent. It marks an element as decorative *structure* — a
+  // layout table, a wrapper — while the content inside it is usually real. Treating it as
+  // chrome deletes article text.
+]);
+var CONTENT_ROLES = /* @__PURE__ */ new Set(["main", "article", "document"]);
+var SCOREABLE_TAGS = /* @__PURE__ */ new Set(["p", "td", "pre", "blockquote", "dd", "li", "figure", "section", "div"]);
+var CANDIDATE_TAGS = /* @__PURE__ */ new Set([
+  "div",
+  "section",
+  "article",
+  "main",
+  "td",
+  "pre",
+  "blockquote",
+  "form",
+  "ul",
+  "ol",
+  "dl"
+]);
+var EMBEDDED_CONTENT_TAGS = /* @__PURE__ */ new Set([
+  "img",
+  "picture",
+  "video",
+  "audio",
+  "iframe",
+  "embed",
+  "object",
+  "svg",
+  // Formulae hold no text of their own, so an empty-wrapper rule would otherwise delete them.
+  "math",
+  "mjx-container",
+  "canvas"
+]);
+var DROP_WHEN_EMPTY_TAGS = /* @__PURE__ */ new Set([
+  "p",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "li",
+  "td",
+  "th",
+  "blockquote",
+  "figcaption",
+  "dd",
+  "dt",
+  "a",
+  "span",
+  "div",
+  "section",
+  "article"
+]);
+var MIN_CONTENT_LENGTH = {
+  ja: 140,
+  zh: 140,
+  ko: 160,
+  th: 160,
+  default: 400
+};
+var minContentLength = (lang) => {
+  if (!lang) {
+    return MIN_CONTENT_LENGTH.default;
+  }
+  const primary = lang.toLowerCase().split("-")[0];
+  return MIN_CONTENT_LENGTH[primary] ?? MIN_CONTENT_LENGTH.default;
+};
+
+// src/extractors/lib/ui-chrome.ts
+var UI_PHRASES = /* @__PURE__ */ new Set([
+  // Documentation and article controls
+  "copy",
+  "copy page",
+  "copy link",
+  "copy code",
+  "copied",
+  "edit this page",
+  "edit on github",
+  "on this page",
+  "in this article",
+  "table of contents",
+  "contents",
+  "print",
+  "print this page",
+  "back to top",
+  "scroll to top",
+  "skip to content",
+  "skip to main content",
+  "permalink",
+  "anchor",
+  // Feedback widgets
+  "was this page helpful",
+  "was this helpful",
+  "was this article helpful",
+  "yes no",
+  "feedback",
+  "send feedback",
+  "report an issue",
+  "rate this page",
+  // Sharing and subscription
+  "share",
+  "share this",
+  "share this article",
+  "share on twitter",
+  "share on facebook",
+  "subscribe",
+  "sign up",
+  "sign in",
+  "log in",
+  "login",
+  "newsletter",
+  // Advertising and promotion
+  "advertisement",
+  "advertisements",
+  "sponsored",
+  "sponsored content",
+  "promoted",
+  // Navigation affordances
+  "read more",
+  "show more",
+  "show less",
+  "see more",
+  "load more",
+  "previous",
+  "next",
+  "previous page",
+  "next page",
+  "menu",
+  "close",
+  "search",
+  // Japanese equivalents
+  "\u76EE\u6B21",
+  "\u5E83\u544A",
+  "\u30B7\u30A7\u30A2",
+  "\u5171\u6709",
+  "\u3053\u306E\u8A18\u4E8B\u3092\u30B7\u30A7\u30A2",
+  "\u7D9A\u304D\u3092\u8AAD\u3080",
+  "\u3082\u3063\u3068\u898B\u308B",
+  "\u95A2\u9023\u8A18\u4E8B",
+  "\u30B9\u30DD\u30F3\u30B5\u30FC\u30EA\u30F3\u30AF",
+  "\u30DA\u30FC\u30B8\u306E\u5148\u982D\u3078",
+  "\u30C8\u30C3\u30D7\u3078\u623B\u308B",
+  "\u524D\u3078",
+  "\u6B21\u3078",
+  "\u9589\u3058\u308B",
+  "\u691C\u7D22",
+  "\u30ED\u30B0\u30A4\u30F3",
+  "\u4F1A\u54E1\u767B\u9332"
+]);
+var MAX_PHRASE_LENGTH = 40;
+var normalize = (text6) => text6.toLowerCase().replace(/[?!.,:;·・…"'“”‘’()[\]{}<>|/\\—–-]/g, " ").replace(/\s+/g, " ").trim();
+var isUiChromeText = (text6) => {
+  if (text6.length === 0 || text6.length > MAX_PHRASE_LENGTH) {
+    return false;
+  }
+  const normalized = normalize(text6);
+  if (normalized.length === 0) {
+    return false;
+  }
+  return UI_PHRASES.has(normalized);
+};
+
+// src/extractors/lib/sanitize.ts
+var INVISIBLE_STYLE = /(^|;)\s*(display\s*:\s*none|visibility\s*:\s*hidden)\s*(;|$)/i;
+var hasHiddenClass = (element3) => {
+  const classes = classList(element3);
+  if (!classes.some((name) => HIDDEN_CLASS_NAMES.has(name))) {
+    return classes.some((name) => REGEXPS.hidden.test(name));
+  }
+  return !classes.some((name) => RESPONSIVE_DISPLAY_OVERRIDE.test(name));
+};
+var isNonContent = (node) => {
+  if (node.type === "comment" || node.type === "doctype") {
+    return true;
+  }
+  return isElement(node) && NON_CONTENT_TAGS.has(node.tagName);
+};
+var isHidden = (element3) => {
+  if (containsMath(element3)) {
+    return false;
+  }
+  if (isTruthyAttribute(element3, "hidden")) {
+    return true;
+  }
+  const style = stringProperty(element3, "style");
+  if (style && INVISIBLE_STYLE.test(style)) {
+    return true;
+  }
+  if (hasHiddenClass(element3)) {
+    return true;
+  }
+  const width = numericProperty(element3, "data-rwidth");
+  const height = numericProperty(element3, "data-rheight");
+  if (width === 0 && height === 0) {
+    return true;
+  }
+  return false;
+};
+var PLACEHOLDER_IMAGE_SRC = /^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
+var isPlaceholderImage = (element3) => {
+  if (element3.tagName === "img") {
+    const src = stringProperty(element3, "src");
+    return src !== void 0 && PLACEHOLDER_IMAGE_SRC.test(src);
+  }
+  if (element3.tagName !== "picture") {
+    return false;
+  }
+  for (const child of element3.children) {
+    if (isElement(child) && child.tagName === "img") {
+      return isPlaceholderImage(child);
+    }
+  }
+  return false;
+};
+var hoistNoscriptImages = (tree) => {
+  const visit = (parent) => {
+    for (let index = 0; index < parent.children.length; index++) {
+      const child = parent.children[index];
+      if (!isElement(child)) {
+        continue;
+      }
+      if (child.tagName !== "noscript") {
+        visit(child);
+        continue;
+      }
+      const hasImage = child.children.some(
+        (node) => isElement(node) && (node.tagName === "img" || node.tagName === "picture")
+      );
+      if (!hasImage) {
+        continue;
+      }
+      const rescued = child.children;
+      parent.children.splice(index, 1, ...rescued);
+      const previous = parent.children[index - 1];
+      if (index > 0 && isElement(previous) && isPlaceholderImage(previous)) {
+        parent.children.splice(index - 1, 1);
+        index -= 1;
+      }
+      index += rescued.length - 1;
+    }
+  };
+  if ("children" in tree) {
+    visit(tree);
+  }
+};
+var containsMath = (element3) => {
+  if (element3.tagName === "math") {
+    return true;
+  }
+  for (const child of element3.children) {
+    if (isElement(child) && containsMath(child)) {
+      return true;
+    }
+  }
+  return false;
+};
+var stripNonContent = (tree) => {
+  hoistNoscriptImages(tree);
+  return pruneInPlace(tree, (node) => {
+    if (isNonContent(node)) {
+      return false;
+    }
+    if (isElement(node) && isHidden(node)) {
+      return false;
+    }
+    return true;
+  });
+};
+var findLandmarkChrome = (tree) => {
+  const chrome = /* @__PURE__ */ new Set();
+  const visit = (node, depth, insideContent) => {
+    if (!("children" in node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (!isElement(child)) {
+        continue;
+      }
+      const nowInsideContent = insideContent || child.tagName === "article" || child.tagName === "main" || stringProperty(child, "role") === "main";
+      if (CONTEXTUAL_LANDMARK_TAGS.has(child.tagName) && !insideContent && depth <= 3) {
+        chrome.add(child);
+        continue;
+      }
+      visit(child, depth + 1, nowInsideContent);
+    }
+  };
+  visit(tree, 0, false);
+  return chrome;
+};
+var isUnlikelyCandidate = (element3) => {
+  if (element3.tagName === "article" || element3.tagName === "main" || element3.tagName === "body") {
+    return false;
+  }
+  const role = stringProperty(element3, "role");
+  if (role && UNLIKELY_ROLES.has(role)) {
+    return true;
+  }
+  if (CHROME_TAGS.has(element3.tagName)) {
+    return true;
+  }
+  const match = matchString(element3);
+  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
+    return true;
+  }
+  for (const name of [...classList(element3), stringProperty(element3, "id") ?? ""]) {
+    if (REGEXPS.stronglyUnlikely.test(name)) {
+      return true;
+    }
+  }
+  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
+    return true;
+  }
+  return false;
+};
+var findUnlikelyElements = (tree) => {
+  const landmarks = findLandmarkChrome(tree);
+  const doomed = [];
+  const visit = (node) => {
+    if (!("children" in node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (!isElement(child)) {
+        continue;
+      }
+      if (landmarks.has(child) || isUnlikelyCandidate(child)) {
+        doomed.push(child);
+        continue;
+      }
+      visit(child);
+    }
+  };
+  visit(tree);
+  return doomed;
+};
+var isEmbeddedContent = (element3) => {
+  if (EMBEDDED_CONTENT_TAGS.has(element3.tagName)) {
+    return true;
+  }
+  return element3.tagName === "br" || element3.tagName === "hr" || element3.tagName === "input";
+};
+var collectPreformattedElements = (tree) => {
+  const protectedElements = /* @__PURE__ */ new Set();
+  const markSubtree = (node) => {
+    if (!("children" in node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (isElement(child)) {
+        protectedElements.add(child);
+        markSubtree(child);
+      }
+    }
+  };
+  walk(tree, (node) => {
+    if (isElement(node) && (node.tagName === "pre" || node.tagName === "code")) {
+      protectedElements.add(node);
+      markSubtree(node);
+      return false;
+    }
+  });
+  return protectedElements;
+};
+var dropEmptyElements = (tree, collector) => {
+  const preformatted = collectPreformattedElements(tree);
+  const visit = (parent) => {
+    for (let index = 0; index < parent.children.length; index++) {
+      const child = parent.children[index];
+      if (!isElement(child)) {
+        continue;
+      }
+      if (preformatted.has(child) || !DROP_WHEN_EMPTY_TAGS.has(child.tagName)) {
+        visit(child);
+        continue;
+      }
+      if (collector.metrics(child).text > 0 || hasEmbeddedDescendant(child)) {
+        visit(child);
+        continue;
+      }
+      if (holdsWhitespace(child)) {
+        parent.children[index] = { type: "text", value: " " };
+        continue;
+      }
+      parent.children.splice(index, 1);
+      index -= 1;
+    }
+  };
+  if ("children" in tree) {
+    visit(tree);
+  }
+  return tree;
+};
+var holdsWhitespace = (element3) => {
+  for (const child of element3.children) {
+    if (child.type === "text" && /\s/.test(child.value)) {
+      return true;
+    }
+    if (isElement(child) && holdsWhitespace(child)) {
+      return true;
+    }
+  }
+  return false;
+};
+var hasEmbeddedDescendant = (element3) => {
+  for (const child of element3.children) {
+    if (!isElement(child)) {
+      continue;
+    }
+    if (isEmbeddedContent(child) || hasEmbeddedDescendant(child)) {
+      return true;
+    }
+  }
+  return false;
+};
+var REFERENCE_HEADING = /^(see also|references?|further reading|external links?|bibliography|sources|footnotes|citations|関連項目|参考文献|外部リンク|脚注|出典|参考リンク|参考資料)$/i;
+var MAX_DANGLING_HEADING_LENGTH = 60;
+var isHeadingTag = (element3) => /^h[1-6]$/.test(element3.tagName);
+var previousElement = (siblings, index) => {
+  for (let cursor = index - 1; cursor >= 0; cursor--) {
+    const sibling = siblings[cursor];
+    if (isElement(sibling)) {
+      return sibling;
+    }
+    if (sibling.type === "text" && sibling.value.trim().length > 0) {
+      return void 0;
+    }
+  }
+  return void 0;
+};
+var condemnLinkBlock = (block, siblings, index, collector, doomed) => {
+  const heading2 = previousElement(siblings, index);
+  const isHeading = heading2 !== void 0 && isHeadingTag(heading2);
+  if (isHeading && REFERENCE_HEADING.test(elementText(heading2).trim())) {
+    return;
+  }
+  if (introducedByColon(siblings, index, collector)) {
+    return;
+  }
+  doomed.push(block);
+  if (isHeading && elementText(heading2).trim().length <= MAX_DANGLING_HEADING_LENGTH) {
+    doomed.push(heading2);
+  }
+};
+var introducedByColon = (siblings, index, collector) => {
+  const before = previousElement(siblings, index);
+  if (!before || collector.metrics(before).text === 0) {
+    return false;
+  }
+  return /[:：]\s*$/.test(elementText(before).trim());
+};
+var findCleanupTargets = (tree, collector) => {
+  const doomed = [];
+  const visit = (node) => {
+    if (!("children" in node)) {
+      return;
+    }
+    const siblings = node.children;
+    for (let index = 0; index < siblings.length; index++) {
+      const child = siblings[index];
+      if (!isElement(child)) {
+        continue;
+      }
+      if (child.tagName === "article" || child.tagName === "main") {
+        visit(child);
+        continue;
+      }
+      if (isLinkOnlyBlock(child, collector)) {
+        condemnLinkBlock(child, siblings, index, collector, doomed);
+        continue;
+      }
+      if (isMidArticleWidget(child)) {
+        doomed.push(child);
+        continue;
+      }
+      visit(child);
+    }
+  };
+  visit(tree);
+  return doomed;
+};
+var LINK_BLOCK_TAGS = /* @__PURE__ */ new Set(["ul", "ol", "nav", "div", "p"]);
+var MIN_LINK_BLOCK_ANCHORS = 3;
+var isLinkOnlyBlock = (element3, collector) => {
+  if (!LINK_BLOCK_TAGS.has(element3.tagName)) {
+    return false;
+  }
+  const metrics = collector.metrics(element3);
+  if (metrics.text === 0 || metrics.commas > 0) {
+    return false;
+  }
+  if (metrics.link / metrics.text <= 0.9) {
+    return false;
+  }
+  return countAnchors(element3) >= MIN_LINK_BLOCK_ANCHORS;
+};
+var countAnchors = (element3) => {
+  let count = 0;
+  const stack = [element3];
+  while (stack.length > 0 && count < MIN_LINK_BLOCK_ANCHORS) {
+    const node = stack.pop();
+    if (isElement(node) && node.tagName === "a") {
+      count += 1;
+    }
+    if ("children" in node) {
+      for (const child of node.children) {
+        stack.push(child);
+      }
+    }
+  }
+  return count;
+};
+var isMidArticleWidget = (element3) => {
+  if (element3.tagName === "form" || element3.tagName === "fieldset") {
+    return true;
+  }
+  if (element3.tagName === "button") {
+    return true;
+  }
+  return isUiChromeText(elementText(element3));
+};
+var cleanContent = (tree, collector) => {
+  const doomed = new Set(findCleanupTargets(tree, collector));
+  if (doomed.size > 0) {
+    pruneInPlace(tree, (node) => !(isElement(node) && doomed.has(node)));
+  }
+  return dropEmptyElements(tree, new MetricsCollector());
+};
+var elementText = (node) => {
+  if (node.type === "text") {
+    return node.value;
+  }
+  if (!("children" in node)) {
+    return "";
+  }
+  let text6 = "";
+  for (const child of node.children) {
+    text6 += elementText(child);
+  }
+  return text6;
+};
+
+// src/adapters/registry.ts
+import { select as select3, selectAll } from "hast-util-select";
+var EMPTY_SIGNATURE = {
+  classes: /* @__PURE__ */ new Set(),
+  ids: /* @__PURE__ */ new Set(),
+  tags: /* @__PURE__ */ new Set(),
+  generator: "",
+  meta: ""
+};
+var buildPageSignature = (root) => {
+  const classes = /* @__PURE__ */ new Set();
+  const ids = /* @__PURE__ */ new Set();
+  const tags = /* @__PURE__ */ new Set();
+  const generatorParts = [];
+  const metaParts = [];
+  walk(root, (node) => {
+    if (!isElement(node)) {
+      return;
+    }
+    tags.add(node.tagName);
+    for (const name of classList(node)) {
+      classes.add(name);
+    }
+    const id = stringProperty(node, "id");
+    if (id) {
+      ids.add(id);
+    }
+    if (node.tagName === "meta") {
+      const key = stringProperty(node, "name") ?? stringProperty(node, "property") ?? "";
+      const content = stringProperty(node, "content") ?? "";
+      metaParts.push(`${key}=${content}`);
+      if (key.toLowerCase() === "generator") {
+        generatorParts.push(content);
+      }
+    }
+  });
+  return {
+    classes,
+    ids,
+    tags,
+    generator: generatorParts.join(" ").toLowerCase(),
+    meta: metaParts.join("\n").toLowerCase()
+  };
+};
+var DEFAULT_MIN_LENGTH = 40;
+var hostOf = (url) => {
+  if (!url) {
+    return void 0;
+  }
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return void 0;
+  }
+};
+var defineSelectorAdapter = (spec) => ({
+  id: spec.id,
+  matches: (context) => {
+    const host = hostOf(context.url);
+    return Boolean(host && spec.hosts?.test(host));
+  },
+  matchesFingerprint: (context) => {
+    const signature = context.signature ?? EMPTY_SIGNATURE;
+    return spec.fingerprint?.(signature) ?? false;
+  },
+  extract: (context) => {
+    for (const selector of spec.content) {
+      const content = tryContentSelector(selector, spec, context);
+      if (content) {
+        return content;
+      }
+    }
+    return void 0;
+  }
+});
+var tryContentSelector = (selector, spec, context) => {
+  const found = selectAll(selector, context.hast);
+  if (found.length === 0) {
+    return void 0;
+  }
+  for (const element3 of found) {
+    removeAll(element3, spec.remove);
+  }
+  const total = found.reduce((sum, element3) => sum + new MetricsCollector().textLength(element3), 0);
+  if (total < (spec.minLength ?? DEFAULT_MIN_LENGTH)) {
+    return void 0;
+  }
+  const decorated = spec.decorate?.(found, context);
+  if (decorated) {
+    return decorated;
+  }
+  return found.length === 1 ? found[0] : { type: "root", children: found };
+};
+var removeAll = (root, selectors) => {
+  if (!selectors || selectors.length === 0) {
+    return;
+  }
+  const doomed = /* @__PURE__ */ new Set();
+  for (const selector of selectors) {
+    for (const element3 of selectAll(selector, root)) {
+      doomed.add(element3);
+    }
+  }
+  if (doomed.size > 0) {
+    pruneInPlace(root, (node) => !(isElement(node) && doomed.has(node)));
+  }
+};
+var resolveAdapterByHost = (context, adapters) => {
+  for (const adapter of adapters) {
+    try {
+      if (adapter.matches(context)) {
+        return adapter;
+      }
+    } catch {
+    }
+  }
+  return void 0;
+};
+var resolveAdapterByFingerprint = (context, adapters) => {
+  for (const adapter of adapters) {
+    try {
+      if (adapter.matchesFingerprint?.(context)) {
+        return adapter;
+      }
+    } catch {
+    }
+  }
+  return void 0;
+};
+
+// src/adapters/sites/declarative.ts
+var COMMON_NOISE = [
+  ".sr-only",
+  "[aria-hidden='true'][class*='icon']",
+  "button",
+  ".social-share",
+  ".share-buttons",
+  ".newsletter-signup"
+];
+var SPECS = [
+  {
+    id: "github",
+    hosts: /(^|\.)github\.com$/,
+    // Issues and PRs expose a testid; repository landing pages render the README instead.
+    content: ["[data-testid='issue-body']", "article.markdown-body", ".markdown-body"],
+    remove: [".js-comment-edit-history", ".Details-content--hidden", "clipboard-copy", ...COMMON_NOISE],
+    minLength: 30
+  },
+  {
+    id: "stackoverflow",
+    hosts: /(^|\.)(stackoverflow\.com|stackexchange\.com|superuser\.com|serverfault\.com|askubuntu\.com)$/,
+    content: ["#mainbar", ".question, .answer"],
+    remove: [
+      ".js-post-menu",
+      ".post-menu",
+      ".user-info",
+      ".votecell",
+      ".comments-link",
+      ".js-add-link",
+      ...COMMON_NOISE
+    ],
+    minLength: 100
+  },
+  {
+    id: "npm",
+    hosts: /(^|\.)npmjs\.com$/,
+    content: ["#readme", "#package-tab-readme", "[class*='markdown']"],
+    remove: COMMON_NOISE,
+    minLength: 50
+  },
+  {
+    id: "zenn",
+    hosts: /(^|\.)zenn\.dev$/,
+    content: [".znc"],
+    remove: COMMON_NOISE,
+    minLength: 50
+  },
+  {
+    id: "qiita",
+    hosts: /(^|\.)qiita\.com$/,
+    content: [".it-MdContent", "[class*='mdContent']", "article"],
+    remove: [".code-copy", ...COMMON_NOISE],
+    minLength: 50
+  },
+  {
+    id: "medium",
+    hosts: /(^|\.)medium\.com$/,
+    // Medium also powers custom domains; the fingerprint catches those.
+    fingerprint: (signature) => signature.meta.includes("al:android:app_name=medium"),
+    content: ["article section", "article", "[data-testid='storyContent']"],
+    remove: ["[data-testid='audioPlayButton']", "[role='separator']", ...COMMON_NOISE],
+    minLength: 200
+  },
+  {
+    id: "substack",
+    hosts: /(^|\.)substack\.com$/,
+    // Substack's custom domains are far more common than *.substack.com.
+    fingerprint: (signature) => signature.classes.has("available-content"),
+    content: [".available-content", ".body.markup", ".post-content"],
+    remove: [".subscription-widget-wrap", ".button-wrapper", ".paywall", ...COMMON_NOISE],
+    minLength: 200
+  },
+  {
+    id: "note",
+    hosts: /(^|\.)note\.com$/,
+    content: [".note-common-styles__textnote-body", "[class*='textnote-body']"],
+    remove: COMMON_NOISE,
+    minLength: 50
+  },
+  {
+    id: "hatena",
+    hosts: /(^|\.)(hatenablog\.com|hatenadiary\.(com|jp)|hateblo\.jp|hatenastaff\.com)$/,
+    fingerprint: (signature) => signature.classes.has("hatenablog-entry") || signature.classes.has("hatena-body"),
+    content: [".entry-content"],
+    remove: [".entry-footer", ".social-buttons", ".hatena-asin-detail", ...COMMON_NOISE],
+    minLength: 100
+  },
+  {
+    id: "wordpress",
+    // Deliberately fingerprint-only: WordPress powers a large share of the web on its own
+    // domains, so keying off a hostname would cover almost none of it.
+    fingerprint: (signature) => signature.generator.includes("wordpress") || signature.classes.has("wp-block-post-content") || signature.classes.has("wp-singular"),
+    content: [".wp-block-post-content", ".entry-content", ".post-content", "article .content"],
+    remove: [".sharedaddy", ".jp-relatedposts", ".wp-block-comments", "#comments", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "wikipedia",
+    hosts: /(^|\.)(wikipedia\.org|wikimedia\.org|wikibooks\.org|wiktionary\.org|wikisource\.org)$/,
+    fingerprint: (signature) => signature.classes.has("mw-parser-output"),
+    content: [".mw-parser-output", "#mw-content-text"],
+    remove: [
+      ".mw-editsection",
+      ".navbox",
+      ".navbox-inner",
+      ".vertical-navbox",
+      ".sidebar",
+      ".side-box",
+      ".metadata",
+      ".reflist",
+      ".refbegin",
+      ".sistersitebox",
+      ".ambox",
+      ".hatnote",
+      ".shortdescription",
+      ".mw-jump-link",
+      ".printfooter",
+      ".catlinks",
+      "#toc",
+      ".toc",
+      "table.mbox-small",
+      ...COMMON_NOISE
+    ],
+    minLength: 200
+  },
+  {
+    id: "reddit",
+    hosts: /(^|\.)reddit\.com$/,
+    content: ["shreddit-post [slot='text-body']", "[data-test-id='post-content']", ".Post"],
+    remove: ["faceplate-tracker[noun='comment_action_bar']", "shreddit-comment-action-row", ...COMMON_NOISE],
+    // Reddit renders most of a thread into shadow roots that never reach the serialised HTML,
+    // so a weak match here is worse than generic extraction. Decline unless a substantial
+    // post body is actually present in the document.
+    minLength: 600
+  },
+  {
+    id: "docusaurus",
+    fingerprint: (signature) => signature.classes.has("theme-doc-markdown") || signature.classes.has("docusaurus-mt-lg"),
+    content: [".theme-doc-markdown", "article .markdown"],
+    remove: [".theme-doc-toc-mobile", ".pagination-nav", ".theme-doc-version-banner", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "vitepress",
+    fingerprint: (signature) => signature.classes.has("vp-doc") || signature.classes.has("VPDoc"),
+    content: [".vp-doc > div", ".vp-doc", ".VPDoc .content-container"],
+    remove: [".VPDocFooter", ".vp-doc-footer", ".header-anchor", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "mkdocs",
+    fingerprint: (signature) => signature.classes.has("md-content__inner"),
+    content: [".md-content__inner", "[data-md-component='content']"],
+    remove: [".md-source-file", ".md-feedback", ".headerlink", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "readthedocs",
+    fingerprint: (signature) => signature.classes.has("rst-content"),
+    content: [".rst-content", "[role='main']"],
+    remove: [".headerlink", ".rst-footer-buttons", ".wy-breadcrumbs", ...COMMON_NOISE],
+    minLength: 150
+  },
+  {
+    id: "gitbook",
+    fingerprint: (signature) => signature.classes.has("gitbook-root"),
+    content: ["[data-testid='page.contentEditor']", "main article"],
+    remove: COMMON_NOISE,
+    minLength: 150
+  }
+];
+var DECLARATIVE_ADAPTERS = SPECS.map(defineSelectorAdapter);
+
+// src/adapters/sites/hackernews.ts
+import { select as select4, selectAll as selectAll2 } from "hast-util-select";
+var MAX_QUOTE_DEPTH = 6;
+var text3 = (value) => ({ type: "text", value });
+var element = (tagName, children) => ({
+  type: "element",
+  tagName,
+  properties: {},
+  children
+});
+var commentDepth = (row) => {
+  const cell = select4("td.ind", row);
+  if (!cell) {
+    return 0;
+  }
+  const indent = numericProperty(cell, "indent");
+  if (indent !== void 0) {
+    return indent;
+  }
+  const spacer = select4("img", cell);
+  const width = spacer ? numericProperty(spacer, "width") : void 0;
+  return width === void 0 ? 0 : Math.round(width / 40);
+};
+var nest = (content, depth) => {
+  let current = content;
+  for (let level = 0; level < Math.min(depth, MAX_QUOTE_DEPTH); level++) {
+    current = [element("blockquote", current)];
+  }
+  return current[0];
+};
+var storyNodes = (root) => {
+  const nodes = [];
+  const titleLink = select4(".titleline > a", root);
+  if (titleLink) {
+    nodes.push(element("h1", [...titleLink.children]));
+    const href = stringProperty(titleLink, "href");
+    if (href && /^https?:/.test(href)) {
+      nodes.push(element("p", [{ ...titleLink, children: [text3(href)] }]));
+    }
+  }
+  const topText = select4(".toptext", root);
+  if (topText) {
+    nodes.push(element("div", [...topText.children]));
+  }
+  return nodes;
+};
+var commentNodes = (root) => {
+  const nodes = [];
+  for (const row of selectAll2("tr.comtr, tr.athing.comtr", root)) {
+    const body = select4(".commtext", row);
+    if (!body) {
+      continue;
+    }
+    const author = select4(".hnuser", row);
+    const header = author ? [element("strong", [...author.children])] : [];
+    const content = [
+      ...header.length > 0 ? [element("p", header)] : [],
+      element("div", [...body.children])
+    ];
+    nodes.push(nest(content, commentDepth(row)));
+  }
+  return nodes;
+};
+var hackernewsAdapter = {
+  id: "hackernews",
+  matches: ({ url }) => {
+    if (!url) {
+      return false;
+    }
+    try {
+      return /(^|\.)ycombinator\.com$/.test(new URL(url).hostname);
+    } catch {
+      return false;
+    }
+  },
+  matchesFingerprint: ({ signature }) => Boolean(signature?.ids.has("hnmain") || signature?.classes.has("fatitem")),
+  extract: ({ hast }) => {
+    const story = storyNodes(hast);
+    const comments = commentNodes(hast);
+    if (story.length === 0 && comments.length === 0) {
+      return void 0;
+    }
+    const children = [...story];
+    if (comments.length > 0) {
+      children.push(element("h2", [text3("Comments")]));
+      children.push(...comments);
+    }
+    return { type: "root", children };
+  }
+};
+
+// src/adapters/sites/youtube.ts
+var findObjectEnd = (source, start) => {
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let index = start; index < source.length; index++) {
+    const character = source[index];
+    if (escaped) {
+      escaped = false;
+    } else if (character === "\\") {
+      escaped = true;
+    } else if (character === '"') {
+      inString = !inString;
+    } else if (!inString) {
+      if (character === "{") {
+        depth += 1;
+      } else if (character === "}" && --depth === 0) {
+        return index + 1;
+      }
+    }
+  }
+  return -1;
+};
+var extractAssignedJson = (source, name) => {
+  const markerIndex = source.indexOf(`${name} =`);
+  if (markerIndex === -1) {
+    return void 0;
+  }
+  const start = source.indexOf("{", markerIndex);
+  if (start === -1) {
+    return void 0;
+  }
+  const end = findObjectEnd(source, start);
+  if (end === -1) {
+    return void 0;
+  }
+  try {
+    return JSON.parse(source.slice(start, end));
+  } catch {
+    return void 0;
+  }
+};
+var scriptText = (root) => collectElements(root, (element3) => element3.tagName === "script").map(
+  (script) => script.children.map((child) => child.type === "text" ? child.value : "").join("")
+);
+var findPlayerResponse = (root) => {
+  for (const source of scriptText(root)) {
+    if (!source.includes("ytInitialPlayerResponse")) {
+      continue;
+    }
+    const parsed = extractAssignedJson(source, "ytInitialPlayerResponse");
+    if (parsed && typeof parsed === "object") {
+      return parsed;
+    }
+  }
+  return void 0;
+};
+var youtubeCaptionTracks = (root) => {
+  const player = findPlayerResponse(root);
+  const renderer = player?.captions?.playerCaptionsTracklistRenderer;
+  const tracks = renderer?.captionTracks;
+  if (!Array.isArray(tracks)) {
+    return [];
+  }
+  return tracks.flatMap((track) => {
+    const url = typeof track.baseUrl === "string" ? track.baseUrl : void 0;
+    const languageCode = typeof track.languageCode === "string" ? track.languageCode : void 0;
+    if (!(url && languageCode)) {
+      return [];
+    }
+    const name = track.name?.simpleText;
+    return [
+      {
+        url,
+        languageCode,
+        name: typeof name === "string" ? name : void 0,
+        isAutomatic: track.kind === "asr"
+      }
+    ];
+  });
+};
+var text4 = (value) => ({ type: "text", value });
+var element2 = (tagName, children) => ({
+  type: "element",
+  tagName,
+  properties: {},
+  children
+});
+var heading = (depth, value) => element2(`h${depth}`, [text4(value)]);
+var paragraph = (value) => element2("p", [text4(value)]);
+var descriptionNodes = (description) => description.split(/\n{2,}/).map((block) => block.trim()).filter((block) => block.length > 0).map(
+  (block) => element2(
+    "p",
+    block.split("\n").flatMap((line, index) => index === 0 ? [text4(line)] : [element2("br", []), text4(line)])
+  )
+);
+var formatDuration = (seconds) => {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor(seconds % 3600 / 60);
+  const remainder = seconds % 60;
+  const pad = (value) => String(value).padStart(2, "0");
+  return hours > 0 ? `${hours}:${pad(minutes)}:${pad(remainder)}` : `${minutes}:${pad(remainder)}`;
+};
+var metadataList = (details) => {
+  const items = [];
+  if (details.author) {
+    items.push(element2("li", [text4(`Channel: ${details.author}`)]));
+  }
+  const seconds = Number(details.lengthSeconds);
+  if (Number.isFinite(seconds) && seconds > 0) {
+    items.push(element2("li", [text4(`Duration: ${formatDuration(seconds)}`)]));
+  }
+  const views = Number(details.viewCount);
+  if (Number.isFinite(views) && views > 0) {
+    items.push(element2("li", [text4(`Views: ${views.toLocaleString("en-US")}`)]));
+  }
+  return items.length > 0 ? element2("ul", items) : void 0;
+};
+var youtubeAdapter = {
+  id: "youtube",
+  matches: ({ url }) => {
+    if (!url) {
+      return false;
+    }
+    try {
+      return /(^|\.)(youtube\.com|youtu\.be|youtube-nocookie\.com)$/.test(new URL(url).hostname);
+    } catch {
+      return false;
+    }
+  },
+  // Scanning every script body for the JSON blob would cost a full traversal on every page, so
+  // the fallback keys off markers the signature already collected.
+  matchesFingerprint: ({ signature }) => Boolean(signature?.ids.has("movie_player") || signature?.classes.has("ytd-app")),
+  extract: ({ hast }) => {
+    const player = findPlayerResponse(hast);
+    const details = player?.videoDetails;
+    if (!details?.title) {
+      return void 0;
+    }
+    const children = [heading(1, details.title)];
+    const metadata = metadataList(details);
+    if (metadata) {
+      children.push(metadata);
+    }
+    const description = details.shortDescription?.trim();
+    if (description) {
+      children.push(heading(2, "Description"));
+      children.push(...descriptionNodes(description));
+    }
+    const tracks = youtubeCaptionTracks(hast);
+    if (tracks.length > 0) {
+      const languages2 = [...new Set(tracks.map((track) => track.languageCode))].sort();
+      children.push(paragraph(`Captions available: ${languages2.join(", ")}`));
+    }
+    return { type: "root", children };
+  }
+};
+
+// src/adapters/index.ts
+var BUILTIN_ADAPTERS = [youtubeAdapter, hackernewsAdapter, ...DECLARATIVE_ADAPTERS];
+var MIN_ADAPTER_TEXT = 40;
+var runAdapters = (context, adapters = BUILTIN_ADAPTERS) => {
+  let matched = resolveAdapterByHost(context, adapters);
+  let withSignature = context;
+  if (!matched) {
+    withSignature = { ...context, signature: context.signature ?? buildPageSignature(context.hast) };
+    matched = resolveAdapterByFingerprint(withSignature, adapters);
+  }
+  const adapter = matched;
+  if (!adapter) {
+    return void 0;
+  }
+  const target = withSignature.owned ? withSignature : { ...withSignature, hast: cloneHast(withSignature.hast), owned: true };
+  let extracted;
+  try {
+    extracted = adapter.extract(target);
+  } catch {
+    return void 0;
+  }
+  if (!extracted) {
+    return void 0;
+  }
+  stripNonContent(extracted);
+  return new MetricsCollector().textLength(extracted) >= MIN_ADAPTER_TEXT ? extracted : void 0;
+};
 
-// src/extractors/presets/utils.ts
-var matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")}`;
-var classnames = (element) => {
-  if (Array.isArray(element.properties.className)) {
-    return element.properties.className;
+// src/extractors/lib/score.ts
+var TAG_BASE_SCORE = {
+  div: 5,
+  pre: 3,
+  td: 3,
+  blockquote: 3,
+  article: 10,
+  main: 10,
+  section: 5,
+  address: -3,
+  ol: -3,
+  ul: -3,
+  dl: -3,
+  dd: -3,
+  dt: -3,
+  li: -3,
+  form: -3,
+  h1: -5,
+  h2: -5,
+  h3: -5,
+  h4: -5,
+  h5: -5,
+  h6: -5,
+  th: -5
+};
+var MAX_ANCESTOR_DEPTH = 5;
+var MIN_SCOREABLE_TEXT = 25;
+var buildScoringContext = (root, collector, lang) => {
+  const parents = /* @__PURE__ */ new WeakMap();
+  const scoreable = [];
+  let hasGeometry = false;
+  walk(root, (node, parent) => {
+    if (parent) {
+      parents.set(node, parent);
+    }
+    if (!isElement(node)) {
+      return;
+    }
+    if (!hasGeometry && numericProperty(node, "data-rheight") !== void 0) {
+      hasGeometry = true;
+    }
+    if (isParagraphLike(node)) {
+      scoreable.push(node);
+    }
+  });
+  return { collector, parents, lang, hasGeometry, scoreable };
+};
+var classWeight = (element3) => {
+  let weight = 0;
+  const id = stringProperty(element3, "id") ?? "";
+  const classes = classList(element3).join(" ");
+  if (REGEXPS.negative.test(classes)) {
+    weight -= 25;
   }
-  return [];
+  if (REGEXPS.positive.test(classes)) {
+    weight += 25;
+  }
+  if (REGEXPS.negative.test(id)) {
+    weight -= 25;
+  }
+  if (REGEXPS.positive.test(id)) {
+    weight += 25;
+  }
+  return weight;
 };
-var isStrInclude = (value, match) => {
-  if (typeof value === "string") {
-    return value.includes(match);
+var semanticBonus = (element3) => {
+  let bonus = 0;
+  if (element3.tagName === "article") {
+    bonus += 60;
   }
-  return false;
+  if (element3.tagName === "main") {
+    bonus += 50;
+  }
+  const role = stringProperty(element3, "role");
+  if (role && CONTENT_ROLES.has(role)) {
+    bonus += 40;
+  }
+  const itemprop = stringProperty(element3, "itemprop");
+  if (itemprop?.includes("articleBody")) {
+    bonus += 80;
+  }
+  const match = matchString(element3);
+  if (/(^|[\s_-])(markdown|prose|article-?content|post-?content|entry-?content|doc-?content|md-?content)([\s_-]|$)/i.test(
+    match
+  )) {
+    bonus += 35;
+  }
+  return bonus;
 };
-
-// src/extractors/presets/takumi.ts
-var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
-var REGEXPS = {
-  hidden: /hidden|invisible|fallback-image/i,
-  byline: /byline|author|dateline|writtenby|p-author/i,
-  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
-  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
-};
-var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
-var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
-var CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
-var TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
-var BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
-var metadataFilter = (node) => {
-  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
-};
-var universalElementFilter = (node) => {
-  if (node.type !== "element") {
-    return true;
+var geometryMultiplier = (element3, context) => {
+  if (!context.hasGeometry) {
+    return 1;
   }
-  const element = node;
-  if (["aside", "nav"].includes(element.tagName)) {
-    return false;
+  const height = numericProperty(element3, "data-rheight");
+  const width = numericProperty(element3, "data-rwidth");
+  if (height === void 0 || width === void 0) {
+    return 1;
   }
-  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
-    return false;
+  if (height === 0 || width === 0) {
+    return 0;
   }
-  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
-    return false;
+  if (width < 220) {
+    return 0.35;
   }
-  if (element.tagName === "dialog") {
-    return false;
+  return 1;
+};
+var paragraphScore = (textLength, commas, densityDivisor) => 1 + commas + Math.min(Math.floor(textLength / densityDivisor), 3);
+var ancestorWeight = (level) => {
+  if (level === 0) {
+    return 1;
   }
-  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
-    return false;
+  if (level === 1) {
+    return 2;
   }
-  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
+  return level * 3;
+};
+var isParagraphLike = (element3) => {
+  if (!SCOREABLE_TAGS.has(element3.tagName)) {
     return false;
   }
-  if (REGEXPS.byline.test(matchString(element))) {
-    return false;
+  if (element3.tagName !== "div" && element3.tagName !== "section") {
+    return true;
   }
-  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
-    return false;
+  return !element3.children.some((child) => isElement(child) && CANDIDATE_TAGS.has(child.tagName));
+};
+var scoreCandidates = (_root, context) => {
+  const { collector, parents } = context;
+  const scores = /* @__PURE__ */ new Map();
+  const densityDivisor = minContentLength(context.lang) / 4;
+  for (const node of context.scoreable) {
+    const metrics = collector.metrics(node);
+    if (metrics.text < MIN_SCOREABLE_TEXT) {
+      continue;
+    }
+    const contribution = paragraphScore(metrics.text, metrics.commas, densityDivisor);
+    let current = node;
+    for (let level = 0; level < MAX_ANCESTOR_DEPTH && current; level++) {
+      const parent = parents.get(current);
+      if (!(parent && isElement(parent))) {
+        break;
+      }
+      if (CANDIDATE_TAGS.has(parent.tagName) || parent.tagName === "article" || parent.tagName === "main") {
+        const base = scores.get(parent) ?? (TAG_BASE_SCORE[parent.tagName] ?? 0) + classWeight(parent) + semanticBonus(parent);
+        scores.set(parent, base + contribution / ancestorWeight(level));
+      }
+      current = parent;
+    }
   }
-  return true;
+  const candidates = [];
+  for (const [element3, rawScore] of scores) {
+    const density = collector.linkDensity(element3);
+    const score = rawScore * (1 - density) * geometryMultiplier(element3, context);
+    if (score > 0) {
+      candidates.push({ element: element3, score });
+    }
+  }
+  candidates.sort((a, b) => b.score - a.score);
+  return candidates;
 };
-var unlikelyElementFilter = (node) => {
-  if (node.type !== "element") {
-    return true;
+var collectArticleNodes = (top, candidates, context) => {
+  const { collector, parents } = context;
+  const parent = parents.get(top.element);
+  if (!(parent && isElement(parent))) {
+    return [top.element];
   }
-  const element = node;
-  if (CONTENTABLE_TAGS.includes(element.tagName)) {
-    return true;
+  const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
+  const threshold = Math.max(10, top.score * 0.2);
+  const selected = [];
+  for (const child of parent.children) {
+    if (!isElement(child)) {
+      continue;
+    }
+    if (child === top.element) {
+      selected.push(child);
+      continue;
+    }
+    const siblingScore = scoreOf.get(child) ?? 0;
+    if (siblingScore >= threshold) {
+      selected.push(child);
+      continue;
+    }
+    if (child.tagName === "p" || child.tagName === "section") {
+      const metrics = collector.metrics(child);
+      const density = collector.linkDensity(child);
+      if (metrics.text > 80 && density < 0.25) {
+        selected.push(child);
+      }
+    }
   }
-  const match = matchString(element);
-  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
+  return selected.length > 0 ? selected : [top.element];
+};
+var MAX_CLIMB_HOPS = 5;
+var MIN_CLIMB_RELATIVE_GAIN = 0.08;
+var MIN_CLIMB_ABSOLUTE_GAIN = 500;
+var MAX_CLIMB_GAIN_LINK_DENSITY = 1 / 3;
+var CLIMB_SCORE_FLOOR = 1 / 3;
+var climbToArticleBoundary = (top, candidates, context) => {
+  const { collector, parents } = context;
+  const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
+  const scoreFloor = top.score * CLIMB_SCORE_FLOOR;
+  let current = top.element;
+  for (let hop = 0; hop < MAX_CLIMB_HOPS; hop++) {
+    const parent = parents.get(current);
+    if (!(parent && isElement(parent))) {
+      break;
+    }
+    const parentScore = scoreOf.get(parent);
+    if (parentScore === void 0 || parentScore < scoreFloor) {
+      break;
+    }
+    const currentMetrics = collector.metrics(current);
+    const parentMetrics = collector.metrics(parent);
+    const gainedText = parentMetrics.text - currentMetrics.text;
+    if (gainedText < Math.max(MIN_CLIMB_ABSOLUTE_GAIN, currentMetrics.text * MIN_CLIMB_RELATIVE_GAIN)) {
+      break;
+    }
+    const gainedLink = parentMetrics.link - currentMetrics.link;
+    if (gainedLink / gainedText > MAX_CLIMB_GAIN_LINK_DENSITY) {
+      break;
+    }
+    current = parent;
+  }
+  return current;
+};
+
+// src/extractors/lib/terminators.ts
+var TERMINATOR_PATTERNS = [
+  /^was this (page|article|document) helpful/i,
+  /^thanks for the feedback/i,
+  /^(this page was )?last (modified|updated) (on|at|:)?\s/i,
+  /^learn how to contribute/i,
+  /^help (us )?improve\b/i,
+  // The stranded Yes/No pair of a feedback widget whose question was removed separately.
+  /^yes ?no$/i,
+  /^(related (articles?|posts?|stories|content)|you might also like|recommended for you|more from|read next)$/i,
+  /^(comments?|leave a (comment|reply)|join the (conversation|discussion))$/i,
+  /^(この(ページ|記事)は(参考|役に立ち)ました?か)/,
+  /^(関連記事|あわせて読みたい|こちらもおすすめ|おすすめ記事)$/,
+  /^(コメントを(残す|書く)|コメント一覧)$/,
+  /^最終更新日?[:：\s]/
+];
+var TAIL_START_FRACTION = 0.8;
+var MAX_TERMINATOR_BLOCK_LENGTH = 500;
+var BLOCK_TAGS = /* @__PURE__ */ new Set(["p", "div", "section", "footer", "aside", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol"]);
+var isTerminatorBlock = (element3, start, tailStart) => {
+  if (!BLOCK_TAGS.has(element3.tagName) || start < tailStart) {
     return false;
   }
-  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
+  const text6 = textOf(element3).replace(/\s+/g, " ").trim();
+  if (text6.length === 0 || text6.length > MAX_TERMINATOR_BLOCK_LENGTH) {
     return false;
   }
-  return true;
+  return TERMINATOR_PATTERNS.some((pattern) => pattern.test(text6));
 };
-var removeEmptyFilter = (node, _lang) => {
-  if (node.type !== "element") {
-    return true;
+var textOf = (node) => {
+  if (node.type === "text") {
+    return node.value;
   }
-  const element = node;
-  if (PARAGRAPH_TAGS.includes(element.tagName)) {
-    return true;
+  if (!isParent(node)) {
+    return "";
+  }
+  let text6 = "";
+  for (const child of node.children) {
+    text6 += textOf(child);
   }
-  if (element.tagName === "img" && !element.properties.src) {
+  return text6;
+};
+var truncateTrailingBoilerplate = (tree) => {
+  const starts = /* @__PURE__ */ new Map();
+  let total = 0;
+  const measure = (node) => {
+    starts.set(node, total);
+    if (node.type === "text") {
+      total += node.value.trim().length;
+      return;
+    }
+    if (isParent(node)) {
+      for (const child of node.children) {
+        measure(child);
+      }
+    }
+  };
+  measure(tree);
+  if (total === 0) {
     return false;
   }
-  if (TEXTABLE_TAGS.includes(element.tagName) && hastToString(element).length === 0) {
+  const tailStart = total * TAIL_START_FRACTION;
+  let cutoff = -1;
+  const find = (node) => {
+    if (cutoff !== -1 || !isParent(node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (cutoff !== -1) {
+        return;
+      }
+      if (!isElement(child)) {
+        continue;
+      }
+      const start = starts.get(child) ?? 0;
+      if (isTerminatorBlock(child, start, tailStart)) {
+        cutoff = start;
+        return;
+      }
+      find(child);
+    }
+  };
+  find(tree);
+  if (cutoff === -1) {
     return false;
   }
+  pruneInPlace(tree, (node) => {
+    if (node === tree) {
+      return true;
+    }
+    const start = starts.get(node);
+    if (start === void 0) {
+      return true;
+    }
+    if (start >= cutoff) {
+      return false;
+    }
+    return true;
+  });
   return true;
 };
-var takumiExtractor = (params) => {
-  const { hast, lang = "en" } = params;
-  const body = select("body", hast) ?? hast;
-  const metadataFilteredHast = filter2(body, (node) => metadataFilter(node));
-  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
-  if (!(metadataFilteredHast && metadataFilteredHastText)) {
-    return body;
-  }
-  const baseFilterd = filter2(metadataFilteredHast, (node) => universalElementFilter(node));
-  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
-  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
-  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
-  if (baseText.length < minimalLength) {
-    minimalLength = Math.max(0, baseText.length - 200);
-  }
-  let extractedTree = baseTree;
-  let extractedText = baseText;
-  for (const selector of BODY_SELECTORS) {
-    const content = { type: "root", children: selectAll(selector, baseFilterd) };
-    const contentText = hastToString(content);
-    if (contentText.length < 25) {
+
+// src/extractors/presets/takumi.ts
+var MIN_RETAINED_FRACTION = 0.25;
+var SEMANTIC_SELECTORS = [
+  (element3) => stringProperty(element3, "itemprop")?.includes("articleBody") ?? false,
+  (element3) => element3.tagName === "article",
+  (element3) => element3.tagName === "main",
+  (element3) => stringProperty(element3, "role") === "main"
+];
+var findBody = (hast) => findElement(hast, (element3) => element3.tagName === "body") ?? hast;
+var semanticShortcut = (tree, collector, minLength) => {
+  for (const matches of SEMANTIC_SELECTORS) {
+    const found = [];
+    collectMatching(tree, matches, found);
+    if (found.length !== 1) {
       continue;
     }
-    const links = selectAll("a", content);
-    const linkText = links.map((link) => hastToString(link)).join("");
-    const linkDensity = linkText.length / contentText.length;
-    if (linkDensity > 0.4) {
+    const candidate = found[0];
+    const metrics = collector.metrics(candidate);
+    if (metrics.text >= minLength && collector.linkDensity(candidate) < 0.5) {
+      return candidate;
+    }
+  }
+  return void 0;
+};
+var collectMatching = (tree, matches, into) => {
+  const stack = [tree];
+  while (stack.length > 0 && into.length < 2) {
+    const node = stack.pop();
+    if (isElement(node) && matches(node)) {
+      into.push(node);
       continue;
     }
-    if (contentText.length > minimalLength) {
-      extractedTree = content;
-      extractedText = contentText;
-      break;
+    if ("children" in node) {
+      for (const child of node.children) {
+        stack.push(child);
+      }
     }
   }
-  const finalFilteredTree = filter2(extractedTree, (node) => {
-    if (!removeEmptyFilter(node, lang)) {
-      return false;
-    }
-    if (!unlikelyElementFilter(node)) {
-      return false;
+};
+var asRootOf = (nodes) => ({ type: "root", children: nodes });
+var scoredSelection = (tree, collector, lang) => {
+  const context = buildScoringContext(tree, collector, lang);
+  const candidates = scoreCandidates(tree, context);
+  const top = candidates[0];
+  if (!top) {
+    return void 0;
+  }
+  const promoted = climbToArticleBoundary(top, candidates, context);
+  const anchor = promoted === top.element ? top : { element: promoted, score: top.score };
+  const selected = collectArticleNodes(anchor, candidates, context);
+  return selected.length === 1 ? selected[0] : asRootOf(selected);
+};
+var takumiExtractor = (params) => {
+  const { hast, lang, owned } = params;
+  const body = owned ? findBody(hast) : cloneHast(findBody(hast));
+  stripNonContent(body);
+  let collector = new MetricsCollector();
+  const baseText = collector.textLength(body);
+  if (baseText === 0) {
+    return body;
+  }
+  const minLength = Math.min(minContentLength(lang), Math.max(0, baseText - 200));
+  const shortcut = semanticShortcut(body, collector, minLength);
+  const searchRoot = shortcut ?? body;
+  const searchText = collector.textLength(searchRoot);
+  const doomed = findUnlikelyElements(searchRoot);
+  const doomedText = doomed.reduce((sum, element3) => sum + collector.metrics(element3).text, 0);
+  if (searchText - doomedText > Math.max(minLength, searchText * MIN_RETAINED_FRACTION)) {
+    const doomedSet = new Set(doomed);
+    pruneInPlace(searchRoot, (node) => !(isElement(node) && doomedSet.has(node)));
+    collector = new MetricsCollector();
+  }
+  const scoringRoot = searchRoot;
+  const scoringRootText = collector.textLength(scoringRoot);
+  const selection = shortcut ? scoringRoot : scoredSelection(scoringRoot, collector, lang) ?? scoringRoot;
+  const selectionText = collector.textLength(selection);
+  const selectionIsSound = selectionText >= minLength || selectionText > scoringRootText * MIN_RETAINED_FRACTION;
+  const chosen = selectionIsSound ? selection : scoringRoot;
+  truncateTrailingBoilerplate(chosen);
+  const cleanupCollector = new MetricsCollector();
+  const chosenText = cleanupCollector.textLength(chosen);
+  const widgets = findCleanupTargets(chosen, cleanupCollector);
+  const widgetText = widgets.reduce((sum, element3) => sum + cleanupCollector.metrics(element3).text, 0);
+  const remainingText = chosenText - widgetText;
+  if (remainingText < chosenText * MIN_RETAINED_FRACTION && remainingText < minLength) {
+    return chosen;
+  }
+  return cleanContent(chosen, cleanupCollector);
+};
+
+// src/extractors/presets/auto.ts
+var createAutoExtractor = (options = {}) => {
+  const { adapters } = options;
+  return (params) => {
+    if (adapters !== false) {
+      const extracted = runAdapters({ hast: params.hast, url: params.url, lang: params.lang }, adapters ?? void 0);
+      if (extracted) {
+        return extracted;
+      }
     }
-    return true;
-  });
-  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
-  return finalTree;
+    return takumiExtractor(params);
+  };
 };
+var autoExtractor = createAutoExtractor();
 
 // src/constants.ts
-var DEFAULT_EXTRACTORS = [takumiExtractor];
+var DEFAULT_EXTRACTORS = [autoExtractor];
 
 // src/extractors/pipeExtractors.ts
 var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
-  const { hast, lang } = params;
-  const _extractors = Array.isArray(extractors) ? extractors : [extractors];
-  const extracted = _extractors.reduce((acc, extractor) => {
+  const { hast, lang, url, owned } = params;
+  const list = Array.isArray(extractors) ? extractors : [extractors];
+  let current = hast;
+  let currentOwned = owned ?? false;
+  for (const extractor of list) {
     if (extractor === false) {
-      return acc;
+      continue;
     }
-    if (typeof extractor === "function") {
-      return extractor({ hast: acc, lang });
+    if (typeof extractor !== "function") {
+      throw new Error(`Invalid extractor: ${String(extractor)}`);
     }
-    throw new Error(`Invalid extractor: ${extractor}`);
-  }, hast) || hast;
-  return extracted;
+    current = extractor({ hast: current, lang, url, owned: currentOwned });
+    currentOwned = true;
+  }
+  return current;
+};
+
+// src/extractors/presets/minimal-filter.ts
+var MIN_RETAINED_FRACTION2 = 1 / 3;
+var ALWAYS_ACCEPT_LENGTH = 5e3;
+var isChrome = (element3) => {
+  if (element3.tagName === "aside" || element3.tagName === "nav" || element3.tagName === "dialog") {
+    return true;
+  }
+  const role = stringProperty(element3, "role");
+  if (role && UNLIKELY_ROLES.has(role)) {
+    return true;
+  }
+  if (role === "dialog" && isTruthyAttribute(element3, "aria-modal")) {
+    return true;
+  }
+  if (stringProperty(element3, "rel") === "author") {
+    return true;
+  }
+  if (REGEXPS.byline.test(matchString(element3))) {
+    return true;
+  }
+  return false;
+};
+var minimalFilter = (params) => {
+  const { hast } = params;
+  const body = cloneHast(findElement(hast, (element3) => element3.tagName === "body") ?? hast);
+  stripNonContent(body);
+  const collector = new MetricsCollector();
+  const baseText = collector.textLength(body);
+  if (baseText === 0) {
+    return body;
+  }
+  const filtered = cloneHast(body);
+  pruneInPlace(filtered, (node) => !(isElement(node) && isChrome(node)));
+  const filteredText = new MetricsCollector().textLength(filtered);
+  const isOverExtracted = filteredText < baseText * MIN_RETAINED_FRACTION2 && filteredText < ALWAYS_ACCEPT_LENGTH;
+  return isOverExtracted ? body : filtered;
 };
 
 // src/mdast-handlers/custom-a-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
-import { toString as hastToString2 } from "hast-util-to-string";
+import { toString as hastToString } from "hast-util-to-string";
+var ALLOWED_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:", "mailto:", "tel:", "ftp:", "ftps:"]);
+var isFollowableHref = (href) => {
+  const trimmed = href?.trim();
+  if (!trimmed) {
+    return false;
+  }
+  const scheme = trimmed.match(/^[a-z][a-z0-9+.-]*:/i)?.[0];
+  if (!scheme) {
+    return true;
+  }
+  return ALLOWED_SCHEMES.has(scheme.toLowerCase());
+};
 var customAHandler = (options) => (state, node) => {
-  if (options?.asText) {
-    const text3 = hastToString2(node);
-    if (3 >= text3.length) {
+  const asPlainText = () => {
+    const text6 = hastToString(node);
+    if (text6.length === 0) {
       return void 0;
     }
-    const link2 = { type: "text", value: text3 };
-    state.patch(node, link2);
-    return link2;
+    const result = { type: "text", value: text6 };
+    state.patch(node, result);
+    return result;
+  };
+  if (options?.asText) {
+    return hastToString(node).length <= 3 ? void 0 : asPlainText();
   }
-  const link = defaultHandlers.a(state, node);
-  if (link.children.length > 0) {
-    return link;
+  if (!isFollowableHref(stringProperty(node, "href"))) {
+    return asPlainText();
   }
-  return void 0;
+  const link = defaultHandlers.a(state, node);
+  return link.children.length > 0 ? link : void 0;
 };
 
 // src/mdast-handlers/custom-code-handler.ts
@@ -435,9 +2699,9 @@ var customCodeHandler = (state, node) => {
 };
 
 // src/mdast-handlers/custom-div-handler.ts
-import { select as select2 } from "hast-util-select";
+import { select as select5 } from "hast-util-select";
 import { defaultHandlers as defaultHandlers2 } from "hast-util-to-mdast";
-import { toString as hastToString3 } from "hast-util-to-string";
+import { toString as hastToString2 } from "hast-util-to-string";
 import { toText as toText2 } from "hast-util-to-text";
 import { trimTrailingLines as trimTrailingLines2 } from "trim-trailing-lines";
 var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
@@ -472,8 +2736,8 @@ var customDivHandler = (state, node) => {
   if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
     const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
     const codeValue = trimTrailingLines2(toText2(codeBlock)).trim();
-    const filenameElement = select2(CODE_FILENAME_SELECTORS, node);
-    const fileLang = filenameElement ? hastToString3(filenameElement).match(/\.(\w+)$/)?.[1] : null;
+    const filenameElement = select5(CODE_FILENAME_SELECTORS, node);
+    const fileLang = filenameElement ? hastToString2(filenameElement).match(/\.(\w+)$/)?.[1] : null;
     const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
       const match = LANGUAGE_MATCH_REGEX2.map((regex) => className.match(regex)).find((match2) => match2);
       return match?.[1];
@@ -488,51 +2752,317 @@ var customDivHandler = (state, node) => {
 
 // src/mdast-handlers/custom-img-handler.ts
 import { defaultHandlers as defaultHandlers3 } from "hast-util-to-mdast";
+var MAX_DATA_URL_LENGTH = 256;
+var hasNoUsableSource = (node) => {
+  const src = stringProperty(node, "src")?.trim();
+  if (!src) {
+    return true;
+  }
+  if (src.startsWith("data:")) {
+    return src.length > MAX_DATA_URL_LENGTH;
+  }
+  return false;
+};
 var customImgHandler = (options) => (state, node) => {
   if (options?.hideImage) {
     return void 0;
   }
-  return defaultHandlers3.image(state, node);
+  if (hasNoUsableSource(node)) {
+    return void 0;
+  }
+  const image = defaultHandlers3.image(state, node);
+  if (!image.alt) {
+    const title = stringProperty(node, "title");
+    if (title) {
+      image.alt = title;
+    }
+  }
+  return image;
 };
 
 // src/mdast-handlers/custom-table-handler.ts
 import { defaultHandlers as defaultHandlers4 } from "hast-util-to-mdast";
 import { toText as toText3 } from "hast-util-to-text";
+var MAX_SPAN = 1e3;
+var clampSpans = (node) => {
+  walk(node, (candidate) => {
+    if (!isElement(candidate) || candidate.tagName !== "td" && candidate.tagName !== "th") {
+      return;
+    }
+    for (const attribute of ["colSpan", "rowSpan"]) {
+      const span = numericProperty(candidate, attribute);
+      if (span !== void 0 && span > MAX_SPAN) {
+        candidate.properties[attribute] = MAX_SPAN;
+      }
+    }
+  });
+};
 var customTableHandler = (options) => (state, node) => {
   if (options?.asText) {
-    const paragraph = { type: "paragraph", children: [{ type: "text", value: toText3(node) }] };
-    state.patch(node, paragraph);
-    return paragraph;
+    const paragraph2 = { type: "paragraph", children: [{ type: "text", value: toText3(node) }] };
+    state.patch(node, paragraph2);
+    return paragraph2;
   }
+  clampSpans(node);
   return defaultHandlers4.table(state, node);
 };
 
+// src/mdast-handlers/definition-list-handler.ts
+var hoistBoundaryWhitespace = (nodes) => {
+  const result = [];
+  for (const node of nodes) {
+    if (node.type !== "emphasis" && node.type !== "strong") {
+      result.push(node);
+      continue;
+    }
+    node.children = hoistBoundaryWhitespace(node.children);
+    const first = node.children[0];
+    if (first?.type === "text" && /^\s/.test(first.value)) {
+      result.push({ type: "text", value: " " });
+      first.value = first.value.replace(/^\s+/, "");
+    }
+    let trailing = false;
+    const last = node.children[node.children.length - 1];
+    if (last?.type === "text" && /\s$/.test(last.value)) {
+      last.value = last.value.replace(/\s+$/, "");
+      trailing = true;
+    }
+    node.children = node.children.filter((child) => !(child.type === "text" && child.value.length === 0));
+    if (node.children.length > 0) {
+      result.push(node);
+    }
+    if (trailing) {
+      result.push({ type: "text", value: " " });
+    }
+  }
+  return result;
+};
+var definitionListHandler = (state, node) => {
+  const items = [];
+  let current;
+  const flush = () => {
+    if (!current) {
+      return;
+    }
+    const heading2 = {
+      type: "paragraph",
+      children: current.term.length > 0 ? [{ type: "strong", children: current.term }] : []
+    };
+    items.push({ type: "listItem", spread: false, children: [heading2, ...current.definitions] });
+    current = void 0;
+  };
+  for (const child of node.children) {
+    if (child.type !== "element") {
+      continue;
+    }
+    const element3 = child;
+    if (element3.tagName === "dt") {
+      if (current && current.definitions.length > 0) {
+        flush();
+      }
+      const phrasing = hoistBoundaryWhitespace(state.all(element3));
+      current = current ? { term: [...current.term, { type: "text", value: ", " }, ...phrasing], definitions: [] } : { term: phrasing, definitions: [] };
+      continue;
+    }
+    if (element3.tagName === "dd") {
+      if (!current) {
+        current = { term: [], definitions: [] };
+      }
+      const content = state.toFlow(state.all(element3));
+      current.definitions.push(...content);
+    }
+  }
+  flush();
+  if (items.length === 0) {
+    return void 0;
+  }
+  const result = { type: "list", ordered: false, spread: false, children: items };
+  state.patch(node, result);
+  return result;
+};
+
 // src/mdast-handlers/math-handler.ts
 import { toHtml } from "hast-util-to-html";
 import { MathMLToLaTeX } from "mathml-to-latex";
+var readAnnotation = (node) => {
+  const alttext = stringProperty(node, "alttext");
+  if (!alttext) {
+    return void 0;
+  }
+  const unwrapped = alttext.trim().replace(/^\{\\displaystyle\s*([\s\S]*)\}$/, "$1").trim();
+  return unwrapped.length > 0 ? unwrapped : void 0;
+};
+var isDisplayMath = (node) => stringProperty(node, "display") === "block";
 var mathHandler = (state, node) => {
-  const mathMl = toHtml(node);
-  const latex = MathMLToLaTeX.convert(mathMl);
-  const result = { type: "inlineMath", value: latex };
+  const latex = readAnnotation(node) ?? MathMLToLaTeX.convert(toHtml(node));
+  const result = isDisplayMath(node) ? { type: "math", value: latex, meta: null } : { type: "inlineMath", value: latex };
+  state.patch(node, result);
+  return result;
+};
+
+// src/mdast-handlers/sup-sub-handler.ts
+import { toString as hastToString3 } from "hast-util-to-string";
+var MAX_ANNOTATION_LENGTH = 32;
+var supSubHandler = (tagName) => (state, node) => {
+  const inner = hastToString3(node).trim();
+  if (inner.length === 0 || inner.length > MAX_ANNOTATION_LENGTH) {
+    return state.all(node);
+  }
+  const escaped = inner.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+  const result = { type: "html", value: `<${tagName}>${escaped}</${tagName}>` };
   state.patch(node, result);
   return result;
 };
 
+// src/normalize/index.ts
+var LAZY_SRC_ATTRIBUTES = [
+  "data-src",
+  "data-original",
+  "data-lazy-src",
+  "data-original-src",
+  "data-actualsrc",
+  "data-hi-res-src",
+  "data-echo"
+];
+var PLACEHOLDER_SRC = /^$|^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
+var readSrcset = (element3, attribute) => {
+  const raw = property(element3, attribute);
+  if (typeof raw === "string") {
+    return raw;
+  }
+  if (Array.isArray(raw)) {
+    return raw.join(", ");
+  }
+  return void 0;
+};
+var bestFromSrcset = (srcset) => {
+  let best;
+  for (const entry of srcset.split(",")) {
+    const parts = entry.trim().split(/\s+/);
+    const url = parts[0];
+    if (!url) {
+      continue;
+    }
+    const descriptor = parts[1];
+    let weight = 1;
+    if (descriptor?.endsWith("w")) {
+      weight = Number.parseFloat(descriptor) || 1;
+    } else if (descriptor?.endsWith("x")) {
+      weight = (Number.parseFloat(descriptor) || 1) * 1e3;
+    }
+    if (!best || weight > best.weight) {
+      best = { url, weight };
+    }
+  }
+  return best?.url;
+};
+var resolveImageSources = (tree) => {
+  walk(tree, (node) => {
+    if (!isElement(node) || node.tagName !== "img") {
+      return;
+    }
+    const current = stringProperty(node, "src");
+    if (current && !PLACEHOLDER_SRC.test(current)) {
+      return;
+    }
+    for (const attribute of LAZY_SRC_ATTRIBUTES) {
+      const candidate = stringProperty(node, attribute);
+      if (candidate && !PLACEHOLDER_SRC.test(candidate)) {
+        node.properties.src = candidate;
+        return;
+      }
+    }
+    for (const attribute of ["srcSet", "srcset", "data-srcset"]) {
+      const srcset = readSrcset(node, attribute);
+      const best = srcset ? bestFromSrcset(srcset) : void 0;
+      if (best) {
+        node.properties.src = best;
+        return;
+      }
+    }
+  });
+};
+var katexDuplicates = (node) => {
+  const classes = classList(node);
+  if (!(classes.includes("katex") || classes.includes("katex-display"))) {
+    return [];
+  }
+  const hasMathml = node.children.some((child) => isElement(child) && classList(child).includes("katex-mathml"));
+  if (!hasMathml) {
+    return [];
+  }
+  return node.children.filter((child) => isElement(child) && classList(child).includes("katex-html"));
+};
+var mediawikiDuplicates = (node) => {
+  if (!classList(node).includes("mwe-math-element")) {
+    return [];
+  }
+  return node.children.filter(
+    (child) => isElement(child) && classList(child).some((name) => name.startsWith("mwe-math-fallback-image"))
+  );
+};
+var mathjaxDuplicates = (node) => {
+  if (node.tagName !== "mjx-container") {
+    return [];
+  }
+  return node.children.filter((child) => isElement(child) && child.tagName !== "mjx-assistive-mml");
+};
+var dedupeRenderedMath = (tree) => {
+  const doomed = /* @__PURE__ */ new Set();
+  walk(tree, (node) => {
+    if (!isElement(node)) {
+      return;
+    }
+    for (const duplicate of [...katexDuplicates(node), ...mathjaxDuplicates(node), ...mediawikiDuplicates(node)]) {
+      doomed.add(duplicate);
+    }
+  });
+  if (doomed.size > 0) {
+    pruneInPlace(tree, (node) => !(isElement(node) && doomed.has(node)));
+  }
+};
+var promoteAriaHeadings = (tree) => {
+  walk(tree, (node) => {
+    if (!isElement(node) || stringProperty(node, "role") !== "heading") {
+      return;
+    }
+    if (/^h[1-6]$/.test(node.tagName)) {
+      return;
+    }
+    const level = numericProperty(node, "aria-level") ?? 2;
+    const clamped = Math.min(6, Math.max(1, Math.round(level)));
+    node.tagName = `h${clamped}`;
+  });
+};
+var normalizeHast = (tree, options = {}) => {
+  const { images = true, math = true, ariaHeadings = true } = options;
+  if (images) {
+    resolveImageSources(tree);
+  }
+  if (math) {
+    dedupeRenderedMath(tree);
+  }
+  if (ariaHeadings) {
+    promoteAriaHeadings(tree);
+  }
+  return tree;
+};
+
 // src/utils/hast-utils.ts
-import { select as select3, selectAll as selectAll2 } from "hast-util-select";
+import { select as select6, selectAll as selectAll3 } from "hast-util-select";
 var getLangFromHast = (node) => {
-  const html = select3("html", node);
+  const html = select6("html", node);
   if (html && typeof html.properties.lang === "string") {
     return html.properties.lang;
   }
   if (node.type !== "element") {
     return;
   }
-  const element = node;
-  if (element.tagName !== "html") {
+  const element3 = node;
+  if (element3.tagName !== "html") {
     return;
   }
-  const langAttr = element.properties.lang || element.properties["xml:lang"];
+  const langAttr = element3.properties.lang || element3.properties["xml:lang"];
   if (langAttr) {
     return langAttr;
   }
@@ -550,10 +3080,10 @@ var getUrlFromHast = (node) => {
     return void 0;
   }
   const metaTagAttributes = ["og:url", "twitter:url"];
-  const metaTags = selectAll2("meta", node);
+  const metaTags = selectAll3("meta", node);
   for (const meta of metaTags) {
-    const property = meta.properties.property || meta.properties.name;
-    if (typeof property === "string" && metaTagAttributes.includes(property)) {
+    const property2 = meta.properties.property || meta.properties.name;
+    if (typeof property2 === "string" && metaTagAttributes.includes(property2)) {
       return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
     }
   }
@@ -564,13 +3094,18 @@ var getUrlFromHast = (node) => {
 var htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
   const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
+  const isOwned = typeof htmlOrHast === "string" || (options?.owned ?? false);
   const url = defaultUrl || getUrlFromHast(hast);
-  const extractedHast = pipeExtractors({ hast, lang, url }, extractors);
+  const extractedHast = pipeExtractors({ hast, lang, url, owned: isOwned }, extractors);
+  normalizeHast(extractedHast, options?.normalize);
   const mdast = toMdast(extractedHast, {
     handlers: {
       math: mathHandler,
       div: customDivHandler,
       pre: customCodeHandler,
+      dl: definitionListHandler,
+      sup: supSubHandler("sup"),
+      sub: supSubHandler("sub"),
       a: customAHandler({ asText: options?.linkAsText }),
       img: customImgHandler({ hideImage: options?.hideImage }),
       table: customTableHandler({ asText: options?.tableAsText })
@@ -623,222 +3158,241 @@ var mdastToMarkdown = (mdast, options) => {
   return markdown;
 };
 
-// src/html-to-markdown.ts
-var htmlToMarkdown = (htmlOrHast, options) => {
-  const { baseUrl, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
-  const mdast = htmlToMdast(htmlOrHast, toMdastOptions);
-  const markdown = mdastToMarkdown(mdast, { baseUrl, ...toMarkdownOptions });
-  return markdown;
+// src/metadata/index.ts
+var readMetaTags = (tree) => {
+  const tags = /* @__PURE__ */ new Map();
+  for (const meta of collectElements(tree, (element3) => element3.tagName === "meta")) {
+    const key = stringProperty(meta, "property") ?? stringProperty(meta, "name") ?? stringProperty(meta, "itemprop");
+    const content = stringProperty(meta, "content");
+    if (!(key && content)) {
+      continue;
+    }
+    const normalized = key.toLowerCase();
+    if (!tags.has(normalized)) {
+      tags.set(normalized, content);
+    }
+  }
+  return tags;
+};
+var readJsonLd = (tree) => {
+  const entries = [];
+  for (const script of collectElements(tree, (element3) => element3.tagName === "script")) {
+    const type = stringProperty(script, "type");
+    if (type?.toLowerCase() !== "application/ld+json") {
+      continue;
+    }
+    const source = script.children.map((child) => child.type === "text" ? child.value : "").join("").trim();
+    if (source.length === 0) {
+      continue;
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(source);
+    } catch {
+      continue;
+    }
+    for (const entry of flattenJsonLd(parsed)) {
+      entries.push(entry);
+    }
+  }
+  return entries;
+};
+var flattenJsonLd = (value) => {
+  if (Array.isArray(value)) {
+    return value.flatMap(flattenJsonLd);
+  }
+  if (typeof value !== "object" || value === null) {
+    return [];
+  }
+  const record = value;
+  const graph = record["@graph"];
+  if (graph) {
+    return flattenJsonLd(graph);
+  }
+  return [record];
 };
-
-// src/cli/helpers/inputOutputPath.ts
-import fs2 from "node:fs";
-import { confirm, text } from "@clack/prompts";
-
-// src/cli/utils.ts
-import fs from "node:fs";
-import path from "node:path";
-var isUrl = (maybeUrl) => {
-  try {
-    new URL(maybeUrl);
-    return true;
-  } catch {
-    return false;
+var ARTICLE_TYPES = /* @__PURE__ */ new Set([
+  "article",
+  "newsarticle",
+  "blogposting",
+  "techarticle",
+  "scholarlyarticle",
+  "report",
+  "webpage"
+]);
+var jsonLdType = (entry) => {
+  const type = entry["@type"];
+  if (typeof type === "string") {
+    return [type.toLowerCase()];
+  }
+  if (Array.isArray(type)) {
+    return type.filter((value) => typeof value === "string").map((value) => value.toLowerCase());
   }
+  return [];
 };
-function changeFileExtension(filePath, newExtension) {
-  const parsedPath = filePath.split("/");
-  const fileName = parsedPath[parsedPath.length - 1];
-  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
-  if (fileName.startsWith(".")) {
-    const parts = fileName.split(".");
-    if (parts.length === 2) {
-      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
-    }
-    parts[parts.length - 1] = newExtension.replace(/^\./, "");
-    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+var readName = (value) => {
+  if (typeof value === "string") {
+    return value;
   }
-  const lastDotIndex = fileName.lastIndexOf(".");
-  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
-  const newFileName = `${baseName}${formattedNewExtension}`;
-  parsedPath[parsedPath.length - 1] = newFileName;
-  return parsedPath.join("/");
-}
-function urlToFilename(url) {
-  try {
-    const urlObj = new URL(url);
-    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
-      if (index === 0) {
-        return acc;
-      }
-      if (acc.length >= 2) {
-        return acc;
-      }
-      if (part === "www") {
-        return acc;
+  if (Array.isArray(value)) {
+    for (const entry of value) {
+      const name = readName(entry);
+      if (name) {
+        return name;
       }
-      return [part, ...acc];
-    }, []);
-    const domainString = domainParts.reverse().join("-");
-    const pathParts = urlObj.pathname.split("/").filter(Boolean);
-    const relevantPathParts = pathParts.slice(-2);
-    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
-    let filename = [domainString, pathString].filter(Boolean).join("-");
-    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
-    return filename || "output";
-  } catch {
-    return "output";
+    }
+    return void 0;
   }
-}
-var sourcePathToOutputPath = (sourcePath) => {
-  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
+  if (typeof value === "object" && value !== null) {
+    const name = value.name;
+    return typeof name === "string" ? name : void 0;
+  }
+  return void 0;
 };
-function getNextAvailableFilePath(filePath) {
-  const parsedPath = path.parse(filePath);
-  const directory = parsedPath.dir;
-  const fullName = parsedPath.base;
-  const [firstPart, ...restParts] = fullName.split(".");
-  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
-  const baseName = firstPart.replace(/_\d+$/, "");
-  let counter = 1;
-  let nextFilePath = filePath;
-  while (fs.existsSync(nextFilePath)) {
-    const match = firstPart.match(/_(\d+)$/);
-    if (match) {
-      counter = Number.parseInt(match[1], 10) + 1;
-    }
-    const newName = `${baseName}_${counter}${restName}`;
-    nextFilePath = path.join(directory, newName);
-    counter++;
+var asString = (value) => typeof value === "string" && value.length > 0 ? value : void 0;
+var MAX_TITLE_LENGTH = 160;
+var asTitle = (value) => {
+  const cleaned = clean(value);
+  return cleaned && cleaned.length <= MAX_TITLE_LENGTH ? cleaned : void 0;
+};
+var clean = (value) => {
+  if (value === void 0) {
+    return void 0;
   }
-  return nextFilePath;
-}
-
-// src/cli/helpers/assertContinue.ts
-import { cancel, isCancel } from "@clack/prompts";
-function assertContinue(message, cancelMessage = "Canceled.") {
-  if (isCancel(message)) {
-    cancel(cancelMessage);
-    process.exit(1);
+  const trimmed = value.replace(/\s+/g, " ").trim();
+  return trimmed.length > 0 ? trimmed : void 0;
+};
+var documentTitle = (tree) => {
+  const title = findElement(tree, (element3) => element3.tagName === "title");
+  if (!title) {
+    return void 0;
   }
-}
-
-// src/cli/helpers/inputOutputPath.ts
-var inputOutputPath = async (sourcePath) => {
-  const outputPath = await text({
-    message: "Enter the output file path:",
-    placeholder: "output.md",
-    initialValue: sourcePathToOutputPath(sourcePath),
-    validate: (value) => {
-      if (value.trim() === "") {
-        return "Output path is required";
-      }
-      if (fs2.existsSync(value) && fs2.statSync(value).isDirectory()) {
-        return "No directory can be specified.";
-      }
-      if (!fs2.existsSync(value) && value.endsWith("/")) {
-        return "No directory can be specified.";
-      }
+  return title.children.map((child) => child.type === "text" ? child.value : "").join("");
+};
+var firstHeading = (tree) => {
+  const heading2 = findElement(tree, (element3) => element3.tagName === "h1");
+  if (!heading2) {
+    return void 0;
+  }
+  let text6 = "";
+  walk(heading2, (node) => {
+    if (node.type === "text") {
+      text6 += node.value;
     }
   });
-  assertContinue(outputPath);
-  if (!fs2.existsSync(outputPath)) {
-    return outputPath;
+  return text6;
+};
+var linkHref = (tree, rel) => {
+  const link = findElement(
+    tree,
+    (element3) => element3.tagName === "link" && (stringProperty(element3, "rel") ?? "").toLowerCase().includes(rel)
+  );
+  return link ? stringProperty(link, "href") : void 0;
+};
+var documentLang = (tree) => {
+  const html = findElement(tree, (element3) => element3.tagName === "html");
+  if (html) {
+    const declared = stringProperty(html, "lang") ?? stringProperty(html, "xml:lang");
+    if (declared) {
+      return declared;
+    }
   }
-  const isOutputFileOverwrite = await confirm({
-    message: "The file already exists. Overwrite?",
-    initialValue: false
-  });
-  assertContinue(isOutputFileOverwrite);
-  if (isOutputFileOverwrite) {
-    return outputPath;
+  if (isElement(tree)) {
+    const own = stringProperty(tree, "lang");
+    if (own) {
+      return own;
+    }
   }
-  const escapedOutputPath = await text({
-    message: "Enter the output file path:",
-    placeholder: "output.md",
-    initialValue: getNextAvailableFilePath(outputPath),
-    validate: (value) => {
-      if (value.trim() === "") {
-        return "Output path is required";
-      }
-      if (fs2.existsSync(value)) {
-        return "The file already exists";
-      }
-      if (!fs2.existsSync(value) && value.endsWith("/")) {
-        return "No directory can be specified.";
+  const tagged = findElement(tree, (element3) => Boolean(stringProperty(element3, "lang")));
+  return tagged ? stringProperty(tagged, "lang") : void 0;
+};
+var extractMetadata = (tree) => {
+  const meta = readMetaTags(tree);
+  const jsonLd = readJsonLd(tree);
+  const article = jsonLd.find((entry) => jsonLdType(entry).some((type) => ARTICLE_TYPES.has(type))) ?? jsonLd[0];
+  const get = (...keys) => {
+    for (const key of keys) {
+      const value = meta.get(key);
+      if (value) {
+        return value;
       }
     }
-  });
-  assertContinue(escapedOutputPath);
-  return escapedOutputPath;
+    return void 0;
+  };
+  const title = asTitle(get("og:title", "twitter:title")) ?? asTitle(asString(article?.headline)) ?? asTitle(asString(article?.name)) ?? asTitle(documentTitle(tree)) ?? asTitle(firstHeading(tree));
+  return {
+    title,
+    description: clean(get("og:description", "twitter:description", "description")) ?? clean(asString(article?.description)),
+    author: clean(readName(article?.author)) ?? clean(get("author", "article:author", "twitter:creator")),
+    published: clean(asString(article?.datePublished)) ?? clean(get("article:published_time", "date", "pubdate")),
+    modified: clean(asString(article?.dateModified)) ?? clean(get("article:modified_time", "og:updated_time")),
+    siteName: clean(get("og:site_name", "application-name")) ?? clean(readName(article?.publisher)),
+    canonicalUrl: clean(linkHref(tree, "canonical")) ?? clean(get("og:url", "twitter:url")),
+    lang: clean(documentLang(tree)),
+    image: clean(get("og:image", "twitter:image", "twitter:image:src")),
+    type: clean(get("og:type"))
+  };
 };
-
-// src/cli/helpers/inputSourcePath.ts
-import fs3 from "node:fs";
-import { text as text2 } from "@clack/prompts";
-
-// src/cli/constants.ts
-var DEFAULT_PATH = "https://example.com";
-var LOADERS = ["fetch", "playwright"];
-var MODES = ["default", "ai"];
-
-// src/cli/helpers/inputSourcePath.ts
-var inputSourcePath = async () => {
-  const result = await text2({
-    message: "Enter the URL or html path to be converted to markdown:",
-    placeholder: DEFAULT_PATH,
-    initialValue: "",
-    validate: (value) => {
-      if (value.trim() === "") {
-        return "Source is required";
-      }
-      if (!isUrl(value)) {
-        if (!fs3.existsSync(value)) {
-          return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
-        }
-        if (fs3.statSync(value).isDirectory()) {
-          return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
-        }
-      }
+var yamlString = (value) => `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+var toFrontmatter = (metadata) => {
+  const lines = [];
+  for (const [key, value] of Object.entries(metadata)) {
+    if (typeof value === "string" && value.length > 0) {
+      lines.push(`${key}: ${yamlString(value)}`);
     }
-  });
-  assertContinue(result);
-  return result;
+  }
+  if (lines.length === 0) {
+    return "";
+  }
+  return `---
+${lines.join("\n")}
+---
+
+`;
 };
 
-// src/cli/helpers/selectExtractMode.ts
-import { select as select4 } from "@clack/prompts";
-var selectExtractMode = async () => {
-  const result = await select4({
-    message: "Select processing mode:",
-    options: MODES.map((mode) => ({ value: mode, label: mode })),
-    initialValue: MODES[0]
+// src/html-to-markdown.ts
+var htmlToMarkdownWithMetadata = (htmlOrHast, options) => {
+  const { baseUrl, frontmatter, title, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
+  const hast = typeof htmlOrHast === "string" ? fromHtml2(htmlOrHast, { fragment: true }) : htmlOrHast;
+  const isOwned = typeof htmlOrHast === "string";
+  const declaredLang = typeof htmlOrHast === "string" ? getLangFromStr(htmlOrHast) : void 0;
+  const metadata = extractMetadata(hast);
+  if (declaredLang && !metadata.lang) {
+    metadata.lang = declaredLang;
+  }
+  const mdast = htmlToMdast(hast, {
+    ...toMdastOptions,
+    owned: isOwned,
+    lang: toMdastOptions.lang ?? metadata.lang,
+    url: toMdastOptions.url ?? metadata.canonicalUrl
   });
-  assertContinue(result);
-  return result;
+  const body = mdastToMarkdown(mdast, { baseUrl, ...toMarkdownOptions });
+  const titled = title === false ? body : withTitle(body, metadata.title);
+  const markdown = frontmatter ? `${toFrontmatter(metadata)}${titled}` : titled;
+  return { markdown, metadata };
 };
+var TITLE_LOOKAHEAD = 400;
+var withTitle = (body, title) => {
+  if (!title) {
+    return body;
+  }
+  const opening = body.slice(0, TITLE_LOOKAHEAD);
+  if (/^#\s+\S/m.test(opening)) {
+    return body;
+  }
+  if (normalizeForCompare(opening).includes(normalizeForCompare(title))) {
+    return body;
+  }
+  return `# ${title}
 
-// src/cli/helpers/selectLoader.ts
-import { select as select5 } from "@clack/prompts";
-var loadersHint = {
-  fetch: "Fetch HTML content from the given URL",
-  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
-  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
-};
-var selectLoader = async () => {
-  const result = await select5({
-    message: "Select loader:",
-    initialValue: "fetch",
-    options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
-  });
-  assertContinue(result);
-  return result;
+${body}`;
 };
+var normalizeForCompare = (value) => value.replace(/\s+/g, " ").trim().toLowerCase();
 
 // src/cli/commands/webforai/loadHtml.ts
-import fs4 from "node:fs/promises";
+import fs5 from "node:fs/promises";
 import { fileURLToPath } from "node:url";
-import { log } from "@clack/prompts";
 import boxen from "boxen";
 import pc from "picocolors";
 import { chromium as chromium2 } from "playwright-core";
@@ -885,10 +3439,10 @@ var loadHtml2 = async (url, options) => {
   await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
   await page.evaluate(() => {
     const elements = document.querySelectorAll("*");
-    for (const element of elements) {
-      const rect = element.getBoundingClientRect();
-      element.setAttribute("data-rwidth", rect.width.toString());
-      element.setAttribute("data-rheight", rect.height.toString());
+    for (const element3 of elements) {
+      const rect = element3.getBoundingClientRect();
+      element3.setAttribute("data-rwidth", rect.width.toString());
+      element3.setAttribute("data-rheight", rect.height.toString());
     }
   });
   const html = await page.content();
@@ -902,37 +3456,42 @@ var loadHtml2 = async (url, options) => {
 };
 
 // src/cli/commands/webforai/loadHtml.ts
+var debugLog = (enabled, message) => {
+  if (enabled) {
+    console.error(`[webforai] ${message}`);
+  }
+};
 var checkPlaywrightAvailable = async () => {
-  const path3 = chromium2.executablePath();
+  const path4 = chromium2.executablePath();
   try {
-    await fs4.access(path3);
+    await fs5.access(path4);
     return true;
   } catch {
     return false;
   }
 };
 var getPlaywrightVersion = async () => {
-  const path3 = await import.meta.resolve("playwright-core/package.json");
-  const pwPackageJson = await fs4.readFile(fileURLToPath(path3), "utf-8").then((res) => JSON.parse(res.toString())).catch(() => null);
+  const path4 = await import.meta.resolve("playwright-core/package.json");
+  const pwPackageJson = await fs5.readFile(fileURLToPath(path4), "utf-8").then((res) => JSON.parse(res.toString())).catch(() => null);
   return pwPackageJson?.version;
 };
 var loadHtml3 = async (sourcePath, loader, options) => {
   if (loader === "local") {
-    options.debug && log.info(`Loading HTML from local file: ${sourcePath}`);
-    const content = await fs4.readFile(sourcePath, "utf-8");
-    options.debug && log.info(`HTML loaded: ${content.slice(0, 100)}`);
+    debugLog(options.debug, `Loading HTML from local file: ${sourcePath}`);
+    const content = await fs5.readFile(sourcePath, "utf-8");
+    debugLog(options.debug, `HTML loaded: ${content.slice(0, 100)}`);
     return content;
   }
   if (loader === "fetch") {
-    options.debug && log.info(`Loading HTML from URL: ${sourcePath}`);
+    debugLog(options.debug, `Loading HTML from URL: ${sourcePath}`);
     const content = await loadHtml(sourcePath);
-    options.debug && log.info(`HTML loaded: ${content.slice(0, 100)}`);
+    debugLog(options.debug, `HTML loaded: ${content.slice(0, 100)}`);
     return content;
   }
   if (loader === "playwright") {
-    options.debug && log.info(`Loading HTML from playwright: ${sourcePath}`);
+    debugLog(options.debug, `Loading HTML with playwright: ${sourcePath}`);
     const isPlaywrightAvailable = await checkPlaywrightAvailable();
-    options.debug && log.info(`Playwright available: ${isPlaywrightAvailable}`);
+    debugLog(options.debug, `Playwright available: ${isPlaywrightAvailable}`);
     const pwVersion = await getPlaywrightVersion();
     if (!isPlaywrightAvailable) {
       const message = [
@@ -952,56 +3511,297 @@ var loadHtml3 = async (sourcePath, loader, options) => {
         "",
         `  sudo npx playwright@${pwVersion} install-deps`
       ];
-      log.error(boxen(message.join("\n"), { padding: 1, borderStyle: "round" }));
+      console.error(boxen(message.join("\n"), { padding: 1, borderStyle: "round" }));
       throw new Error("Playwright is not available");
     }
     const content = await loadHtml2(sourcePath);
-    options.debug && log.info(`HTML loaded: ${content.slice(0, 100)}`);
+    debugLog(options.debug, `HTML loaded: ${content.slice(0, 100)}`);
     return content;
   }
   throw new Error(`Unsupported loader: ${loader}`);
 };
 
-// src/cli/commands/webforai/index.ts
-var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
-var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
-var webforaiCommand = async (initialPath, options) => {
+// src/cli/commands/webforai/run.ts
+var debugLog2 = (enabled, message) => {
+  if (enabled) {
+    console.error(`[webforai] ${message}`);
+  }
+};
+var AI_MODE_OPTIONS = { linkAsText: true, tableAsText: true, hideImage: true };
+var extractorOptions = (name) => {
+  switch (name) {
+    case "takumi":
+      return takumiExtractor;
+    case "minimal":
+      return minimalFilter;
+    case "none":
+      return false;
+    default:
+      return void 0;
+  }
+};
+var convertViaPlatform = async (run) => {
+  const platform = createPlatformClient({ apiKey: run.apiKey, baseUrl: run.platformUrl });
+  if (run.mode !== "default") {
+    console.error("[webforai] --mode is applied by local conversion only and is ignored by the platform loader");
+  }
+  debugLog2(run.debug, `platform scrape: ${run.source} (engine=${run.engine ?? "fetch"})`);
+  const result = await platform.scrape({
+    url: run.source,
+    engine: run.engine,
+    region: run.region,
+    screenshot: run.screenshot ? true : void 0,
+    convert: { extractor: run.extractor, frontmatter: run.frontmatter }
+  });
+  return {
+    source: run.source,
+    loader: "platform",
+    url: result.url,
+    engine: result.engine,
+    region: run.region,
+    markdown: result.markdown,
+    metadata: result.metadata,
+    credits: result.credits,
+    screenshotUrl: result.screenshotUrl
+  };
+};
+var convertLocally = async (run) => {
+  const html = await loadHtml3(run.source, run.loader, { debug: run.debug });
+  const sourceUrl = isUrl(run.source) ? run.source : void 0;
+  const { markdown, metadata } = htmlToMarkdownWithMetadata(html, {
+    baseUrl: sourceUrl,
+    url: sourceUrl,
+    frontmatter: run.frontmatter,
+    extractors: extractorOptions(run.extractor),
+    ...run.mode === "ai" ? AI_MODE_OPTIONS : {}
+  });
+  return {
+    source: run.source,
+    loader: run.loader,
+    url: sourceUrl,
+    markdown,
+    metadata
+  };
+};
+var executeRun = (run) => run.loader === "platform" ? convertViaPlatform(run) : convertLocally(run);
+var writeOutputFile = async (outputPath, markdown) => {
+  await fs6.mkdir(path3.dirname(path3.resolve(outputPath)), { recursive: true });
+  await fs6.writeFile(outputPath, markdown);
+};
+var runCommand = async (run) => {
+  const envelope = await executeRun(run);
+  if (run.output) {
+    await writeOutputFile(run.output, envelope.markdown);
+    envelope.output = run.output;
+    console.error(`[webforai] markdown written to ${run.output}`);
+  }
+  if (run.json) {
+    process.stdout.write(`${JSON.stringify(envelope, null, 2)}
+`);
+    return;
+  }
+  if (!run.output) {
+    process.stdout.write(envelope.markdown.endsWith("\n") ? envelope.markdown : `${envelope.markdown}
+`);
+  }
+  if (envelope.screenshotUrl) {
+    console.error(`[webforai] screenshot (expires ~24h): ${envelope.screenshotUrl}`);
+  }
+  if (envelope.credits !== void 0) {
+    console.error(`[webforai] credits used: ${envelope.credits}`);
+  }
+};
+
+// src/cli/commands/webforai/interactive.ts
+var ENGINE_HINTS = {
+  fetch: "Plain fetch \xB7 1 credit",
+  browser: "Browser rendering, screenshots \xB7 5 credits",
+  "proxy-fetch": "Rotating-proxy egress \xB7 2 credits",
+  "proxy-browser": "Browser rendering behind the proxy \xB7 5 credits"
+};
+var promptPlatformOptions = async (flags, env) => {
+  const engine = flags.engine ?? await select7({
+    message: "Select platform engine:",
+    initialValue: "fetch",
+    options: ENGINES.map((value) => ({ value, label: value, hint: ENGINE_HINTS[value] }))
+  });
+  assertContinue(engine);
+  const region = flags.region ?? await select7({
+    message: "Select egress region (proxy engines only):",
+    initialValue: "auto",
+    options: REGIONS.map((value) => ({ value, label: value }))
+  });
+  assertContinue(region);
+  let apiKey = flags.apiKey ?? env[API_KEY_ENV];
+  if (!apiKey) {
+    const entered = await text5({
+      message: `Enter your platform API key (or set ${API_KEY_ENV}):`,
+      placeholder: "wfa_...",
+      validate: (value) => value.trim() === "" ? "An API key is required for the platform loader" : void 0
+    });
+    assertContinue(entered);
+    apiKey = entered;
+  }
+  return { engine, region, apiKey };
+};
+var interactiveCommand = async (initialSource, flags, env = process.env) => {
   intro(pc2.bold(pc2.green(`webforai CLI version ${package_default.version}`)));
-  const sourcePath = initialPath ?? await inputSourcePath();
-  options.debug && log2.info(`sourcePath: ${sourcePath}`);
-  const loader = isUrl(sourcePath) ? options.loader ?? await selectLoader() : "local";
-  options.debug && log2.info(`loader: ${loader}`);
-  const outputPath = options.output ?? await inputOutputPath(sourcePath);
-  options.debug && log2.info(`outputPath: ${outputPath}`);
-  const mode = options.mode ?? await selectExtractMode();
-  options.debug && log2.info(`mode: ${mode}`);
-  let html;
+  const source = initialSource ?? await inputSourcePath();
+  const loader = isUrl(source) ? flags.loader ?? await selectLoader() : "local";
+  const run = {
+    source,
+    loader,
+    mode: "default",
+    extractor: flags.extractor ?? "auto",
+    frontmatter: flags.frontmatter ?? false,
+    json: false,
+    debug: flags.debug ?? false,
+    screenshot: flags.screenshot ?? false
+  };
+  if (loader === "platform") {
+    const platform = await promptPlatformOptions(flags, env);
+    run.engine = platform.engine;
+    run.region = platform.region;
+    run.apiKey = platform.apiKey;
+    run.platformUrl = flags.platformUrl;
+  } else {
+    run.mode = flags.mode ?? await selectExtractMode();
+  }
+  const outputPath = flags.output ?? await inputOutputPath(source);
   const s = spinner();
+  s.start(loader === "platform" ? "Converting via the platform..." : "Loading and converting...");
+  let markdown;
   try {
-    s.start("Loading content...");
-    html = await loadHtml3(sourcePath, loader, { debug: options.debug });
-    s.stop(pc2.green("Content loaded!"));
+    const envelope = await executeRun(run);
+    markdown = envelope.markdown;
+    s.stop(pc2.green("Converted!"));
+    if (envelope.credits !== void 0) {
+      log.info(`Credits used: ${envelope.credits}`);
+    }
+    if (envelope.screenshotUrl) {
+      log.info(`Screenshot (expires ~24h): ${envelope.screenshotUrl}`);
+    }
   } catch (error) {
-    s.stop(pc2.red("Content loading failed!"));
-    console.error(error);
-    process.exit(1);
-  }
-  options.debug && log2.info(`html: ${html}`);
-  const markdown = htmlToMarkdown(html, {
-    baseUrl: isUrl(sourcePath) ? sourcePath : void 0,
-    ...mode === "ai" ? aiModeOptions : readabilityModeOptions
-  });
-  options.debug && log2.info(`markdown: ${markdown}`);
-  const directory = path2.dirname(outputPath);
-  const isDirectoryExists = await fs5.stat(directory).then((stat) => stat.isDirectory());
-  if (!isDirectoryExists) {
-    await fs5.mkdir(directory, { recursive: true });
+    s.stop(pc2.red("Conversion failed!"));
+    throw error;
   }
-  await fs5.writeFile(outputPath, markdown);
+  await writeOutputFile(outputPath, markdown);
   outro(pc2.green(`${pc2.bold("Done!")} Markdown saved to ${outputPath}`));
 };
 
+// src/cli/commands/webforai/options.ts
+import fs7 from "node:fs";
+var UsageError = class extends Error {
+};
+var oneOf = (name, value, allowed) => {
+  if (!allowed.includes(value)) {
+    throw new UsageError(`invalid --${name} "${value}" (expected: ${allowed.join(" | ")})`);
+  }
+  return value;
+};
+var impliesPlatform = (flags) => flags.engine !== void 0 || flags.region !== void 0;
+var resolveLoader = (sourceIsUrl, flags) => {
+  if (flags.loader) {
+    return oneOf("loader", flags.loader, LOADERS);
+  }
+  if (impliesPlatform(flags)) {
+    return "platform";
+  }
+  return sourceIsUrl ? "fetch" : "local";
+};
+var assertUsableLocalSource = (source, loader, flags) => {
+  if (loader !== "local" && flags.loader) {
+    throw new UsageError(`--loader ${loader} needs a URL source, but "${source}" is a local path`);
+  }
+  if (impliesPlatform(flags)) {
+    throw new UsageError("--engine/--region need a URL source (the platform cannot read local files)");
+  }
+  if (!fs7.existsSync(source)) {
+    throw new UsageError(`source not found: "${source}" (URLs must start with http:// or https://)`);
+  }
+};
+var applyPlatformFields = (resolved, flags, env) => {
+  resolved.engine = flags.engine ? oneOf("engine", flags.engine, ENGINES) : void 0;
+  resolved.region = flags.region ? oneOf("region", flags.region, REGIONS) : void 0;
+  resolved.apiKey = flags.apiKey ?? env[API_KEY_ENV];
+  resolved.platformUrl = flags.platformUrl ?? env[PLATFORM_URL_ENV];
+  if (!resolved.apiKey) {
+    throw new UsageError(
+      `the platform loader needs an API key: pass --api-key or set ${API_KEY_ENV} (create one at https://platform.webforai.dev/dashboard)`
+    );
+  }
+};
+var resolveRunOptions = (source, flags, env = process.env) => {
+  const sourceIsUrl = isUrl(source);
+  const loader = resolveLoader(sourceIsUrl, flags);
+  if (!sourceIsUrl) {
+    assertUsableLocalSource(source, loader, flags);
+  }
+  const resolved = {
+    source,
+    loader: sourceIsUrl ? loader : "local",
+    mode: flags.mode ? oneOf("mode", flags.mode, MODES) : "default",
+    extractor: flags.extractor ? oneOf("extractor", flags.extractor, EXTRACTORS) : "auto",
+    frontmatter: flags.frontmatter ?? false,
+    output: flags.output,
+    json: flags.json ?? false,
+    debug: flags.debug ?? false,
+    screenshot: flags.screenshot ?? false
+  };
+  if (resolved.loader === "platform") {
+    applyPlatformFields(resolved, flags, env);
+  } else if (flags.screenshot) {
+    throw new UsageError("--screenshot is only available with the platform loader (browser engines)");
+  }
+  return resolved;
+};
+
 // src/cli/bin.ts
-program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(package_default.version, "-v, --version", "output the current version");
-program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(webforaiCommand);
-program.parse();
+var HELP_EPILOGUE = `
+Examples:
+  $ webforai https://example.com/article              Markdown to stdout
+  $ webforai https://example.com -o article.md        write to a file
+  $ webforai ./page.html --extractor none             convert a local file, no extraction
+  $ webforai https://example.com --json | jq .markdown
+  $ webforai https://spa.example.com -l playwright    render JS in local Chromium
+  $ webforai https://example.com --engine browser     render via the hosted platform
+  $ webforai -i                                       guided interactive mode
+  $ webforai skill --install                          install the Agent Skill for this CLI
+
+Environment:
+  ${API_KEY_ENV}        platform API key (used by the platform loader)
+  ${PLATFORM_URL_ENV}   platform base URL override (self-hosted deployments)
+
+Exit codes:
+  0 success \xB7 1 runtime failure \xB7 2 usage error
+
+Output contract:
+  stdout carries only the Markdown (or the --json envelope); all logs go to stderr.
+`;
+program.name("webforai").description("Convert web pages and local HTML to clean, LLM-ready Markdown.").version(package_default.version, "-v, --version", "output the current version").addHelpText("after", HELP_EPILOGUE).showSuggestionAfterError();
+program.argument("[source]", "URL or local HTML file to convert").option("-o, --output <path>", "write Markdown to a file instead of stdout").option("-l, --loader <loader>", `how HTML is acquired (${LOADERS.join(" | ")}; local files are automatic)`).option("-m, --mode <mode>", `conversion mode (${MODES.join(" | ")}); ai strips links/tables/images`).option("--extractor <extractor>", `main-content extraction preset (${EXTRACTORS.join(" | ")})`, void 0).option("--frontmatter", "prepend YAML front matter built from the page metadata").option("--json", "print a JSON envelope {source, loader, url, markdown, metadata, ...} to stdout").option("--engine <engine>", "platform engine (fetch | browser | proxy-fetch | proxy-browser); implies -l platform").option("--region <region>", "platform proxy egress region (auto | us | eu | uk | jp | asia); implies -l platform").option("--screenshot", "platform browser engines: also capture a screenshot (expiring URL)").option("--api-key <key>", `platform API key (defaults to $${API_KEY_ENV})`).option("--platform-url <url>", `platform base URL (defaults to $${PLATFORM_URL_ENV} or the hosted instance)`).option("-i, --interactive", "guided prompt flow (the pre-v3 default)").option("-d, --debug", "verbose logs on stderr").action(async (source, flags) => {
+  if (flags.interactive) {
+    await interactiveCommand(source, flags);
+    return;
+  }
+  if (!source) {
+    program.help({ error: true });
+  }
+  await runCommand(resolveRunOptions(source, flags));
+});
+program.command("skill").description("print the Agent Skill (SKILL.md) for this CLI, or install it via the skills CLI").option("--install", "install through `npx skills add` (interactive agent picker; flags below forward to it)").option("-g, --global", "forward to skills add: install user-level instead of project-level").option("-a, --agent <agents>", "forward to skills add: agents to install to ('*' for all)").option("-y, --yes", "forward to skills add: skip confirmation prompts").option("--copy", "forward to skills add: copy files instead of symlinking").option("--all", "forward to skills add: all skills, all agents, no prompts").option("--dir <path>", "offline fallback: write <path>/webforai/SKILL.md directly, without the skills CLI").addHelpText("after", "\nAlso installable straight from the repository: npx skills add inaridiy/webforai\n").action(async (flags) => {
+  await skillCommand(flags);
+});
+var fail = (error) => {
+  if (error instanceof CommanderError) {
+    process.exit(error.exitCode === 0 ? 0 : 2);
+  }
+  if (error instanceof UsageError) {
+    console.error(`error: ${error.message}`);
+    process.exit(2);
+  }
+  console.error(error instanceof Error ? `error: ${error.message}` : `error: ${String(error)}`);
+  process.exit(1);
+};
+program.exitOverride();
+program.parseAsync().catch(fail);