webforai 3.0.0 → 4.1.0

package/dist/cjs/extractors/lib/takumi-signal.js

@@ -0,0 +1,76 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var takumi_signal_exports = {};
+__export(takumi_signal_exports, {
+  takumiSelection: () => takumiSelection
+});
+module.exports = __toCommonJS(takumi_signal_exports);
+var import_hast_fast = require("../../utils/hast-fast");
+var import_takumi = require("../presets/takumi");
+const MARK = "dataWebforaiIndex";
+const cloneElements = (root) => {
+  const copyOf = (node) => "children" in node ? {
+    ...node,
+    ...(0, import_hast_fast.isElement)(node) ? { properties: { ...node.properties } } : {},
+    children: [...node.children]
+  } : node;
+  const copy = copyOf(root);
+  const stack = [copy];
+  while (stack.length > 0) {
+    const parent = stack.pop();
+    if (!("children" in parent)) {
+      continue;
+    }
+    const children = parent.children;
+    for (let index = 0; index < children.length; index++) {
+      if ("children" in children[index]) {
+        children[index] = copyOf(children[index]);
+        stack.push(children[index]);
+      }
+    }
+  }
+  return copy;
+};
+const takumiSelection = (root, lang, url) => {
+  const elements = [];
+  (0, import_hast_fast.walk)(root, (node) => {
+    if ((0, import_hast_fast.isElement)(node)) {
+      node.properties[MARK] = elements.length;
+      elements.push(node);
+    }
+  });
+  const copy = cloneElements(root);
+  for (const element of elements) {
+    delete element.properties[MARK];
+  }
+  const kept = /* @__PURE__ */ new Set();
+  (0, import_hast_fast.walk)((0, import_takumi.takumiExtractor)({ hast: copy, lang, url, owned: true }), (node) => {
+    if ((0, import_hast_fast.isElement)(node)) {
+      const index = node.properties[MARK];
+      if (typeof index === "number" && elements[index]) {
+        kept.add(elements[index]);
+      }
+    }
+  });
+  return kept;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  takumiSelection
+});

package/dist/cjs/extractors/lib/ui-chrome.js

@@ -18,6 +18,8 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var ui_chrome_exports = {};
 __export(ui_chrome_exports, {
+  MAX_CONSENT_PLACEHOLDER_LENGTH: () => MAX_CONSENT_PLACEHOLDER_LENGTH,
+  isConsentPlaceholderText: () => isConsentPlaceholderText,
   isUiChromeText: () => isUiChromeText
 });
 module.exports = __toCommonJS(ui_chrome_exports);
@@ -113,7 +115,12 @@ const isUiChromeText = (text) => {
   }
   return UI_PHRASES.has(normalized);
 };
+const CONSENT_PLACEHOLDER = /^(this content (isn['’]?t|is not) (visible|available) (due to|because of) your (cookie|privacy) (preferences|settings)|we need your (consent|permission) to load|to (view|see|load) this (content|embed|video|post)[, ]+(please )?(enable|accept|allow|update)|(please )?(accept|allow|enable) (all |marketing |functional )?cookies to (view|see|load|watch)|このコンテンツを表示するには.{0,30}(cookie|クッキー|同意|許可))/i;
+const MAX_CONSENT_PLACEHOLDER_LENGTH = 400;
+const isConsentPlaceholderText = (text) => text.length <= MAX_CONSENT_PLACEHOLDER_LENGTH && CONSENT_PLACEHOLDER.test(text.trimStart());
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  MAX_CONSENT_PLACEHOLDER_LENGTH,
+  isConsentPlaceholderText,
   isUiChromeText
 });

package/dist/cjs/extractors/presets/auto.js

@@ -23,9 +23,9 @@ __export(auto_exports, {
 });
 module.exports = __toCommonJS(auto_exports);
 var import_adapters = require("../../adapters");
-var import_takumi = require("./takumi");
+var import_kiwame = require("./kiwame");
 const createAutoExtractor = (options = {}) => {
-  const { adapters } = options;
+  const { adapters, fallback = import_kiwame.kiwameExtractor } = options;
   return (params) => {
     if (adapters !== false) {
       const extracted = (0, import_adapters.runAdapters)({ hast: params.hast, url: params.url, lang: params.lang }, adapters ?? void 0);
@@ -33,7 +33,7 @@ const createAutoExtractor = (options = {}) => {
         return extracted;
       }
     }
-    return (0, import_takumi.takumiExtractor)(params);
+    return fallback(params);
   };
 };
 const autoExtractor = createAutoExtractor();

package/dist/cjs/extractors/presets/kiwame.js

@@ -0,0 +1,231 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var kiwame_exports = {};
+__export(kiwame_exports, {
+  DEFAULT_THRESHOLD: () => DEFAULT_THRESHOLD,
+  createKiwameExtractor: () => createKiwameExtractor,
+  kiwameExtractor: () => kiwameExtractor
+});
+module.exports = __toCommonJS(kiwame_exports);
+var import_hast_fast = require("../../utils/hast-fast");
+var import_block_features = require("../lib/block-features");
+var import_block_model = require("../lib/block-model");
+var import_block_model2 = require("../lib/block-model.generated");
+var import_blocks = require("../lib/blocks");
+var import_layout_tables = require("../lib/layout-tables");
+var import_sanitize = require("../lib/sanitize");
+var import_takumi_signal = require("../lib/takumi-signal");
+var import_takumi = require("./takumi");
+const DEFAULT_THRESHOLD = 0.47;
+const findBody = (hast) => (0, import_hast_fast.findElement)(hast, (element) => element.tagName === "body") ?? hast;
+const completeTables = (kept, blocks) => {
+  const rows = /* @__PURE__ */ new Set();
+  const tables = /* @__PURE__ */ new Set();
+  for (const block of blocks) {
+    if (!kept.has(block.owner)) {
+      continue;
+    }
+    const { row, table } = enclosingTable(block.frame);
+    if (row) {
+      rows.add(row);
+    }
+    if (table) {
+      tables.add(table);
+    }
+  }
+  for (const table of tables) {
+    const first = (0, import_hast_fast.findElement)(table, (element) => element.tagName === "tr");
+    if (first) {
+      rows.add(first);
+    }
+  }
+  for (const block of blocks) {
+    const { row } = enclosingTable(block.frame);
+    if (row && rows.has(row)) {
+      kept.add(block.owner);
+    }
+  }
+};
+const enclosingTable = (start) => {
+  let row;
+  for (let frame = start; frame; frame = frame.parent) {
+    const tag = frame.element.tagName;
+    if (tag === "tr" && !row) {
+      row = frame.element;
+    } else if (tag === "table") {
+      return { row, table: frame.element };
+    }
+  }
+  return { row };
+};
+const HEADING_LOOKAHEAD = 3;
+const keepSectionHeadings = (kept, blocks) => {
+  let seenKept = false;
+  for (let index = 0; index < blocks.length; index++) {
+    const block = blocks[index];
+    if (kept.has(block.owner)) {
+      seenKept = true;
+      continue;
+    }
+    if (!(seenKept && /^h[1-6]$/.test(block.owner.tagName))) {
+      continue;
+    }
+    for (let next = index + 1; next < Math.min(blocks.length, index + 1 + HEADING_LOOKAHEAD); next++) {
+      if (/^h[1-6]$/.test(blocks[next].owner.tagName)) {
+        break;
+      }
+      if (kept.has(blocks[next].owner)) {
+        kept.add(block.owner);
+        break;
+      }
+    }
+  }
+};
+const CODE_IN_CONTENT_THRESHOLD = 0.2;
+const CODE_CONTEXT = 6;
+const keepCodeInContent = (kept, blocks, probabilities) => {
+  const keptIndex = blocks.map((block) => kept.has(block.owner));
+  const near = (from, step) => {
+    for (let distance = 1; distance <= CODE_CONTEXT; distance++) {
+      const index = from + step * distance;
+      if (index < 0 || index >= blocks.length) {
+        return false;
+      }
+      if (keptIndex[index]) {
+        return true;
+      }
+    }
+    return false;
+  };
+  for (const block of blocks) {
+    if (block.owner.tagName === "pre" && !keptIndex[block.index] && probabilities[block.index] >= CODE_IN_CONTENT_THRESHOLD && near(block.index, -1) && near(block.index, 1)) {
+      kept.add(block.owner);
+    }
+  }
+};
+const listOf = (block) => {
+  if (block.owner.tagName !== "li") {
+    return void 0;
+  }
+  for (let frame = block.frame.parent; frame; frame = frame.parent) {
+    const tag = frame.element.tagName;
+    if (tag === "ul" || tag === "ol") {
+      return frame.element;
+    }
+  }
+  return void 0;
+};
+const keepAuthoredLists = (kept, blocks) => {
+  let seenKept = false;
+  for (let index = 0; index < blocks.length - 1; index++) {
+    const block = blocks[index];
+    const introducesList = kept.has(block.owner) && /[:：]\s*$/.test(block.text) || seenKept && /^h[1-6]$/.test(block.owner.tagName) && import_sanitize.REFERENCE_HEADING.test(block.text.trim());
+    seenKept ||= kept.has(block.owner);
+    const list = listOf(blocks[index + 1]);
+    if (!(introducesList && list)) {
+      continue;
+    }
+    kept.add(block.owner);
+    for (let next = index + 1; next < blocks.length && listOf(blocks[next]) === list; next++) {
+      kept.add(blocks[next].owner);
+    }
+  }
+};
+const pruneToBlocks = (element, kept, ownerKept) => {
+  const isBlock = element.type === "element" && (import_blocks.BLOCK_TAGS.has(element.tagName) || element.tagName === "pre");
+  const selfKept = isBlock ? kept.has(element) : ownerKept;
+  if (element.type === "element" && element.tagName === "pre") {
+    return selfKept;
+  }
+  let keepChildren = false;
+  const children = [];
+  for (const child of element.children) {
+    if (child.type === "element") {
+      if (pruneToBlocks(child, kept, selfKept)) {
+        children.push(child);
+        keepChildren = true;
+      }
+    } else if (child.type === "text") {
+      if (selfKept) {
+        children.push(child);
+      }
+    } else {
+      children.push(child);
+    }
+  }
+  element.children = children;
+  return selfKept || keepChildren;
+};
+const createKiwameExtractor = (options = {}) => {
+  const { threshold = DEFAULT_THRESHOLD, model = import_block_model2.BLOCK_MODEL, cleanup = true } = options;
+  const stackModel = options.stackModel === void 0 ? import_block_model2.BLOCK_STACK_MODEL : options.stackModel;
+  const usable = (0, import_block_model.isUsableModel)(model);
+  const stackUsable = stackModel ? (0, import_block_model.isUsableModel)(stackModel, import_block_features.STACKED_FEATURE_COUNT) : false;
+  return (params) => {
+    if (!usable) {
+      return (0, import_takumi.takumiExtractor)(params);
+    }
+    const { hast, lang, owned } = params;
+    const body = owned ? findBody(hast) : (0, import_hast_fast.cloneHast)(findBody(hast));
+    (0, import_sanitize.stripNonContent)(body);
+    const blocks = (0, import_blocks.segmentBlocks)(body);
+    if (blocks.length === 0 || !((0, import_hast_fast.isElement)(body) || body.type === "root")) {
+      return body;
+    }
+    const collector = new import_hast_fast.MetricsCollector();
+    const takumiKept = (0, import_takumi_signal.takumiSelection)(body, lang, params.url);
+    const features = (0, import_block_features.blockFeatures)({ root: body, blocks, collector, lang, takumiKept });
+    let probabilities = (0, import_block_model.scoreBlocks)(features, blocks.length, model, import_block_features.FEATURE_COUNT);
+    if (stackModel && stackUsable) {
+      const stacked = (0, import_block_features.stackedFeatures)(blocks, features, probabilities);
+      probabilities = (0, import_block_model.scoreBlocks)(stacked, blocks.length, stackModel, import_block_features.STACKED_FEATURE_COUNT);
+    }
+    const kept = /* @__PURE__ */ new Set();
+    for (const block of blocks) {
+      if (probabilities[block.index] >= threshold) {
+        kept.add(block.owner);
+      }
+    }
+    if (kept.size === 0) {
+      return (0, import_takumi.takumiExtractor)({ hast: body, lang, url: params.url, owned: true });
+    }
+    keepCodeInContent(kept, blocks, probabilities);
+    keepSectionHeadings(kept, blocks);
+    keepAuthoredLists(kept, blocks);
+    completeTables(kept, blocks);
+    pruneToBlocks(body, kept, false);
+    (0, import_layout_tables.unwrapLayoutTables)(body);
+    if (!cleanup) {
+      return body;
+    }
+    (0, import_hast_fast.walk)(body, (node) => {
+      if ((0, import_hast_fast.isElement)(node) && (node.tagName === "form" || node.tagName === "fieldset")) {
+        node.tagName = "div";
+      }
+    });
+    return (0, import_sanitize.cleanContent)(body, new import_hast_fast.MetricsCollector());
+  };
+};
+const kiwameExtractor = createKiwameExtractor();
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DEFAULT_THRESHOLD,
+  createKiwameExtractor,
+  kiwameExtractor
+});

package/dist/cjs/extractors/presets/takumi.js

@@ -27,6 +27,7 @@ var import_sanitize = require("../lib/sanitize");
 var import_score = require("../lib/score");
 var import_terminators = require("../lib/terminators");
 const MIN_RETAINED_FRACTION = 0.25;
+const FURNITURE_LINK_DENSITY = 0.5;
 const SEMANTIC_SELECTORS = [
   (element) => (0, import_hast_fast.stringProperty)(element, "itemprop")?.includes("articleBody") ?? false,
   (element) => element.tagName === "article",
@@ -64,6 +65,31 @@ const collectMatching = (tree, matches, into) => {
     }
   }
 };
+const containsTag = (element, tagName) => element.children.some((child) => (0, import_hast_fast.isElement)(child) && (child.tagName === tagName || containsTag(child, tagName)));
+const removesEveryTitle = (root, doomed) => {
+  if (!doomed.some((element) => containsTag(element, "h1"))) {
+    return false;
+  }
+  const doomedSet = new Set(doomed);
+  let survivor = false;
+  const visit = (node) => {
+    if (survivor || !("children" in node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (!(0, import_hast_fast.isElement)(child) || doomedSet.has(child)) {
+        continue;
+      }
+      if (child.tagName === "h1") {
+        survivor = true;
+        return;
+      }
+      visit(child);
+    }
+  };
+  visit(root);
+  return !survivor;
+};
 const asRootOf = (nodes) => ({ type: "root", children: nodes });
 const scoredSelection = (tree, collector, lang) => {
   const context = (0, import_score.buildScoringContext)(tree, collector, lang);
@@ -90,9 +116,14 @@ const takumiExtractor = (params) => {
   const shortcut = semanticShortcut(body, collector, minLength);
   const searchRoot = shortcut ?? body;
   const searchText = collector.textLength(searchRoot);
-  const doomed = (0, import_sanitize.findUnlikelyElements)(searchRoot);
-  const doomedText = doomed.reduce((sum, element) => sum + collector.metrics(element).text, 0);
-  if (searchText - doomedText > Math.max(minLength, searchText * MIN_RETAINED_FRACTION)) {
+  let doomed = (0, import_sanitize.findUnlikelyElements)(searchRoot);
+  const textOf = (elements) => elements.reduce((sum, element) => sum + collector.metrics(element).text, 0);
+  let acceptable = searchText - textOf(doomed) > Math.max(minLength, searchText * MIN_RETAINED_FRACTION);
+  if (!acceptable) {
+    doomed = (0, import_sanitize.findUnlikelyElements)(searchRoot, (element) => collector.linkDensity(element) < FURNITURE_LINK_DENSITY);
+    acceptable = doomed.length > 0 && searchText - textOf(doomed) >= minLength && !removesEveryTitle(searchRoot, doomed);
+  }
+  if (acceptable) {
     const doomedSet = new Set(doomed);
     (0, import_hast_fast.pruneInPlace)(searchRoot, (node) => !((0, import_hast_fast.isElement)(node) && doomedSet.has(node)));
     collector = new import_hast_fast.MetricsCollector();

package/dist/cjs/html-to-markdown.js

@@ -18,18 +18,19 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var html_to_markdown_exports = {};
 __export(html_to_markdown_exports, {
+  headingTitle: () => headingTitle,
   htmlToMarkdown: () => htmlToMarkdown,
   htmlToMarkdownWithMetadata: () => htmlToMarkdownWithMetadata
 });
 module.exports = __toCommonJS(html_to_markdown_exports);
-var import_hast_util_from_html = require("hast-util-from-html");
+var import_parse_html = require("./utils/parse-html");
 var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
 var import_metadata = require("./metadata");
 var import_hast_utils = require("./utils/hast-utils");
 const htmlToMarkdownWithMetadata = (htmlOrHast, options) => {
   const { baseUrl, frontmatter, title, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
-  const hast = typeof htmlOrHast === "string" ? (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true }) : htmlOrHast;
+  const hast = typeof htmlOrHast === "string" ? (0, import_parse_html.parseHtml)(htmlOrHast, { fragment: true }) : htmlOrHast;
   const isOwned = typeof htmlOrHast === "string";
   const declaredLang = typeof htmlOrHast === "string" ? (0, import_hast_utils.getLangFromStr)(htmlOrHast) : void 0;
   const metadata = (0, import_metadata.extractMetadata)(hast);
@@ -43,10 +44,54 @@ const htmlToMarkdownWithMetadata = (htmlOrHast, options) => {
     url: toMdastOptions.url ?? metadata.canonicalUrl
   });
   const body = (0, import_mdast_to_markdown.mdastToMarkdown)(mdast, { baseUrl, ...toMarkdownOptions });
-  const titled = title === false ? body : withTitle(body, metadata.title);
+  const pageUrl = toMdastOptions.url ?? metadata.canonicalUrl;
+  const titled = title === false ? body : withTitle(body, headingTitle(metadata.title, metadata.siteName, pageUrl));
   const markdown = frontmatter ? `${(0, import_metadata.toFrontmatter)(metadata)}${titled}` : titled;
   return { markdown, metadata };
 };
+const TITLE_SEPARATOR = /\s+[-|–—·•:]\s+|\s*[｜|]\s*/;
+const SITE_SECTION = /^(docs|documentation|blog|news|wiki|developers?|devblog|help|support|reference)$/;
+const compact = (value) => value.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "");
+const siteNames = (siteName, url) => {
+  const names = siteName ? siteName.split(TITLE_SEPARATOR).map(compact) : [];
+  try {
+    const labels = url ? new URL(url).hostname.split(".") : [];
+    const meaningful = labels.slice(0, -1).filter((label) => label !== "www");
+    names.push(...meaningful.map(compact), compact(meaningful.join("")), compact(labels.join("").replace(/^www/, "")));
+  } catch {
+  }
+  return names.filter((name) => name.length >= 3);
+};
+const headingTitle = (title, siteName, url) => {
+  if (!title) {
+    return title;
+  }
+  const segments = title.split(TITLE_SEPARATOR);
+  if (segments.length < 2) {
+    return title;
+  }
+  const names = siteNames(siteName, url);
+  if (names.length === 0) {
+    return title;
+  }
+  const isSite = (segment) => {
+    const value = compact(segment);
+    return value.length > 0 && names.some((name) => value === name || value.startsWith(name) && SITE_SECTION.test(value.slice(name.length)));
+  };
+  const last = segments[segments.length - 1];
+  if (isSite(last)) {
+    return stripSegment(title, last, "end");
+  }
+  if (siteName && compact(segments[0]) === compact(siteName)) {
+    return stripSegment(title, segments[0], "start");
+  }
+  return title;
+};
+const stripSegment = (title, segment, edge) => {
+  const rest = edge === "end" ? title.slice(0, title.lastIndexOf(segment)) : title.slice(segment.length);
+  const trimmed = rest.replace(edge === "end" ? /[\s\-|–—·•:｜]+$/ : /^[\s\-|–—·•:｜]+/, "");
+  return trimmed.length > 0 ? trimmed : title;
+};
 const TITLE_LOOKAHEAD = 400;
 const withTitle = (body, title) => {
   if (!title) {
@@ -56,17 +101,26 @@ const withTitle = (body, title) => {
   if (/^#\s+\S/m.test(opening)) {
     return body;
   }
-  if (normalizeForCompare(opening).includes(normalizeForCompare(title))) {
+  if (mentionsTitle(opening, title)) {
     return body;
   }
   return `# ${title}
 
 ${body}`;
 };
+const MIN_EMBEDDED_TITLE_LENGTH = 24;
+const mentionsTitle = (opening, title) => {
+  const wanted = normalizeForCompare(title);
+  if (wanted.length >= MIN_EMBEDDED_TITLE_LENGTH) {
+    return normalizeForCompare(opening).includes(wanted);
+  }
+  return opening.split("\n").some((line) => normalizeForCompare(line.replace(/[#*_`>]/g, "")) === wanted);
+};
 const normalizeForCompare = (value) => value.replace(/\s+/g, " ").trim().toLowerCase();
 const htmlToMarkdown = (htmlOrHast, options) => htmlToMarkdownWithMetadata(htmlOrHast, options).markdown;
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  headingTitle,
   htmlToMarkdown,
   htmlToMarkdownWithMetadata
 });

package/dist/cjs/html-to-mdast.js

@@ -21,8 +21,8 @@ __export(html_to_mdast_exports, {
   htmlToMdast: () => htmlToMdast
 });
 module.exports = __toCommonJS(html_to_mdast_exports);
-var import_hast_util_from_html = require("hast-util-from-html");
 var import_hast_util_to_mdast = require("hast-util-to-mdast");
+var import_parse_html = require("./utils/parse-html");
 var import_extract_mdast = require("./extract-mdast");
 var import_extractors = require("./extractors");
 var import_custom_a_handler = require("./mdast-handlers/custom-a-handler");
@@ -34,13 +34,14 @@ var import_definition_list_handler = require("./mdast-handlers/definition-list-h
 var import_math_handler = require("./mdast-handlers/math-handler");
 var import_sup_sub_handler = require("./mdast-handlers/sup-sub-handler");
 var import_normalize = require("./normalize");
+var import_hast_fast = require("./utils/hast-fast");
 var import_hast_utils = require("./utils/hast-utils");
 const htmlToMdast = (htmlOrHast, options) => {
   const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
-  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
+  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_parse_html.parseHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
   const isOwned = typeof htmlOrHast === "string" || (options?.owned ?? false);
   const url = defaultUrl || (0, import_hast_utils.getUrlFromHast)(hast);
-  const extractedHast = (0, import_extractors.pipeExtractors)({ hast, lang, url, owned: isOwned }, extractors);
+  const extractedHast = (0, import_extractors.pipeExtractors)({ hast: isOwned ? hast : (0, import_hast_fast.cloneHast)(hast), lang, url, owned: true }, extractors);
   (0, import_normalize.normalizeHast)(extractedHast, options?.normalize);
   const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
     handlers: {

package/dist/cjs/index.js

@@ -19,15 +19,22 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var src_exports = {};
 __export(src_exports, {
   BUILTIN_ADAPTERS: () => import_adapters.BUILTIN_ADAPTERS,
+  MAX_META_REFRESH_DELAY_SECONDS: () => import_extract_meta_refresh.MAX_META_REFRESH_DELAY_SECONDS,
+  SHELL_EMPTY_THRESHOLD: () => import_detect_client_shell.SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD: () => import_detect_client_shell.SHELL_SUSPICION_THRESHOLD,
   adapterById: () => import_adapters.adapterById,
   autoExtractor: () => import_extractors.autoExtractor,
   buildPageSignature: () => import_adapters.buildPageSignature,
   createAutoExtractor: () => import_extractors.createAutoExtractor,
+  createKiwameExtractor: () => import_extractors.createKiwameExtractor,
   defineSelectorAdapter: () => import_adapters.defineSelectorAdapter,
+  detectClientShell: () => import_detect_client_shell.detectClientShell,
+  extractMetaRefresh: () => import_extract_meta_refresh.extractMetaRefresh,
   extractMetadata: () => import_metadata.extractMetadata,
   htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
   htmlToMarkdownWithMetadata: () => import_html_to_markdown.htmlToMarkdownWithMetadata,
   htmlToMdast: () => import_html_to_mdast.htmlToMdast,
+  kiwameExtractor: () => import_extractors.kiwameExtractor,
   mdastSplitter: () => import_md_splitter.mdastSplitter,
   mdastToMarkdown: () => import_mdast_to_markdown.mdastToMarkdown,
   minimalFilter: () => import_extractors.minimalFilter,
@@ -46,20 +53,29 @@ var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
 var import_extractors = require("./extractors");
 var import_adapters = require("./adapters");
+var import_detect_client_shell = require("./detect-client-shell");
+var import_extract_meta_refresh = require("./extract-meta-refresh");
 var import_metadata = require("./metadata");
 var import_normalize = require("./normalize");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BUILTIN_ADAPTERS,
+  MAX_META_REFRESH_DELAY_SECONDS,
+  SHELL_EMPTY_THRESHOLD,
+  SHELL_SUSPICION_THRESHOLD,
   adapterById,
   autoExtractor,
   buildPageSignature,
   createAutoExtractor,
+  createKiwameExtractor,
   defineSelectorAdapter,
+  detectClientShell,
+  extractMetaRefresh,
   extractMetadata,
   htmlToMarkdown,
   htmlToMarkdownWithMetadata,
   htmlToMdast,
+  kiwameExtractor,
   mdastSplitter,
   mdastToMarkdown,
   minimalFilter,

package/dist/cjs/link-replacer.js

@@ -21,19 +21,25 @@ __export(link_replacer_exports, {
   linkReplacer: () => linkReplacer
 });
 module.exports = __toCommonJS(link_replacer_exports);
-const linkReplacer = (markdown, base) => {
-  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
-  return markdown.replace(regex, (match, pre, url, post) => {
-    if (/^(https?:|#)/.test(url)) {
-      return match;
-    }
-    try {
-      const absoluteUrl = new URL(url, base).href;
-      return `${pre}${absoluteUrl}${post}`;
-    } catch {
-      return match;
-    }
-  });
+const absoluteUrl = (url, base) => {
+  if (/^(?:[a-z][a-z\d+.-]*:|#)/i.test(url)) {
+    return url;
+  }
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return url;
+  }
+};
+const linkReplacer = (node, base) => {
+  let result = node;
+  if (node.type === "link" || node.type === "image" || node.type === "definition") {
+    result = { ...node, url: absoluteUrl(node.url, base) };
+  }
+  if ("children" in result) {
+    return { ...result, children: result.children.map((child) => linkReplacer(child, base)) };
+  }
+  return result;
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/loaders/cf-puppeteer.js

@@ -32,15 +32,22 @@ __export(cf_puppeteer_exports, {
 });
 module.exports = __toCommonJS(cf_puppeteer_exports);
 var import_puppeteer = __toESM(require("@cloudflare/puppeteer"), 1);
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const loadHtml = async (url, ctx) => {
   const browser = await import_puppeteer.default.launch(ctx);
-  const page = await browser.newPage();
-  await page.goto(url);
-  const html = await page.content();
-  await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
-  await page.close();
-  return html;
+  try {
+    const page = await browser.newPage();
+    await page.goto(url);
+    try {
+      await page.waitForNetworkIdle({ timeout: 1e4 });
+    } catch (error) {
+      if (!(error instanceof import_puppeteer.TimeoutError)) {
+        throw error;
+      }
+    }
+    return await page.content();
+  } finally {
+    await browser.close();
+  }
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {