webforai 2.1.1 → 3.0.0

package/dist/extractors/lib/terminators.js

@@ -0,0 +1,107 @@
+// src/extractors/lib/terminators.ts
+import { isElement, isParent, pruneInPlace } from "../../utils/hast-fast.js";
+var TERMINATOR_PATTERNS = [
+  /^was this (page|article|document) helpful/i,
+  /^thanks for the feedback/i,
+  /^(this page was )?last (modified|updated) (on|at|:)?\s/i,
+  /^learn how to contribute/i,
+  /^help (us )?improve\b/i,
+  // The stranded Yes/No pair of a feedback widget whose question was removed separately.
+  /^yes ?no$/i,
+  /^(related (articles?|posts?|stories|content)|you might also like|recommended for you|more from|read next)$/i,
+  /^(comments?|leave a (comment|reply)|join the (conversation|discussion))$/i,
+  /^(この(ページ|記事)は(参考|役に立ち)ました?か)/,
+  /^(関連記事|あわせて読みたい|こちらもおすすめ|おすすめ記事)$/,
+  /^(コメントを(残す|書く)|コメント一覧)$/,
+  /^最終更新日?[:：\s]/
+];
+var TAIL_START_FRACTION = 0.8;
+var MAX_TERMINATOR_BLOCK_LENGTH = 500;
+var BLOCK_TAGS = /* @__PURE__ */ new Set(["p", "div", "section", "footer", "aside", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol"]);
+var isTerminatorBlock = (element, start, tailStart) => {
+  if (!BLOCK_TAGS.has(element.tagName) || start < tailStart) {
+    return false;
+  }
+  const text = textOf(element).replace(/\s+/g, " ").trim();
+  if (text.length === 0 || text.length > MAX_TERMINATOR_BLOCK_LENGTH) {
+    return false;
+  }
+  return TERMINATOR_PATTERNS.some((pattern) => pattern.test(text));
+};
+var textOf = (node) => {
+  if (node.type === "text") {
+    return node.value;
+  }
+  if (!isParent(node)) {
+    return "";
+  }
+  let text = "";
+  for (const child of node.children) {
+    text += textOf(child);
+  }
+  return text;
+};
+var truncateTrailingBoilerplate = (tree) => {
+  const starts = /* @__PURE__ */ new Map();
+  let total = 0;
+  const measure = (node) => {
+    starts.set(node, total);
+    if (node.type === "text") {
+      total += node.value.trim().length;
+      return;
+    }
+    if (isParent(node)) {
+      for (const child of node.children) {
+        measure(child);
+      }
+    }
+  };
+  measure(tree);
+  if (total === 0) {
+    return false;
+  }
+  const tailStart = total * TAIL_START_FRACTION;
+  let cutoff = -1;
+  const find = (node) => {
+    if (cutoff !== -1 || !isParent(node)) {
+      return;
+    }
+    for (const child of node.children) {
+      if (cutoff !== -1) {
+        return;
+      }
+      if (!isElement(child)) {
+        continue;
+      }
+      const start = starts.get(child) ?? 0;
+      if (isTerminatorBlock(child, start, tailStart)) {
+        cutoff = start;
+        return;
+      }
+      find(child);
+    }
+  };
+  find(tree);
+  if (cutoff === -1) {
+    return false;
+  }
+  pruneInPlace(tree, (node) => {
+    if (node === tree) {
+      return true;
+    }
+    const start = starts.get(node);
+    if (start === void 0) {
+      return true;
+    }
+    if (start >= cutoff) {
+      return false;
+    }
+    return true;
+  });
+  return true;
+};
+var TERMINATOR_TAIL_FRACTION = TAIL_START_FRACTION;
+export {
+  TERMINATOR_TAIL_FRACTION,
+  truncateTrailingBoilerplate
+};

package/dist/extractors/lib/ui-chrome.js

@@ -0,0 +1,96 @@
+// src/extractors/lib/ui-chrome.ts
+var UI_PHRASES = /* @__PURE__ */ new Set([
+  // Documentation and article controls
+  "copy",
+  "copy page",
+  "copy link",
+  "copy code",
+  "copied",
+  "edit this page",
+  "edit on github",
+  "on this page",
+  "in this article",
+  "table of contents",
+  "contents",
+  "print",
+  "print this page",
+  "back to top",
+  "scroll to top",
+  "skip to content",
+  "skip to main content",
+  "permalink",
+  "anchor",
+  // Feedback widgets
+  "was this page helpful",
+  "was this helpful",
+  "was this article helpful",
+  "yes no",
+  "feedback",
+  "send feedback",
+  "report an issue",
+  "rate this page",
+  // Sharing and subscription
+  "share",
+  "share this",
+  "share this article",
+  "share on twitter",
+  "share on facebook",
+  "subscribe",
+  "sign up",
+  "sign in",
+  "log in",
+  "login",
+  "newsletter",
+  // Advertising and promotion
+  "advertisement",
+  "advertisements",
+  "sponsored",
+  "sponsored content",
+  "promoted",
+  // Navigation affordances
+  "read more",
+  "show more",
+  "show less",
+  "see more",
+  "load more",
+  "previous",
+  "next",
+  "previous page",
+  "next page",
+  "menu",
+  "close",
+  "search",
+  // Japanese equivalents
+  "\u76EE\u6B21",
+  "\u5E83\u544A",
+  "\u30B7\u30A7\u30A2",
+  "\u5171\u6709",
+  "\u3053\u306E\u8A18\u4E8B\u3092\u30B7\u30A7\u30A2",
+  "\u7D9A\u304D\u3092\u8AAD\u3080",
+  "\u3082\u3063\u3068\u898B\u308B",
+  "\u95A2\u9023\u8A18\u4E8B",
+  "\u30B9\u30DD\u30F3\u30B5\u30FC\u30EA\u30F3\u30AF",
+  "\u30DA\u30FC\u30B8\u306E\u5148\u982D\u3078",
+  "\u30C8\u30C3\u30D7\u3078\u623B\u308B",
+  "\u524D\u3078",
+  "\u6B21\u3078",
+  "\u9589\u3058\u308B",
+  "\u691C\u7D22",
+  "\u30ED\u30B0\u30A4\u30F3",
+  "\u4F1A\u54E1\u767B\u9332"
+]);
+var MAX_PHRASE_LENGTH = 40;
+var normalize = (text) => text.toLowerCase().replace(/[?!.,:;·・…"'“”‘’()[\]{}<>|/\\—–-]/g, " ").replace(/\s+/g, " ").trim();
+var isUiChromeText = (text) => {
+  if (text.length === 0 || text.length > MAX_PHRASE_LENGTH) {
+    return false;
+  }
+  const normalized = normalize(text);
+  if (normalized.length === 0) {
+    return false;
+  }
+  return UI_PHRASES.has(normalized);
+};
+export {
+  isUiChromeText
+};

package/dist/extractors/pipeExtractors.js

@@ -1,18 +1,21 @@
 // src/extractors/pipeExtractors.ts
 import { DEFAULT_EXTRACTORS } from "../constants.js";
 var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
-  const { hast, lang } = params;
-  const _extractors = Array.isArray(extractors) ? extractors : [extractors];
-  const extracted = _extractors.reduce((acc, extractor) => {
+  const { hast, lang, url, owned } = params;
+  const list = Array.isArray(extractors) ? extractors : [extractors];
+  let current = hast;
+  let currentOwned = owned ?? false;
+  for (const extractor of list) {
     if (extractor === false) {
-      return acc;
+      continue;
     }
-    if (typeof extractor === "function") {
-      return extractor({ hast: acc, lang });
+    if (typeof extractor !== "function") {
+      throw new Error(`Invalid extractor: ${String(extractor)}`);
     }
-    throw new Error(`Invalid extractor: ${extractor}`);
-  }, hast) || hast;
-  return extracted;
+    current = extractor({ hast: current, lang, url, owned: currentOwned });
+    currentOwned = true;
+  }
+  return current;
 };
 export {
   pipeExtractors

package/dist/extractors/presets/auto.js

@@ -0,0 +1,20 @@
+// src/extractors/presets/auto.ts
+import { runAdapters } from "../../adapters/index.js";
+import { takumiExtractor } from "./takumi.js";
+var createAutoExtractor = (options = {}) => {
+  const { adapters } = options;
+  return (params) => {
+    if (adapters !== false) {
+      const extracted = runAdapters({ hast: params.hast, url: params.url, lang: params.lang }, adapters ?? void 0);
+      if (extracted) {
+        return extracted;
+      }
+    }
+    return takumiExtractor(params);
+  };
+};
+var autoExtractor = createAutoExtractor();
+export {
+  autoExtractor,
+  createAutoExtractor
+};

package/dist/extractors/presets/minimal-filter.js

@@ -1,63 +1,51 @@
 // src/extractors/presets/minimal-filter.ts
-import { select } from "hast-util-select";
-import { toString as hastToString } from "hast-util-to-string";
-import { filter } from "unist-util-filter";
-import { classnames, isStrInclude, matchString } from "./utils.js";
-var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
-var REGEXPS = {
-  hidden: /hidden|invisible|fallback-image/i,
-  byline: /byline|author|dateline|writtenby|p-author/i,
-  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
-  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
-};
-var metadataFilter = (node) => {
-  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
-};
-var universalElementFilter = (node) => {
-  if (node.type !== "element") {
+import {
+  MetricsCollector,
+  cloneHast,
+  findElement,
+  isElement,
+  isTruthyAttribute,
+  matchString,
+  pruneInPlace,
+  stringProperty
+} from "../../utils/hast-fast.js";
+import { REGEXPS, UNLIKELY_ROLES } from "../lib/constants.js";
+import { stripNonContent } from "../lib/sanitize.js";
+var MIN_RETAINED_FRACTION = 1 / 3;
+var ALWAYS_ACCEPT_LENGTH = 5e3;
+var isChrome = (element) => {
+  if (element.tagName === "aside" || element.tagName === "nav" || element.tagName === "dialog") {
     return true;
   }
-  const element = node;
-  if (["aside", "nav"].includes(element.tagName)) {
-    return false;
-  }
-  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
-    return false;
-  }
-  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
-    return false;
-  }
-  if (element.tagName === "dialog") {
-    return false;
+  const role = stringProperty(element, "role");
+  if (role && UNLIKELY_ROLES.has(role)) {
+    return true;
   }
-  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
-    return false;
+  if (role === "dialog" && isTruthyAttribute(element, "aria-modal")) {
+    return true;
   }
-  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
-    return false;
+  if (stringProperty(element, "rel") === "author") {
+    return true;
   }
   if (REGEXPS.byline.test(matchString(element))) {
-    return false;
-  }
-  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
-    return false;
+    return true;
   }
-  return true;
+  return false;
 };
 var minimalFilter = (params) => {
   const { hast } = params;
-  const body = select("body", hast) ?? hast;
-  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
-  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
-  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+  const body = cloneHast(findElement(hast, (element) => element.tagName === "body") ?? hast);
+  stripNonContent(body);
+  const collector = new MetricsCollector();
+  const baseText = collector.textLength(body);
+  if (baseText === 0) {
     return body;
   }
-  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
-  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
-  const isOverExtracted = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3;
-  const baseTree = isOverExtracted && baseFilterd ? baseFilterd : metadataFilteredHast;
-  return baseTree;
+  const filtered = cloneHast(body);
+  pruneInPlace(filtered, (node) => !(isElement(node) && isChrome(node)));
+  const filteredText = new MetricsCollector().textLength(filtered);
+  const isOverExtracted = filteredText < baseText * MIN_RETAINED_FRACTION && filteredText < ALWAYS_ACCEPT_LENGTH;
+  return isOverExtracted ? body : filtered;
 };
 export {
   minimalFilter

package/dist/extractors/presets/takumi.js

@@ -1,134 +1,103 @@
 // src/extractors/presets/takumi.ts
-import { select, selectAll } from "hast-util-select";
-import { toString as hastToString } from "hast-util-to-string";
-import { filter } from "unist-util-filter";
-import { classnames, isStrInclude, matchString } from "./utils.js";
-var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
-var REGEXPS = {
-  hidden: /hidden|invisible|fallback-image/i,
-  byline: /byline|author|dateline|writtenby|p-author/i,
-  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
-  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
-};
-var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
-var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
-var CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
-var TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
-var BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
-var metadataFilter = (node) => {
-  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
-};
-var universalElementFilter = (node) => {
-  if (node.type !== "element") {
-    return true;
-  }
-  const element = node;
-  if (["aside", "nav"].includes(element.tagName)) {
-    return false;
-  }
-  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
-    return false;
-  }
-  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
-    return false;
-  }
-  if (element.tagName === "dialog") {
-    return false;
-  }
-  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
-    return false;
-  }
-  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
-    return false;
-  }
-  if (REGEXPS.byline.test(matchString(element))) {
-    return false;
-  }
-  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
-    return false;
+import {
+  MetricsCollector,
+  cloneHast,
+  findElement,
+  isElement,
+  pruneInPlace,
+  stringProperty
+} from "../../utils/hast-fast.js";
+import { minContentLength } from "../lib/constants.js";
+import { cleanContent, findCleanupTargets, findUnlikelyElements, stripNonContent } from "../lib/sanitize.js";
+import { buildScoringContext, climbToArticleBoundary, collectArticleNodes, scoreCandidates } from "../lib/score.js";
+import { truncateTrailingBoilerplate } from "../lib/terminators.js";
+var MIN_RETAINED_FRACTION = 0.25;
+var SEMANTIC_SELECTORS = [
+  (element) => stringProperty(element, "itemprop")?.includes("articleBody") ?? false,
+  (element) => element.tagName === "article",
+  (element) => element.tagName === "main",
+  (element) => stringProperty(element, "role") === "main"
+];
+var findBody = (hast) => findElement(hast, (element) => element.tagName === "body") ?? hast;
+var semanticShortcut = (tree, collector, minLength) => {
+  for (const matches of SEMANTIC_SELECTORS) {
+    const found = [];
+    collectMatching(tree, matches, found);
+    if (found.length !== 1) {
+      continue;
+    }
+    const candidate = found[0];
+    const metrics = collector.metrics(candidate);
+    if (metrics.text >= minLength && collector.linkDensity(candidate) < 0.5) {
+      return candidate;
+    }
   }
-  return true;
+  return void 0;
 };
-var unlikelyElementFilter = (node) => {
-  if (node.type !== "element") {
-    return true;
-  }
-  const element = node;
-  if (CONTENTABLE_TAGS.includes(element.tagName)) {
-    return true;
-  }
-  const match = matchString(element);
-  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
-    return false;
-  }
-  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
-    return false;
+var collectMatching = (tree, matches, into) => {
+  const stack = [tree];
+  while (stack.length > 0 && into.length < 2) {
+    const node = stack.pop();
+    if (isElement(node) && matches(node)) {
+      into.push(node);
+      continue;
+    }
+    if ("children" in node) {
+      for (const child of node.children) {
+        stack.push(child);
+      }
+    }
   }
-  return true;
 };
-var removeEmptyFilter = (node, _lang) => {
-  if (node.type !== "element") {
-    return true;
-  }
-  const element = node;
-  if (PARAGRAPH_TAGS.includes(element.tagName)) {
-    return true;
-  }
-  if (element.tagName === "img" && !element.properties.src) {
-    return false;
-  }
-  if (TEXTABLE_TAGS.includes(element.tagName) && hastToString(element).length === 0) {
-    return false;
-  }
-  return true;
+var asRootOf = (nodes) => ({ type: "root", children: nodes });
+var scoredSelection = (tree, collector, lang) => {
+  const context = buildScoringContext(tree, collector, lang);
+  const candidates = scoreCandidates(tree, context);
+  const top = candidates[0];
+  if (!top) {
+    return void 0;
+  }
+  const promoted = climbToArticleBoundary(top, candidates, context);
+  const anchor = promoted === top.element ? top : { element: promoted, score: top.score };
+  const selected = collectArticleNodes(anchor, candidates, context);
+  return selected.length === 1 ? selected[0] : asRootOf(selected);
 };
 var takumiExtractor = (params) => {
-  const { hast, lang = "en" } = params;
-  const body = select("body", hast) ?? hast;
-  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
-  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
-  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+  const { hast, lang, owned } = params;
+  const body = owned ? findBody(hast) : cloneHast(findBody(hast));
+  stripNonContent(body);
+  let collector = new MetricsCollector();
+  const baseText = collector.textLength(body);
+  if (baseText === 0) {
     return body;
   }
-  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
-  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
-  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
-  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
-  if (baseText.length < minimalLength) {
-    minimalLength = Math.max(0, baseText.length - 200);
-  }
-  let extractedTree = baseTree;
-  let extractedText = baseText;
-  for (const selector of BODY_SELECTORS) {
-    const content = { type: "root", children: selectAll(selector, baseFilterd) };
-    const contentText = hastToString(content);
-    if (contentText.length < 25) {
-      continue;
-    }
-    const links = selectAll("a", content);
-    const linkText = links.map((link) => hastToString(link)).join("");
-    const linkDensity = linkText.length / contentText.length;
-    if (linkDensity > 0.4) {
-      continue;
-    }
-    if (contentText.length > minimalLength) {
-      extractedTree = content;
-      extractedText = contentText;
-      break;
-    }
-  }
-  const finalFilteredTree = filter(extractedTree, (node) => {
-    if (!removeEmptyFilter(node, lang)) {
-      return false;
-    }
-    if (!unlikelyElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
-  return finalTree;
+  const minLength = Math.min(minContentLength(lang), Math.max(0, baseText - 200));
+  const shortcut = semanticShortcut(body, collector, minLength);
+  const searchRoot = shortcut ?? body;
+  const searchText = collector.textLength(searchRoot);
+  const doomed = findUnlikelyElements(searchRoot);
+  const doomedText = doomed.reduce((sum, element) => sum + collector.metrics(element).text, 0);
+  if (searchText - doomedText > Math.max(minLength, searchText * MIN_RETAINED_FRACTION)) {
+    const doomedSet = new Set(doomed);
+    pruneInPlace(searchRoot, (node) => !(isElement(node) && doomedSet.has(node)));
+    collector = new MetricsCollector();
+  }
+  const scoringRoot = searchRoot;
+  const scoringRootText = collector.textLength(scoringRoot);
+  const selection = shortcut ? scoringRoot : scoredSelection(scoringRoot, collector, lang) ?? scoringRoot;
+  const selectionText = collector.textLength(selection);
+  const selectionIsSound = selectionText >= minLength || selectionText > scoringRootText * MIN_RETAINED_FRACTION;
+  const chosen = selectionIsSound ? selection : scoringRoot;
+  truncateTrailingBoilerplate(chosen);
+  const cleanupCollector = new MetricsCollector();
+  const chosenText = cleanupCollector.textLength(chosen);
+  const widgets = findCleanupTargets(chosen, cleanupCollector);
+  const widgetText = widgets.reduce((sum, element) => sum + cleanupCollector.metrics(element).text, 0);
+  const remainingText = chosenText - widgetText;
+  if (remainingText < chosenText * MIN_RETAINED_FRACTION && remainingText < minLength) {
+    return chosen;
+  }
+  return cleanContent(chosen, cleanupCollector);
 };
 export {
   takumiExtractor

package/dist/html-to-markdown.js

@@ -1,12 +1,48 @@
 // src/html-to-markdown.ts
+import { fromHtml } from "hast-util-from-html";
 import { htmlToMdast } from "./html-to-mdast.js";
 import { mdastToMarkdown } from "./mdast-to-markdown.js";
-var htmlToMarkdown = (htmlOrHast, options) => {
-  const { baseUrl, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
-  const mdast = htmlToMdast(htmlOrHast, toMdastOptions);
-  const markdown = mdastToMarkdown(mdast, { baseUrl, ...toMarkdownOptions });
-  return markdown;
+import { extractMetadata, toFrontmatter } from "./metadata/index.js";
+import { getLangFromStr } from "./utils/hast-utils.js";
+var htmlToMarkdownWithMetadata = (htmlOrHast, options) => {
+  const { baseUrl, frontmatter, title, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
+  const hast = typeof htmlOrHast === "string" ? fromHtml(htmlOrHast, { fragment: true }) : htmlOrHast;
+  const isOwned = typeof htmlOrHast === "string";
+  const declaredLang = typeof htmlOrHast === "string" ? getLangFromStr(htmlOrHast) : void 0;
+  const metadata = extractMetadata(hast);
+  if (declaredLang && !metadata.lang) {
+    metadata.lang = declaredLang;
+  }
+  const mdast = htmlToMdast(hast, {
+    ...toMdastOptions,
+    owned: isOwned,
+    lang: toMdastOptions.lang ?? metadata.lang,
+    url: toMdastOptions.url ?? metadata.canonicalUrl
+  });
+  const body = mdastToMarkdown(mdast, { baseUrl, ...toMarkdownOptions });
+  const titled = title === false ? body : withTitle(body, metadata.title);
+  const markdown = frontmatter ? `${toFrontmatter(metadata)}${titled}` : titled;
+  return { markdown, metadata };
 };
+var TITLE_LOOKAHEAD = 400;
+var withTitle = (body, title) => {
+  if (!title) {
+    return body;
+  }
+  const opening = body.slice(0, TITLE_LOOKAHEAD);
+  if (/^#\s+\S/m.test(opening)) {
+    return body;
+  }
+  if (normalizeForCompare(opening).includes(normalizeForCompare(title))) {
+    return body;
+  }
+  return `# ${title}
+
+${body}`;
+};
+var normalizeForCompare = (value) => value.replace(/\s+/g, " ").trim().toLowerCase();
+var htmlToMarkdown = (htmlOrHast, options) => htmlToMarkdownWithMetadata(htmlOrHast, options).markdown;
 export {
-  htmlToMarkdown
+  htmlToMarkdown,
+  htmlToMarkdownWithMetadata
 };