webforai 1.5.1 → 1.6.0

package/dist/cjs/mdast-handlers/custom-br-handler.js

@@ -0,0 +1,31 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var custom_br_handler_exports = {};
+__export(custom_br_handler_exports, {
+  customBrHandler: () => customBrHandler
+});
+module.exports = __toCommonJS(custom_br_handler_exports);
+var import_hast_util_to_mdast = require("hast-util-to-mdast");
+const customBrHandler = (state, node) => {
+  return import_hast_util_to_mdast.defaultHandlers.br(state, node);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  customBrHandler
+});

package/dist/cjs/mdast-to-markdown.js

@@ -33,7 +33,7 @@ const DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 const mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions);
+  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
   if (baseUrl) {
     markdown = (0, import_link_replacer.linkReplacer)(markdown, baseUrl);
   }

package/dist/cjs/utils/hast-utils.js

@@ -0,0 +1,71 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var hast_utils_exports = {};
+__export(hast_utils_exports, {
+  getLangFromHast: () => getLangFromHast,
+  getLangFromStr: () => getLangFromStr,
+  getUrlFromHast: () => getUrlFromHast
+});
+module.exports = __toCommonJS(hast_utils_exports);
+var import_hast_util_select = require("hast-util-select");
+const getLangFromHast = (node) => {
+  const html = (0, import_hast_util_select.select)("html", node);
+  if (html && typeof html.properties.lang === "string") {
+    return html.properties.lang;
+  }
+  if (node.type !== "element") {
+    return;
+  }
+  const element = node;
+  if (element.tagName !== "html") {
+    return;
+  }
+  const langAttr = element.properties.lang || element.properties["xml:lang"];
+  if (langAttr) {
+    return langAttr;
+  }
+  return void 0;
+};
+const getLangFromStr = (str) => {
+  const match = str.match(/lang=["']([^"']+)["']/);
+  if (match) {
+    return match[1];
+  }
+  return void 0;
+};
+const getUrlFromHast = (node) => {
+  if (node.type !== "element") {
+    return void 0;
+  }
+  const metaTagAttributes = ["og:url", "twitter:url"];
+  const metaTags = (0, import_hast_util_select.selectAll)("meta", node);
+  for (const meta of metaTags) {
+    const property = meta.properties.property || meta.properties.name;
+    if (typeof property === "string" && metaTagAttributes.includes(property)) {
+      return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
+    }
+  }
+  return void 0;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getLangFromHast,
+  getLangFromStr,
+  getUrlFromHast
+});

package/dist/cli/bin.js

@@ -3,10 +3,9 @@ import { intro } from "@clack/prompts";
 import { program } from "commander";
 import pc from "picocolors";
 import packageInfo from "../../package.json";
-import { webforaiCommand } from "./commands/handlers/index.js";
+import { webforaiCommand } from "./commands/webforai/index.js";
 import { LOADERS, MODES } from "./constants.js";
-program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
-  process.env.DEBUG = "true";
-}).action(webforaiCommand);
+program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version");
+program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(webforaiCommand);
 intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
 program.parse();

package/dist/cli/commands/webforai/index.js

@@ -0,0 +1,53 @@
+// src/cli/commands/webforai/index.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+import { intro, outro, spinner } from "@clack/prompts";
+import pc from "picocolors";
+import packageInfo from "../../../../package.json";
+import { htmlToMarkdown } from "../../../html-to-markdown.js";
+import { inputOutputPath } from "../../helpers/inputOutputPath.js";
+import { inputSourcePath } from "../../helpers/inputSourcePath.js";
+import { selectExtractMode } from "../../helpers/selectExtractMode.js";
+import { selectLoader } from "../../helpers/selectLoader.js";
+import { isUrl } from "../../utils.js";
+import { loadHtml } from "./loadHtml.js";
+var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
+var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
+var webforaiCommand = async (initialPath, options) => {
+  intro(pc.bold(pc.green(`webforai CLI version ${packageInfo.version}`)));
+  const sourcePath = initialPath ?? await inputSourcePath();
+  options.debug && console.debug(`sourcePath: ${sourcePath}`);
+  const loader = isUrl(sourcePath) ? options.loader ?? await selectLoader() : "local";
+  options.debug && console.debug(`loader: ${loader}`);
+  const outputPath = options.output ?? await inputOutputPath(sourcePath);
+  options.debug && console.debug(`outputPath: ${outputPath}`);
+  const mode = options.mode ?? await selectExtractMode();
+  options.debug && console.debug(`mode: ${mode}`);
+  let html;
+  const s = spinner();
+  try {
+    s.start("Loading content...");
+    html = await loadHtml(sourcePath, loader, { debug: options.debug });
+    s.stop(pc.green("Content loaded!"));
+  } catch (error) {
+    s.stop(pc.red("Content loading failed!"));
+    console.error(error);
+    process.exit(1);
+  }
+  options.debug && console.debug(`html: ${html}`);
+  const markdown = htmlToMarkdown(html, {
+    baseUrl: isUrl(sourcePath) ? sourcePath : void 0,
+    ...mode === "ai" ? aiModeOptions : readabilityModeOptions
+  });
+  options.debug && console.debug(`markdown: ${markdown}`);
+  const directory = path.dirname(outputPath);
+  const isDirectoryExists = await fs.stat(directory).then((stat) => stat.isDirectory());
+  if (!isDirectoryExists) {
+    await fs.mkdir(directory, { recursive: true });
+  }
+  await fs.writeFile(outputPath, markdown);
+  outro(pc.green(`${pc.bold("Done!")} Markdown saved to ${outputPath}`));
+};
+export {
+  webforaiCommand
+};

package/dist/cli/commands/webforai/loadHtml.js

@@ -0,0 +1,60 @@
+// src/cli/commands/webforai/loadHtml.ts
+import fs from "node:fs/promises";
+import pc from "picocolors";
+import { chromium } from "playwright-core";
+import playwrightPackageJson from "playwright-core/package.json";
+import { loadHtml as loadHtmlByFetch } from "../../../loaders/fetch.js";
+import { loadHtml as loadHtmlByPlaywright } from "../../../loaders/playwright.js";
+var checkPlaywrightAvailable = async () => {
+  const path = chromium.executablePath();
+  const isAvailable = await fs.access(path).catch(() => false);
+  return !!isAvailable;
+};
+var loadHtml = async (sourcePath, loader, options) => {
+  if (loader === "local") {
+    options.debug && console.debug(`Loading HTML from local file: ${sourcePath}`);
+    const content = await fs.readFile(sourcePath, "utf-8");
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "fetch") {
+    options.debug && console.debug(`Loading HTML from URL: ${sourcePath}`);
+    const content = await loadHtmlByFetch(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "playwright") {
+    options.debug && console.debug(`Loading HTML from playwright: ${sourcePath}`);
+    const isPlaywrightAvailable = await checkPlaywrightAvailable();
+    options.debug && console.debug(`Playwright available: ${isPlaywrightAvailable}`);
+    if (!isPlaywrightAvailable) {
+      const message = [
+        "To use playwright loader, you need to install playwright.",
+        "You can install it by running:",
+        "",
+        `	${pc.bold(`npx playwright@${playwrightPackageJson.version} install chromium`)}	`,
+        "",
+        "Hint1: If you get warning like this:",
+        "WARNING: It looks like you are running 'npx playwright install' without first installing your project's dependencies. ",
+        `${pc.bold("Ignore this warning.")}`,
+        "",
+        "Hint2: If you get message like this:",
+        "Host system is missing dependencies to run browsers.",
+        "You should install dependencies by running:",
+        "",
+        `	${pc.bold(`sudo npx playwright@${playwrightPackageJson.version} install-deps`)}	`,
+        "",
+        pc.gray("Note: Good luck with that.")
+      ];
+      console.info(message.join("\n"));
+      throw new Error("Playwright is not available");
+    }
+    const content = await loadHtmlByPlaywright(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  throw new Error(`Unsupported loader: ${loader}`);
+};
+export {
+  loadHtml
+};

package/dist/cli/utils.js

@@ -10,71 +10,38 @@ var isUrl = (maybeUrl) => {
   }
 };
 function changeFileExtension(filePath, newExtension) {
-  const parsedPath = filePath.split("/");
-  const fileName = parsedPath[parsedPath.length - 1];
-  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
-  if (fileName.startsWith(".")) {
-    const parts = fileName.split(".");
-    if (parts.length === 2) {
-      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
-    }
-    parts[parts.length - 1] = newExtension.replace(/^\./, "");
-    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
-  }
-  const lastDotIndex = fileName.lastIndexOf(".");
-  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
-  const newFileName = `${baseName}${formattedNewExtension}`;
-  parsedPath[parsedPath.length - 1] = newFileName;
-  return parsedPath.join("/");
+  const parsed = path.parse(filePath);
+  const ext = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  return path.format({ ...parsed, ext });
 }
 function urlToFilename(url) {
   try {
     const urlObj = new URL(url);
-    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
-      if (index === 0) {
-        return acc;
-      }
-      if (acc.length >= 2) {
-        return acc;
-      }
-      if (part === "www") {
-        return acc;
-      }
-      return [part, ...acc];
-    }, []);
-    const domainString = domainParts.reverse().join("-");
+    const hostname = urlObj.hostname.startsWith("www.") ? urlObj.hostname.slice(4) : urlObj.hostname;
+    const domainParts = hostname.split(".").slice(-2).join("-");
     const pathParts = urlObj.pathname.split("/").filter(Boolean);
-    const relevantPathParts = pathParts.slice(-2);
-    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
-    let filename = [domainString, pathString].filter(Boolean).join("-");
+    const relevantPathParts = pathParts.slice(-2).map(decodeURIComponent).join("-");
+    let filename = [domainParts, relevantPathParts].filter(Boolean).join("-");
     filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
     return filename || "output";
   } catch {
     return "output";
   }
 }
-var sourcePathToOutputPath = (sourcePath) => {
-  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
-};
+var sourcePathToOutputPath = (sourcePath) => isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
 function getNextAvailableFilePath(filePath) {
-  const parsedPath = path.parse(filePath);
-  const directory = parsedPath.dir;
-  const fullName = parsedPath.base;
-  const [firstPart, ...restParts] = fullName.split(".");
-  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
-  const baseName = firstPart.replace(/_\d+$/, "");
-  let counter = 1;
-  let nextFilePath = filePath;
-  while (fs.existsSync(nextFilePath)) {
-    const match = firstPart.match(/_(\d+)$/);
-    if (match) {
-      counter = Number.parseInt(match[1], 10) + 1;
-    }
-    const newName = `${baseName}_${counter}${restName}`;
-    nextFilePath = path.join(directory, newName);
+  const parsed = path.parse(filePath);
+  const dir = parsed.dir;
+  const ext = parsed.ext;
+  const baseName = parsed.name.replace(/_\d+$/, "");
+  const match = parsed.name.match(/_(\d+)$/);
+  let counter = match ? Number(match[1]) + 1 : 1;
+  let newPath = path.join(dir, `${baseName}_${counter}${ext}`);
+  while (fs.existsSync(newPath)) {
     counter++;
+    newPath = path.join(dir, `${baseName}_${counter}${ext}`);
   }
-  return nextFilePath;
+  return newPath;
 }
 export {
   changeFileExtension,

package/dist/extract-hast/index.js

@@ -6,17 +6,18 @@ var PRESET_EXTRACT_HAST = {
   readability: readabilityExtractHast
 };
 var DEFAULT_EXTRACT_HAST = ["readability"];
-var extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
+var extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+  const { hast, lang } = params;
   const _extractors = Array.isArray(extractors) ? extractors : [extractors];
   const extracted = _extractors.reduce((acc, extractor) => {
     if (extractor === false) {
       return acc;
     }
     if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
-      return PRESET_EXTRACT_HAST[extractor](acc);
+      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
     }
     if (typeof extractor === "function") {
-      return extractor(acc);
+      return extractor({ hast: acc, lang });
     }
     throw new Error(`Invalid extractor: ${extractor}`);
   }, hast) || hast;

package/dist/extract-hast/readability.js

@@ -2,41 +2,30 @@
 import { select, selectAll } from "hast-util-select";
 import { toString as hastToString } from "hast-util-to-string";
 import { filter } from "unist-util-filter";
-import { classnames, hasAncestors, isStrInclude, matchString } from "./utils.js";
+import { classnames, isStrInclude, matchString } from "./utils.js";
 var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
 var REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
   specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
-var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
-var PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
+var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
+var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
 var BASE_MINIMAL_LENGTH = {
   ja: 200,
   en: 500
 };
-var tryGetLang = (node) => {
-  if (node.type !== "element") {
-    return;
-  }
-  const element = node;
-  if (element.tagName !== "html") {
-    return;
-  }
-  const langAttr = element.properties.lang || element.properties["xml:lang"];
-  if (langAttr) {
-    return langAttr;
-  }
+var metadataFilter = (node) => {
+  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
 };
-var metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
 var universalElementFilter = (node) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (["script", "aside"].includes(element.tagName)) {
+  if (["aside", "nav"].includes(element.tagName)) {
     return false;
   }
   if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
@@ -67,10 +56,7 @@ var unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
-    return true;
-  }
-  if (hasAncestors(element, ["table", "code"], 3)) {
+  if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
     return true;
   }
   const match = matchString(element);
@@ -82,55 +68,36 @@ var unlikelyElementFilter = (node) => {
   }
   return true;
 };
-var isImageLink = (element) => {
-  const a = select("a", element);
-  const img = select("img", a);
-  if (!(a && img)) {
-    return false;
-  }
-  const imgFilename = img.properties.src?.toString().split("/").pop();
-  const hrefFilename = a.properties.href?.toString().split("/").pop();
-  return imgFilename === hrefFilename && imgFilename;
-};
-var removeEmptyFilter = (node) => {
+var removeEmptyFilter = (node, _lang) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (!PARAGRAPH_TAGS.includes(element.tagName)) {
+  if (PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
-  if (isImageLink(element)) {
-    return true;
-  }
-  const text = hastToString(element);
-  if (text.length < 10) {
+  if (element.tagName === "img" && !element.properties.src) {
     return false;
   }
   return true;
 };
-var readabilityExtractHast = (hast) => {
-  const lang = String(select("html", hast)?.properties.lang || tryGetLang(hast) || "en");
+var readabilityExtractHast = (params) => {
+  const { hast, lang = "en" } = params;
   const body = select("body", hast) ?? hast;
-  const bodyText = hastToString(body);
-  const baseFilterd = filter(body, (node) => {
-    if (!metadataFilter(node)) {
-      return false;
-    }
-    if (!universalElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  if (!baseFilterd) {
-    return { type: "root", children: [] };
-  }
-  const baseFilterdText = hastToString(baseFilterd);
-  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
+  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
+  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
+  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+    return body;
+  }
+  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
+  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
+  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
+  let extractedTree = baseTree;
+  let extractedText = baseText;
   for (const selector of BODY_SELECTORS) {
     const content = { type: "root", children: selectAll(selector, baseFilterd) };
     const contentText = hastToString(content);
@@ -139,18 +106,18 @@ var readabilityExtractHast = (hast) => {
     }
     const links = selectAll("a", content);
     const linkText = links.map((link) => hastToString(link)).join("");
-    const linkDensity = linkText.length / bodyText.length;
+    const linkDensity = linkText.length / contentText.length;
     if (linkDensity > 0.4) {
       continue;
     }
     if (contentText.length > minimalLength) {
-      baseTree = content;
-      baseText = contentText;
+      extractedTree = content;
+      extractedText = contentText;
       break;
     }
   }
-  const finalFilteredTree = filter(baseTree, (node) => {
-    if (!removeEmptyFilter(node)) {
+  const finalFilteredTree = filter(extractedTree, (node) => {
+    if (!removeEmptyFilter(node, lang)) {
       return false;
     }
     if (!unlikelyElementFilter(node)) {
@@ -158,7 +125,7 @@ var readabilityExtractHast = (hast) => {
     }
     return true;
   });
-  const finalTree = hastToString(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
+  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
   return finalTree;
 };
 export {

package/dist/extract-hast/strongly.js

@@ -2,7 +2,8 @@
 import { select } from "hast-util-select";
 import { filter } from "unist-util-filter";
 var REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-var stronglyExtractHast = (hast) => {
+var stronglyExtractHast = (params) => {
+  const { hast } = params;
   const selectors = ["article", "#article", "#content", "main", "div.container"];
   let selectedElement = hast;
   for (const selector of selectors) {

package/dist/extract-hast/utils.js

@@ -12,22 +12,8 @@ var isStrInclude = (value, match) => {
   }
   return false;
 };
-var hasAncestors = (element, tagNames, depth = 3) => {
-  let parent = element.parent;
-  for (let i = 0; i < depth; i++) {
-    if (!parent || parent.type !== "element") {
-      return false;
-    }
-    if (tagNames.includes(parent.tagName)) {
-      return true;
-    }
-    parent = parent.parent;
-  }
-  return false;
-};
 export {
   classnames,
-  hasAncestors,
   isStrInclude,
   matchString
 };

package/dist/html-to-mdast.js

@@ -8,12 +8,13 @@ import { customCodeHandler } from "./mdast-handlers/custom-code-handler.js";
 import { customDivHandler } from "./mdast-handlers/custom-div-handler.js";
 import { customImgHandler } from "./mdast-handlers/custom-img-handler.js";
 import { customTableHandler } from "./mdast-handlers/custom-table-handler.js";
-import { emptyHandler } from "./mdast-handlers/empty-handler.js";
 import { mathHandler } from "./mdast-handlers/math-handler.js";
+import { getLangFromHast, getLangFromStr, getUrlFromHast } from "./utils/hast-utils.js";
 var htmlToMdast = (htmlOrHast, options) => {
-  const { extractors } = options || {};
-  const hast = typeof htmlOrHast === "string" ? fromHtml(htmlOrHast, { fragment: true }) : htmlOrHast;
-  const extractedHast = extractHast(hast, extractors);
+  const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
+  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
+  const url = defaultUrl || getUrlFromHast(hast);
+  const extractedHast = extractHast({ hast, lang, url }, extractors);
   const mdast = toMdast(extractedHast, {
     handlers: {
       math: mathHandler,
@@ -21,8 +22,7 @@ var htmlToMdast = (htmlOrHast, options) => {
       pre: customCodeHandler,
       a: customAHandler({ asText: options?.linkAsText }),
       img: customImgHandler({ hideImage: options?.hideImage }),
-      table: customTableHandler({ asText: options?.tableAsText }),
-      br: emptyHandler
+      table: customTableHandler({ asText: options?.tableAsText })
     }
   });
   const extractedMdast = extractMdast(mdast);

package/dist/link-replacer.js

@@ -1,11 +1,16 @@
 // src/link-replacer.ts
-var linkReplacer = (markdown, link) => {
-  const regex = /(!?\[.*?\]\()(.+?)(\))/g;
-  return markdown.replace(regex, (match, pre, thisUrl, post) => {
-    if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
+var linkReplacer = (markdown, base) => {
+  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
+  return markdown.replace(regex, (match, pre, url, post) => {
+    if (/^(https?:|#)/.test(url)) {
+      return match;
+    }
+    try {
+      const absoluteUrl = new URL(url, base).href;
+      return `${pre}${absoluteUrl}${post}`;
+    } catch {
       return match;
     }
-    return `${pre}${new URL(thisUrl, link).href}${post}`;
   });
 };
 export {