webforai 1.5.1 → 1.6.0

package/dist/cjs/cli/bin.js

@@ -25,10 +25,9 @@ var import_prompts = require("@clack/prompts");
 var import_commander = require("commander");
 var import_picocolors = __toESM(require("picocolors"), 1);
 var import_package = __toESM(require("../../package.json"), 1);
-var import_handlers = require("./commands/handlers");
+var import_webforai = require("./commands/webforai");
 var import_constants = require("./constants");
-import_commander.program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(import_package.default.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${import_constants.MODES.join(", ")})`).option("--loader <loader>", `Loader to use (${import_constants.LOADERS.join(", ")})`).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
-  process.env.DEBUG = "true";
-}).action(import_handlers.webforaiCommand);
+import_commander.program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(import_package.default.version, "-v, --version", "output the current version");
+import_commander.program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${import_constants.MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${import_constants.LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(import_webforai.webforaiCommand);
 (0, import_prompts.intro)(import_picocolors.default.inverse(`webforai CLI version ${import_package.default.version}`));
 import_commander.program.parse();

package/dist/cjs/cli/commands/{handlers → webforai}/index.js

@@ -26,14 +26,16 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   mod
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var handlers_exports = {};
-__export(handlers_exports, {
+var webforai_exports = {};
+__export(webforai_exports, {
   webforaiCommand: () => webforaiCommand
 });
-module.exports = __toCommonJS(handlers_exports);
-var import_node_fs = __toESM(require("node:fs"), 1);
+module.exports = __toCommonJS(webforai_exports);
+var import_promises = __toESM(require("node:fs/promises"), 1);
 var import_node_path = __toESM(require("node:path"), 1);
 var import_prompts = require("@clack/prompts");
+var import_picocolors = __toESM(require("picocolors"), 1);
+var import_package = __toESM(require("../../../../package.json"), 1);
 var import_html_to_markdown = require("../../../html-to-markdown");
 var import_inputOutputPath = require("../../helpers/inputOutputPath");
 var import_inputSourcePath = require("../../helpers/inputSourcePath");
@@ -43,40 +45,40 @@ var import_utils = require("../../utils");
 var import_loadHtml = require("./loadHtml");
 const aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
 const readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
-const webforaiCommand = async (initialPath, initialOutputPath, options) => {
+const webforaiCommand = async (initialPath, options) => {
+  (0, import_prompts.intro)(import_picocolors.default.bold(import_picocolors.default.green(`webforai CLI version ${import_package.default.version}`)));
   const sourcePath = initialPath ?? await (0, import_inputSourcePath.inputSourcePath)();
-  if (options.debug) {
-    console.debug(`sourcePath: ${sourcePath}`);
-  }
-  const loader = (0, import_utils.isUrl)(sourcePath) ? { type: "web", loader: options.loader ?? await (0, import_selectLoader.selectLoader)() } : { type: "local", loader: null };
-  if (options.debug) {
-    console.debug(`loader: ${loader}`);
-  }
-  const outputPath = initialOutputPath ?? await (0, import_inputOutputPath.inputOutputPath)(sourcePath);
-  if (options.debug) {
-    console.debug(`outputPath: ${outputPath}`);
-  }
+  options.debug && console.debug(`sourcePath: ${sourcePath}`);
+  const loader = (0, import_utils.isUrl)(sourcePath) ? options.loader ?? await (0, import_selectLoader.selectLoader)() : "local";
+  options.debug && console.debug(`loader: ${loader}`);
+  const outputPath = options.output ?? await (0, import_inputOutputPath.inputOutputPath)(sourcePath);
+  options.debug && console.debug(`outputPath: ${outputPath}`);
   const mode = options.mode ?? await (0, import_selectExtractMode.selectExtractMode)();
-  if (options.debug) {
-    console.debug(`mode: ${mode}`);
+  options.debug && console.debug(`mode: ${mode}`);
+  let html;
+  const s = (0, import_prompts.spinner)();
+  try {
+    s.start("Loading content...");
+    html = await (0, import_loadHtml.loadHtml)(sourcePath, loader, { debug: options.debug });
+    s.stop(import_picocolors.default.green("Content loaded!"));
+  } catch (error) {
+    s.stop(import_picocolors.default.red("Content loading failed!"));
+    console.error(error);
+    process.exit(1);
   }
-  const content = await (0, import_loadHtml.loadHtml)(sourcePath, loader);
-  if (options.debug) {
-    console.debug(`content: ${content}`);
-  }
-  const markdown = (0, import_html_to_markdown.htmlToMarkdown)(content, {
-    baseUrl: (0, import_utils.isUrl)(sourcePath) ? sourcePath : options.baseUrl,
+  options.debug && console.debug(`html: ${html}`);
+  const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, {
+    baseUrl: (0, import_utils.isUrl)(sourcePath) ? sourcePath : void 0,
     ...mode === "ai" ? aiModeOptions : readabilityModeOptions
   });
-  if (options.debug) {
-    console.debug(`markdown: ${markdown}`);
-  }
+  options.debug && console.debug(`markdown: ${markdown}`);
   const directory = import_node_path.default.dirname(outputPath);
-  if (!import_node_fs.default.existsSync(directory)) {
-    import_node_fs.default.mkdirSync(directory, { recursive: true });
+  const isDirectoryExists = await import_promises.default.stat(directory).then((stat) => stat.isDirectory());
+  if (!isDirectoryExists) {
+    await import_promises.default.mkdir(directory, { recursive: true });
   }
-  import_node_fs.default.writeFileSync(outputPath, markdown);
-  (0, import_prompts.outro)("Markdown conversion is complete!");
+  await import_promises.default.writeFile(outputPath, markdown);
+  (0, import_prompts.outro)(import_picocolors.default.green(`${import_picocolors.default.bold("Done!")} Markdown saved to ${outputPath}`));
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/cli/commands/webforai/loadHtml.js

@@ -0,0 +1,93 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var loadHtml_exports = {};
+__export(loadHtml_exports, {
+  loadHtml: () => loadHtml
+});
+module.exports = __toCommonJS(loadHtml_exports);
+var import_promises = __toESM(require("node:fs/promises"), 1);
+var import_picocolors = __toESM(require("picocolors"), 1);
+var import_playwright_core = require("playwright-core");
+var import_package = __toESM(require("playwright-core/package.json"), 1);
+var import_fetch = require("../../../loaders/fetch");
+var import_playwright = require("../../../loaders/playwright");
+const checkPlaywrightAvailable = async () => {
+  const path = import_playwright_core.chromium.executablePath();
+  const isAvailable = await import_promises.default.access(path).catch(() => false);
+  return !!isAvailable;
+};
+const loadHtml = async (sourcePath, loader, options) => {
+  if (loader === "local") {
+    options.debug && console.debug(`Loading HTML from local file: ${sourcePath}`);
+    const content = await import_promises.default.readFile(sourcePath, "utf-8");
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "fetch") {
+    options.debug && console.debug(`Loading HTML from URL: ${sourcePath}`);
+    const content = await (0, import_fetch.loadHtml)(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "playwright") {
+    options.debug && console.debug(`Loading HTML from playwright: ${sourcePath}`);
+    const isPlaywrightAvailable = await checkPlaywrightAvailable();
+    options.debug && console.debug(`Playwright available: ${isPlaywrightAvailable}`);
+    if (!isPlaywrightAvailable) {
+      const message = [
+        "To use playwright loader, you need to install playwright.",
+        "You can install it by running:",
+        "",
+        `	${import_picocolors.default.bold(`npx playwright@${import_package.default.version} install chromium`)}	`,
+        "",
+        "Hint1: If you get warning like this:",
+        "WARNING: It looks like you are running 'npx playwright install' without first installing your project's dependencies. ",
+        `${import_picocolors.default.bold("Ignore this warning.")}`,
+        "",
+        "Hint2: If you get message like this:",
+        "Host system is missing dependencies to run browsers.",
+        "You should install dependencies by running:",
+        "",
+        `	${import_picocolors.default.bold(`sudo npx playwright@${import_package.default.version} install-deps`)}	`,
+        "",
+        import_picocolors.default.gray("Note: Good luck with that.")
+      ];
+      console.info(message.join("\n"));
+      throw new Error("Playwright is not available");
+    }
+    const content = await (0, import_playwright.loadHtml)(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  throw new Error(`Unsupported loader: ${loader}`);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  loadHtml
+});

package/dist/cjs/cli/utils.js

@@ -46,71 +46,38 @@ const isUrl = (maybeUrl) => {
   }
 };
 function changeFileExtension(filePath, newExtension) {
-  const parsedPath = filePath.split("/");
-  const fileName = parsedPath[parsedPath.length - 1];
-  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
-  if (fileName.startsWith(".")) {
-    const parts = fileName.split(".");
-    if (parts.length === 2) {
-      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
-    }
-    parts[parts.length - 1] = newExtension.replace(/^\./, "");
-    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
-  }
-  const lastDotIndex = fileName.lastIndexOf(".");
-  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
-  const newFileName = `${baseName}${formattedNewExtension}`;
-  parsedPath[parsedPath.length - 1] = newFileName;
-  return parsedPath.join("/");
+  const parsed = import_node_path.default.parse(filePath);
+  const ext = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  return import_node_path.default.format({ ...parsed, ext });
 }
 function urlToFilename(url) {
   try {
     const urlObj = new URL(url);
-    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
-      if (index === 0) {
-        return acc;
-      }
-      if (acc.length >= 2) {
-        return acc;
-      }
-      if (part === "www") {
-        return acc;
-      }
-      return [part, ...acc];
-    }, []);
-    const domainString = domainParts.reverse().join("-");
+    const hostname = urlObj.hostname.startsWith("www.") ? urlObj.hostname.slice(4) : urlObj.hostname;
+    const domainParts = hostname.split(".").slice(-2).join("-");
     const pathParts = urlObj.pathname.split("/").filter(Boolean);
-    const relevantPathParts = pathParts.slice(-2);
-    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
-    let filename = [domainString, pathString].filter(Boolean).join("-");
+    const relevantPathParts = pathParts.slice(-2).map(decodeURIComponent).join("-");
+    let filename = [domainParts, relevantPathParts].filter(Boolean).join("-");
     filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
     return filename || "output";
   } catch {
     return "output";
   }
 }
-const sourcePathToOutputPath = (sourcePath) => {
-  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
-};
+const sourcePathToOutputPath = (sourcePath) => isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
 function getNextAvailableFilePath(filePath) {
-  const parsedPath = import_node_path.default.parse(filePath);
-  const directory = parsedPath.dir;
-  const fullName = parsedPath.base;
-  const [firstPart, ...restParts] = fullName.split(".");
-  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
-  const baseName = firstPart.replace(/_\d+$/, "");
-  let counter = 1;
-  let nextFilePath = filePath;
-  while (import_node_fs.default.existsSync(nextFilePath)) {
-    const match = firstPart.match(/_(\d+)$/);
-    if (match) {
-      counter = Number.parseInt(match[1], 10) + 1;
-    }
-    const newName = `${baseName}_${counter}${restName}`;
-    nextFilePath = import_node_path.default.join(directory, newName);
+  const parsed = import_node_path.default.parse(filePath);
+  const dir = parsed.dir;
+  const ext = parsed.ext;
+  const baseName = parsed.name.replace(/_\d+$/, "");
+  const match = parsed.name.match(/_(\d+)$/);
+  let counter = match ? Number(match[1]) + 1 : 1;
+  let newPath = import_node_path.default.join(dir, `${baseName}_${counter}${ext}`);
+  while (import_node_fs.default.existsSync(newPath)) {
     counter++;
+    newPath = import_node_path.default.join(dir, `${baseName}_${counter}${ext}`);
   }
-  return nextFilePath;
+  return newPath;
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/cjs/extract-hast/index.js

@@ -30,17 +30,18 @@ const PRESET_EXTRACT_HAST = {
   readability: import_readability.readabilityExtractHast
 };
 const DEFAULT_EXTRACT_HAST = ["readability"];
-const extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
+const extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+  const { hast, lang } = params;
   const _extractors = Array.isArray(extractors) ? extractors : [extractors];
   const extracted = _extractors.reduce((acc, extractor) => {
     if (extractor === false) {
       return acc;
     }
     if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
-      return PRESET_EXTRACT_HAST[extractor](acc);
+      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
     }
     if (typeof extractor === "function") {
-      return extractor(acc);
+      return extractor({ hast: acc, lang });
     }
     throw new Error(`Invalid extractor: ${extractor}`);
   }, hast) || hast;

package/dist/cjs/extract-hast/readability.js

@@ -30,35 +30,24 @@ const REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
   specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
-const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
-const PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
+const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
+const PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
 const BASE_MINIMAL_LENGTH = {
   ja: 200,
   en: 500
 };
-const tryGetLang = (node) => {
-  if (node.type !== "element") {
-    return;
-  }
-  const element = node;
-  if (element.tagName !== "html") {
-    return;
-  }
-  const langAttr = element.properties.lang || element.properties["xml:lang"];
-  if (langAttr) {
-    return langAttr;
-  }
+const metadataFilter = (node) => {
+  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
 };
-const metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
 const universalElementFilter = (node) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (["script", "aside"].includes(element.tagName)) {
+  if (["aside", "nav"].includes(element.tagName)) {
     return false;
   }
   if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
@@ -89,10 +78,7 @@ const unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
-    return true;
-  }
-  if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3)) {
+  if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
     return true;
   }
   const match = (0, import_utils.matchString)(element);
@@ -104,55 +90,36 @@ const unlikelyElementFilter = (node) => {
   }
   return true;
 };
-const isImageLink = (element) => {
-  const a = (0, import_hast_util_select.select)("a", element);
-  const img = (0, import_hast_util_select.select)("img", a);
-  if (!(a && img)) {
-    return false;
-  }
-  const imgFilename = img.properties.src?.toString().split("/").pop();
-  const hrefFilename = a.properties.href?.toString().split("/").pop();
-  return imgFilename === hrefFilename && imgFilename;
-};
-const removeEmptyFilter = (node) => {
+const removeEmptyFilter = (node, _lang) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (!PARAGRAPH_TAGS.includes(element.tagName)) {
+  if (PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
-  if (isImageLink(element)) {
-    return true;
-  }
-  const text = (0, import_hast_util_to_string.toString)(element);
-  if (text.length < 10) {
+  if (element.tagName === "img" && !element.properties.src) {
     return false;
   }
   return true;
 };
-const readabilityExtractHast = (hast) => {
-  const lang = String((0, import_hast_util_select.select)("html", hast)?.properties.lang || tryGetLang(hast) || "en");
+const readabilityExtractHast = (params) => {
+  const { hast, lang = "en" } = params;
   const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
-  const bodyText = (0, import_hast_util_to_string.toString)(body);
-  const baseFilterd = (0, import_unist_util_filter.filter)(body, (node) => {
-    if (!metadataFilter(node)) {
-      return false;
-    }
-    if (!universalElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  if (!baseFilterd) {
-    return { type: "root", children: [] };
-  }
-  const baseFilterdText = (0, import_hast_util_to_string.toString)(baseFilterd);
-  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
+  const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
+  const metadataFilteredHastText = metadataFilteredHast && (0, import_hast_util_to_string.toString)(metadataFilteredHast);
+  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+    return body;
+  }
+  const baseFilterd = (0, import_unist_util_filter.filter)(metadataFilteredHast, (node) => universalElementFilter(node));
+  const baseFilterdText = baseFilterd ? (0, import_hast_util_to_string.toString)(baseFilterd) : "";
+  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
+  let extractedTree = baseTree;
+  let extractedText = baseText;
   for (const selector of BODY_SELECTORS) {
     const content = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
     const contentText = (0, import_hast_util_to_string.toString)(content);
@@ -161,18 +128,18 @@ const readabilityExtractHast = (hast) => {
     }
     const links = (0, import_hast_util_select.selectAll)("a", content);
     const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
-    const linkDensity = linkText.length / bodyText.length;
+    const linkDensity = linkText.length / contentText.length;
     if (linkDensity > 0.4) {
       continue;
     }
     if (contentText.length > minimalLength) {
-      baseTree = content;
-      baseText = contentText;
+      extractedTree = content;
+      extractedText = contentText;
       break;
     }
   }
-  const finalFilteredTree = (0, import_unist_util_filter.filter)(baseTree, (node) => {
-    if (!removeEmptyFilter(node)) {
+  const finalFilteredTree = (0, import_unist_util_filter.filter)(extractedTree, (node) => {
+    if (!removeEmptyFilter(node, lang)) {
       return false;
     }
     if (!unlikelyElementFilter(node)) {
@@ -180,7 +147,7 @@ const readabilityExtractHast = (hast) => {
     }
     return true;
   });
-  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
+  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
   return finalTree;
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/extract-hast/strongly.js

@@ -24,7 +24,8 @@ module.exports = __toCommonJS(strongly_exports);
 var import_hast_util_select = require("hast-util-select");
 var import_unist_util_filter = require("unist-util-filter");
 const REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-const stronglyExtractHast = (hast) => {
+const stronglyExtractHast = (params) => {
+  const { hast } = params;
   const selectors = ["article", "#article", "#content", "main", "div.container"];
   let selectedElement = hast;
   for (const selector of selectors) {

package/dist/cjs/extract-hast/utils.js

@@ -19,7 +19,6 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var utils_exports = {};
 __export(utils_exports, {
   classnames: () => classnames,
-  hasAncestors: () => hasAncestors,
   isStrInclude: () => isStrInclude,
   matchString: () => matchString
 });
@@ -37,23 +36,9 @@ const isStrInclude = (value, match) => {
   }
   return false;
 };
-const hasAncestors = (element, tagNames, depth = 3) => {
-  let parent = element.parent;
-  for (let i = 0; i < depth; i++) {
-    if (!parent || parent.type !== "element") {
-      return false;
-    }
-    if (tagNames.includes(parent.tagName)) {
-      return true;
-    }
-    parent = parent.parent;
-  }
-  return false;
-};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   classnames,
-  hasAncestors,
   isStrInclude,
   matchString
 });

package/dist/cjs/html-to-mdast.js

@@ -30,12 +30,13 @@ var import_custom_code_handler = require("./mdast-handlers/custom-code-handler")
 var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
 var import_custom_img_handler = require("./mdast-handlers/custom-img-handler");
 var import_custom_table_handler = require("./mdast-handlers/custom-table-handler");
-var import_empty_handler = require("./mdast-handlers/empty-handler");
 var import_math_handler = require("./mdast-handlers/math-handler");
+var import_hast_utils = require("./utils/hast-utils");
 const htmlToMdast = (htmlOrHast, options) => {
-  const { extractors } = options || {};
-  const hast = typeof htmlOrHast === "string" ? (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true }) : htmlOrHast;
-  const extractedHast = (0, import_extract_hast.extractHast)(hast, extractors);
+  const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
+  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
+  const url = defaultUrl || (0, import_hast_utils.getUrlFromHast)(hast);
+  const extractedHast = (0, import_extract_hast.extractHast)({ hast, lang, url }, extractors);
   const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
     handlers: {
       math: import_math_handler.mathHandler,
@@ -43,8 +44,7 @@ const htmlToMdast = (htmlOrHast, options) => {
       pre: import_custom_code_handler.customCodeHandler,
       a: (0, import_custom_a_handler.customAHandler)({ asText: options?.linkAsText }),
       img: (0, import_custom_img_handler.customImgHandler)({ hideImage: options?.hideImage }),
-      table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText }),
-      br: import_empty_handler.emptyHandler
+      table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText })
     }
   });
   const extractedMdast = (0, import_extract_mdast.extractMdast)(mdast);

package/dist/cjs/link-replacer.js

@@ -21,13 +21,18 @@ __export(link_replacer_exports, {
   linkReplacer: () => linkReplacer
 });
 module.exports = __toCommonJS(link_replacer_exports);
-const linkReplacer = (markdown, link) => {
-  const regex = /(!?\[.*?\]\()(.+?)(\))/g;
-  return markdown.replace(regex, (match, pre, thisUrl, post) => {
-    if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
+const linkReplacer = (markdown, base) => {
+  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
+  return markdown.replace(regex, (match, pre, url, post) => {
+    if (/^(https?:|#)/.test(url)) {
+      return match;
+    }
+    try {
+      const absoluteUrl = new URL(url, base).href;
+      return `${pre}${absoluteUrl}${post}`;
+    } catch {
       return match;
     }
-    return `${pre}${new URL(thisUrl, link).href}${post}`;
   });
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/loaders/playwright.js

@@ -21,15 +21,44 @@ __export(playwright_exports, {
   loadHtml: () => loadHtml
 });
 module.exports = __toCommonJS(playwright_exports);
-var import_playwright = require("playwright");
-const loadHtml = async (url, context) => {
-  const _context = context ?? await import_playwright.chromium.launch({ headless: true });
-  const page = await _context.newPage();
-  await page.goto(url, { waitUntil: "load" });
+var import_playwright_core = require("playwright-core");
+const SUPER_BYPASS_DEVICE = {
+  userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+  viewport: { width: 1920, height: 1080 },
+  deviceScaleFactor: 1,
+  hasTouch: false,
+  isMobile: false,
+  javaScriptEnabled: true,
+  locale: "en-US",
+  timezoneId: "America/New_York"
+};
+const loadHtml = async (url, options) => {
+  const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
+  const _browser = browser ?? await import_playwright_core.chromium.launch({ headless: true });
+  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
+  if (superBypassMode) {
+    await context.addInitScript(() => {
+      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+    });
+  }
+  const page = await context.newPage();
+  if (superBypassMode) {
+    await page.route("**/*.js", (route) => {
+      if (route.request().url().includes("captcha-delivery")) {
+        return route.abort();
+      }
+      return route.continue();
+    });
+  }
+  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
   const html = await page.content();
   await page.close();
-  if (!context) {
-    await _context.close();
+  if (browser) {
+    await context.close();
+  } else {
+    await _browser.close();
   }
   return html;
 };