webforai 1.5.0 → 1.6.0

package/dist/cjs/extract-hast/readability.js

@@ -30,35 +30,24 @@ const REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
   specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
-const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
-const PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
+const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
+const PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
 const BASE_MINIMAL_LENGTH = {
   ja: 200,
   en: 500
 };
-const tryGetLang = (node) => {
-  if (node.type !== "element") {
-    return;
-  }
-  const element = node;
-  if (element.tagName !== "html") {
-    return;
-  }
-  const langAttr = element.properties.lang || element.properties["xml:lang"];
-  if (langAttr) {
-    return langAttr;
-  }
+const metadataFilter = (node) => {
+  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
 };
-const metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
 const universalElementFilter = (node) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (["script", "aside"].includes(element.tagName)) {
+  if (["aside", "nav"].includes(element.tagName)) {
     return false;
   }
   if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
@@ -89,10 +78,7 @@ const unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
-    return true;
-  }
-  if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3)) {
+  if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
     return true;
   }
   const match = (0, import_utils.matchString)(element);
@@ -104,55 +90,36 @@ const unlikelyElementFilter = (node) => {
   }
   return true;
 };
-const isImageLink = (element) => {
-  const a = (0, import_hast_util_select.select)("a", element);
-  const img = (0, import_hast_util_select.select)("img", a);
-  if (!(a && img)) {
-    return false;
-  }
-  const imgFilename = img.properties.src?.toString().split("/").pop();
-  const hrefFilename = a.properties.href?.toString().split("/").pop();
-  return imgFilename === hrefFilename && imgFilename;
-};
-const removeEmptyFilter = (node) => {
+const removeEmptyFilter = (node, _lang) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (!PARAGRAPH_TAGS.includes(element.tagName)) {
+  if (PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
-  if (isImageLink(element)) {
-    return true;
-  }
-  const text = (0, import_hast_util_to_string.toString)(element);
-  if (text.length < 10) {
+  if (element.tagName === "img" && !element.properties.src) {
     return false;
   }
   return true;
 };
-const readabilityExtractHast = (hast) => {
-  const lang = String((0, import_hast_util_select.select)("html", hast)?.properties.lang || tryGetLang(hast) || "en");
+const readabilityExtractHast = (params) => {
+  const { hast, lang = "en" } = params;
   const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
-  const bodyText = (0, import_hast_util_to_string.toString)(body);
-  const baseFilterd = (0, import_unist_util_filter.filter)(body, (node) => {
-    if (!metadataFilter(node)) {
-      return false;
-    }
-    if (!universalElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  if (!baseFilterd) {
-    return { type: "root", children: [] };
-  }
-  const baseFilterdText = (0, import_hast_util_to_string.toString)(baseFilterd);
-  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
+  const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
+  const metadataFilteredHastText = metadataFilteredHast && (0, import_hast_util_to_string.toString)(metadataFilteredHast);
+  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+    return body;
+  }
+  const baseFilterd = (0, import_unist_util_filter.filter)(metadataFilteredHast, (node) => universalElementFilter(node));
+  const baseFilterdText = baseFilterd ? (0, import_hast_util_to_string.toString)(baseFilterd) : "";
+  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
+  let extractedTree = baseTree;
+  let extractedText = baseText;
   for (const selector of BODY_SELECTORS) {
     const content = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
     const contentText = (0, import_hast_util_to_string.toString)(content);
@@ -161,18 +128,18 @@ const readabilityExtractHast = (hast) => {
     }
     const links = (0, import_hast_util_select.selectAll)("a", content);
     const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
-    const linkDensity = linkText.length / bodyText.length;
+    const linkDensity = linkText.length / contentText.length;
     if (linkDensity > 0.4) {
       continue;
     }
     if (contentText.length > minimalLength) {
-      baseTree = content;
-      baseText = contentText;
+      extractedTree = content;
+      extractedText = contentText;
       break;
     }
   }
-  const finalFilteredTree = (0, import_unist_util_filter.filter)(baseTree, (node) => {
-    if (!removeEmptyFilter(node)) {
+  const finalFilteredTree = (0, import_unist_util_filter.filter)(extractedTree, (node) => {
+    if (!removeEmptyFilter(node, lang)) {
       return false;
     }
     if (!unlikelyElementFilter(node)) {
@@ -180,7 +147,7 @@ const readabilityExtractHast = (hast) => {
     }
     return true;
   });
-  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
+  const finalTree = (0, import_hast_util_to_string.toString)(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
   return finalTree;
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/extract-hast/strongly.js

@@ -24,7 +24,8 @@ module.exports = __toCommonJS(strongly_exports);
 var import_hast_util_select = require("hast-util-select");
 var import_unist_util_filter = require("unist-util-filter");
 const REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-const stronglyExtractHast = (hast) => {
+const stronglyExtractHast = (params) => {
+  const { hast } = params;
   const selectors = ["article", "#article", "#content", "main", "div.container"];
   let selectedElement = hast;
   for (const selector of selectors) {

package/dist/cjs/extract-hast/utils.js

@@ -19,7 +19,6 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var utils_exports = {};
 __export(utils_exports, {
   classnames: () => classnames,
-  hasAncestors: () => hasAncestors,
   isStrInclude: () => isStrInclude,
   matchString: () => matchString
 });
@@ -37,23 +36,9 @@ const isStrInclude = (value, match) => {
   }
   return false;
 };
-const hasAncestors = (element, tagNames, depth = 3) => {
-  let parent = element.parent;
-  for (let i = 0; i < depth; i++) {
-    if (!parent || parent.type !== "element") {
-      return false;
-    }
-    if (tagNames.includes(parent.tagName)) {
-      return true;
-    }
-    parent = parent.parent;
-  }
-  return false;
-};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   classnames,
-  hasAncestors,
   isStrInclude,
   matchString
 });

package/dist/cjs/html-to-mdast.js

@@ -30,12 +30,13 @@ var import_custom_code_handler = require("./mdast-handlers/custom-code-handler")
 var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
 var import_custom_img_handler = require("./mdast-handlers/custom-img-handler");
 var import_custom_table_handler = require("./mdast-handlers/custom-table-handler");
-var import_empty_handler = require("./mdast-handlers/empty-handler");
 var import_math_handler = require("./mdast-handlers/math-handler");
+var import_hast_utils = require("./utils/hast-utils");
 const htmlToMdast = (htmlOrHast, options) => {
-  const { extractors } = options || {};
-  const hast = typeof htmlOrHast === "string" ? (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true }) : htmlOrHast;
-  const extractedHast = (0, import_extract_hast.extractHast)(hast, extractors);
+  const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
+  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || (0, import_hast_utils.getLangFromStr)(htmlOrHast), (0, import_hast_util_from_html.fromHtml)(htmlOrHast, { fragment: true })] : [defaultLang || (0, import_hast_utils.getLangFromHast)(htmlOrHast), htmlOrHast];
+  const url = defaultUrl || (0, import_hast_utils.getUrlFromHast)(hast);
+  const extractedHast = (0, import_extract_hast.extractHast)({ hast, lang, url }, extractors);
   const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
     handlers: {
       math: import_math_handler.mathHandler,
@@ -43,8 +44,7 @@ const htmlToMdast = (htmlOrHast, options) => {
       pre: import_custom_code_handler.customCodeHandler,
       a: (0, import_custom_a_handler.customAHandler)({ asText: options?.linkAsText }),
       img: (0, import_custom_img_handler.customImgHandler)({ hideImage: options?.hideImage }),
-      table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText }),
-      br: import_empty_handler.emptyHandler
+      table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText })
     }
   });
   const extractedMdast = (0, import_extract_mdast.extractMdast)(mdast);

package/dist/cjs/link-replacer.js

@@ -21,13 +21,18 @@ __export(link_replacer_exports, {
   linkReplacer: () => linkReplacer
 });
 module.exports = __toCommonJS(link_replacer_exports);
-const linkReplacer = (markdown, link) => {
-  const regex = /(!?\[.*?\]\()(.+?)(\))/g;
-  return markdown.replace(regex, (match, pre, thisUrl, post) => {
-    if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
+const linkReplacer = (markdown, base) => {
+  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
+  return markdown.replace(regex, (match, pre, url, post) => {
+    if (/^(https?:|#)/.test(url)) {
+      return match;
+    }
+    try {
+      const absoluteUrl = new URL(url, base).href;
+      return `${pre}${absoluteUrl}${post}`;
+    } catch {
       return match;
     }
-    return `${pre}${new URL(thisUrl, link).href}${post}`;
   });
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/cjs/loaders/playwright.js

@@ -21,15 +21,44 @@ __export(playwright_exports, {
   loadHtml: () => loadHtml
 });
 module.exports = __toCommonJS(playwright_exports);
-var import_playwright = require("playwright");
-const loadHtml = async (url, context) => {
-  const _context = context ?? await import_playwright.chromium.launch({ headless: true });
-  const page = await _context.newPage();
-  await page.goto(url, { waitUntil: "load" });
+var import_playwright_core = require("playwright-core");
+const SUPER_BYPASS_DEVICE = {
+  userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+  viewport: { width: 1920, height: 1080 },
+  deviceScaleFactor: 1,
+  hasTouch: false,
+  isMobile: false,
+  javaScriptEnabled: true,
+  locale: "en-US",
+  timezoneId: "America/New_York"
+};
+const loadHtml = async (url, options) => {
+  const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
+  const _browser = browser ?? await import_playwright_core.chromium.launch({ headless: true });
+  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
+  if (superBypassMode) {
+    await context.addInitScript(() => {
+      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+    });
+  }
+  const page = await context.newPage();
+  if (superBypassMode) {
+    await page.route("**/*.js", (route) => {
+      if (route.request().url().includes("captcha-delivery")) {
+        return route.abort();
+      }
+      return route.continue();
+    });
+  }
+  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
   const html = await page.content();
   await page.close();
-  if (!context) {
-    await _context.close();
+  if (browser) {
+    await context.close();
+  } else {
+    await _browser.close();
   }
   return html;
 };

package/dist/cjs/mdast-handlers/custom-br-handler.js

@@ -0,0 +1,31 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var custom_br_handler_exports = {};
+__export(custom_br_handler_exports, {
+  customBrHandler: () => customBrHandler
+});
+module.exports = __toCommonJS(custom_br_handler_exports);
+var import_hast_util_to_mdast = require("hast-util-to-mdast");
+const customBrHandler = (state, node) => {
+  return import_hast_util_to_mdast.defaultHandlers.br(state, node);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  customBrHandler
+});

package/dist/cjs/mdast-to-markdown.js

@@ -33,7 +33,7 @@ const DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 const mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions);
+  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
   if (baseUrl) {
     markdown = (0, import_link_replacer.linkReplacer)(markdown, baseUrl);
   }

package/dist/cjs/utils/hast-utils.js

@@ -0,0 +1,71 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var hast_utils_exports = {};
+__export(hast_utils_exports, {
+  getLangFromHast: () => getLangFromHast,
+  getLangFromStr: () => getLangFromStr,
+  getUrlFromHast: () => getUrlFromHast
+});
+module.exports = __toCommonJS(hast_utils_exports);
+var import_hast_util_select = require("hast-util-select");
+const getLangFromHast = (node) => {
+  const html = (0, import_hast_util_select.select)("html", node);
+  if (html && typeof html.properties.lang === "string") {
+    return html.properties.lang;
+  }
+  if (node.type !== "element") {
+    return;
+  }
+  const element = node;
+  if (element.tagName !== "html") {
+    return;
+  }
+  const langAttr = element.properties.lang || element.properties["xml:lang"];
+  if (langAttr) {
+    return langAttr;
+  }
+  return void 0;
+};
+const getLangFromStr = (str) => {
+  const match = str.match(/lang=["']([^"']+)["']/);
+  if (match) {
+    return match[1];
+  }
+  return void 0;
+};
+const getUrlFromHast = (node) => {
+  if (node.type !== "element") {
+    return void 0;
+  }
+  const metaTagAttributes = ["og:url", "twitter:url"];
+  const metaTags = (0, import_hast_util_select.selectAll)("meta", node);
+  for (const meta of metaTags) {
+    const property = meta.properties.property || meta.properties.name;
+    if (typeof property === "string" && metaTagAttributes.includes(property)) {
+      return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
+    }
+  }
+  return void 0;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getLangFromHast,
+  getLangFromStr,
+  getUrlFromHast
+});

package/dist/cli/bin.js

@@ -0,0 +1,11 @@
+// src/cli/bin.ts
+import { intro } from "@clack/prompts";
+import { program } from "commander";
+import pc from "picocolors";
+import packageInfo from "../../package.json";
+import { webforaiCommand } from "./commands/webforai/index.js";
+import { LOADERS, MODES } from "./constants.js";
+program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version");
+program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(webforaiCommand);
+intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
+program.parse();

package/dist/cli/commands/webforai/index.js

@@ -0,0 +1,53 @@
+// src/cli/commands/webforai/index.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+import { intro, outro, spinner } from "@clack/prompts";
+import pc from "picocolors";
+import packageInfo from "../../../../package.json";
+import { htmlToMarkdown } from "../../../html-to-markdown.js";
+import { inputOutputPath } from "../../helpers/inputOutputPath.js";
+import { inputSourcePath } from "../../helpers/inputSourcePath.js";
+import { selectExtractMode } from "../../helpers/selectExtractMode.js";
+import { selectLoader } from "../../helpers/selectLoader.js";
+import { isUrl } from "../../utils.js";
+import { loadHtml } from "./loadHtml.js";
+var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
+var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
+var webforaiCommand = async (initialPath, options) => {
+  intro(pc.bold(pc.green(`webforai CLI version ${packageInfo.version}`)));
+  const sourcePath = initialPath ?? await inputSourcePath();
+  options.debug && console.debug(`sourcePath: ${sourcePath}`);
+  const loader = isUrl(sourcePath) ? options.loader ?? await selectLoader() : "local";
+  options.debug && console.debug(`loader: ${loader}`);
+  const outputPath = options.output ?? await inputOutputPath(sourcePath);
+  options.debug && console.debug(`outputPath: ${outputPath}`);
+  const mode = options.mode ?? await selectExtractMode();
+  options.debug && console.debug(`mode: ${mode}`);
+  let html;
+  const s = spinner();
+  try {
+    s.start("Loading content...");
+    html = await loadHtml(sourcePath, loader, { debug: options.debug });
+    s.stop(pc.green("Content loaded!"));
+  } catch (error) {
+    s.stop(pc.red("Content loading failed!"));
+    console.error(error);
+    process.exit(1);
+  }
+  options.debug && console.debug(`html: ${html}`);
+  const markdown = htmlToMarkdown(html, {
+    baseUrl: isUrl(sourcePath) ? sourcePath : void 0,
+    ...mode === "ai" ? aiModeOptions : readabilityModeOptions
+  });
+  options.debug && console.debug(`markdown: ${markdown}`);
+  const directory = path.dirname(outputPath);
+  const isDirectoryExists = await fs.stat(directory).then((stat) => stat.isDirectory());
+  if (!isDirectoryExists) {
+    await fs.mkdir(directory, { recursive: true });
+  }
+  await fs.writeFile(outputPath, markdown);
+  outro(pc.green(`${pc.bold("Done!")} Markdown saved to ${outputPath}`));
+};
+export {
+  webforaiCommand
+};

package/dist/cli/commands/webforai/loadHtml.js

@@ -0,0 +1,60 @@
+// src/cli/commands/webforai/loadHtml.ts
+import fs from "node:fs/promises";
+import pc from "picocolors";
+import { chromium } from "playwright-core";
+import playwrightPackageJson from "playwright-core/package.json";
+import { loadHtml as loadHtmlByFetch } from "../../../loaders/fetch.js";
+import { loadHtml as loadHtmlByPlaywright } from "../../../loaders/playwright.js";
+var checkPlaywrightAvailable = async () => {
+  const path = chromium.executablePath();
+  const isAvailable = await fs.access(path).catch(() => false);
+  return !!isAvailable;
+};
+var loadHtml = async (sourcePath, loader, options) => {
+  if (loader === "local") {
+    options.debug && console.debug(`Loading HTML from local file: ${sourcePath}`);
+    const content = await fs.readFile(sourcePath, "utf-8");
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "fetch") {
+    options.debug && console.debug(`Loading HTML from URL: ${sourcePath}`);
+    const content = await loadHtmlByFetch(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "playwright") {
+    options.debug && console.debug(`Loading HTML from playwright: ${sourcePath}`);
+    const isPlaywrightAvailable = await checkPlaywrightAvailable();
+    options.debug && console.debug(`Playwright available: ${isPlaywrightAvailable}`);
+    if (!isPlaywrightAvailable) {
+      const message = [
+        "To use playwright loader, you need to install playwright.",
+        "You can install it by running:",
+        "",
+        `	${pc.bold(`npx playwright@${playwrightPackageJson.version} install chromium`)}	`,
+        "",
+        "Hint1: If you get warning like this:",
+        "WARNING: It looks like you are running 'npx playwright install' without first installing your project's dependencies. ",
+        `${pc.bold("Ignore this warning.")}`,
+        "",
+        "Hint2: If you get message like this:",
+        "Host system is missing dependencies to run browsers.",
+        "You should install dependencies by running:",
+        "",
+        `	${pc.bold(`sudo npx playwright@${playwrightPackageJson.version} install-deps`)}	`,
+        "",
+        pc.gray("Note: Good luck with that.")
+      ];
+      console.info(message.join("\n"));
+      throw new Error("Playwright is not available");
+    }
+    const content = await loadHtmlByPlaywright(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  throw new Error(`Unsupported loader: ${loader}`);
+};
+export {
+  loadHtml
+};

package/dist/cli/constants.js

@@ -0,0 +1,9 @@
+// src/cli/constants.ts
+var DEFAULT_PATH = "https://example.com";
+var LOADERS = ["fetch", "playwright"];
+var MODES = ["default", "ai"];
+export {
+  DEFAULT_PATH,
+  LOADERS,
+  MODES
+};

package/dist/cli/helpers/assertContinue.js

@@ -0,0 +1,11 @@
+// src/cli/helpers/assertContinue.ts
+import { cancel, isCancel } from "@clack/prompts";
+function assertContinue(message, cancelMessage = "Canceled.") {
+  if (isCancel(message)) {
+    cancel(cancelMessage);
+    process.exit(1);
+  }
+}
+export {
+  assertContinue
+};

package/dist/cli/helpers/inputOutputPath.js

@@ -0,0 +1,56 @@
+// src/cli/helpers/inputOutputPath.ts
+import fs from "node:fs";
+import { confirm, text } from "@clack/prompts";
+import { getNextAvailableFilePath, sourcePathToOutputPath } from "../utils.js";
+import { assertContinue } from "./assertContinue.js";
+var inputOutputPath = async (sourcePath) => {
+  const outputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: sourcePathToOutputPath(sourcePath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
+      }
+      if (fs.existsSync(value) && fs.statSync(value).isDirectory()) {
+        return "No directory can be specified.";
+      }
+      if (!fs.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
+      }
+    }
+  });
+  assertContinue(outputPath);
+  if (!fs.existsSync(outputPath)) {
+    return outputPath;
+  }
+  const isOutputFileOverwrite = await confirm({
+    message: "The file already exists. Overwrite?",
+    initialValue: false
+  });
+  assertContinue(isOutputFileOverwrite);
+  if (isOutputFileOverwrite) {
+    return outputPath;
+  }
+  const escapedOutputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: getNextAvailableFilePath(outputPath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
+      }
+      if (fs.existsSync(value)) {
+        return "The file already exists";
+      }
+      if (!fs.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
+      }
+    }
+  });
+  assertContinue(escapedOutputPath);
+  return escapedOutputPath;
+};
+export {
+  inputOutputPath
+};

package/dist/cli/helpers/inputSourcePath.js

@@ -0,0 +1,31 @@
+// src/cli/helpers/inputSourcePath.ts
+import fs from "node:fs";
+import { text } from "@clack/prompts";
+import { DEFAULT_PATH } from "../constants.js";
+import { isUrl } from "../utils.js";
+import { assertContinue } from "./assertContinue.js";
+var inputSourcePath = async () => {
+  const result = await text({
+    message: "Enter the URL or html path to be converted to markdown:",
+    placeholder: DEFAULT_PATH,
+    initialValue: "",
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Source is required";
+      }
+      if (!isUrl(value)) {
+        if (!fs.existsSync(value)) {
+          return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
+        }
+        if (fs.statSync(value).isDirectory()) {
+          return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
+        }
+      }
+    }
+  });
+  assertContinue(result);
+  return result;
+};
+export {
+  inputSourcePath
+};