webforai 1.5.1 → 1.6.0

package/dist/bin.js

@@ -1,64 +1,14 @@
-var __defProp = Object.defineProperty;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __esm = (fn, res) => function __init() {
-  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
-};
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-
-// src/loaders/fetch.ts
-var fetch_exports = {};
-__export(fetch_exports, {
-  USER_AGENT: () => USER_AGENT,
-  loadHtml: () => loadHtml
-});
-var USER_AGENT, loadHtml;
-var init_fetch = __esm({
-  "src/loaders/fetch.ts"() {
-    "use strict";
-    USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
-    loadHtml = async (url, userAgent = USER_AGENT) => {
-      const response = await fetch(url, { headers: { "User-Agent": userAgent } });
-      return response.text();
-    };
-  }
-});
-
-// src/loaders/playwright.ts
-var playwright_exports = {};
-__export(playwright_exports, {
-  loadHtml: () => loadHtml2
-});
-import { chromium } from "playwright";
-var loadHtml2;
-var init_playwright = __esm({
-  "src/loaders/playwright.ts"() {
-    "use strict";
-    loadHtml2 = async (url, context) => {
-      const _context = context ?? await chromium.launch({ headless: true });
-      const page = await _context.newPage();
-      await page.goto(url, { waitUntil: "load" });
-      const html = await page.content();
-      await page.close();
-      if (!context) {
-        await _context.close();
-      }
-      return html;
-    };
-  }
-});
+#!/usr/bin/env node
 
 // src/cli/bin.ts
-import { intro } from "@clack/prompts";
+import { intro as intro2 } from "@clack/prompts";
 import { program } from "commander";
-import pc2 from "picocolors";
+import pc3 from "picocolors";
 
 // package.json
 var package_default = {
   name: "webforai",
-  version: "1.5.1",
+  version: "1.6.0",
   description: "A library that provides a web interface for AI",
   author: "inaridiy",
   license: "MIT",
@@ -146,20 +96,30 @@ var package_default = {
   },
   peerDependencies: {
     "@cloudflare/puppeteer": ">=0.0.6",
-    playwright: ">=1.4",
+    "playwright-core": ">=1.4",
     puppeteer: ">=22"
   },
+  peerDependenciesMeta: {
+    "@cloudflare/puppeteer": {
+      optional: true
+    },
+    "playwright-core": {
+      optional: false
+    },
+    puppeteer: {
+      optional: true
+    }
+  },
   dependencies: {
     "@clack/prompts": "^0.7.0",
     commander: "^12.1.0",
-    "fastest-levenshtein": "^1.0.16",
-    "hast-util-from-html": "^2.0.1",
-    "hast-util-select": "^6.0.1",
-    "hast-util-to-html": "^9.0.0",
+    "hast-util-from-html": "^2.0.3",
+    "hast-util-select": "^6.0.2",
+    "hast-util-to-html": "^9.0.3",
     "hast-util-to-mdast": "^10.1.0",
     "hast-util-to-string": "^3.0.0",
     "hast-util-to-text": "^4.0.0",
-    "mathml-to-latex": "^1.3.0",
+    "mathml-to-latex": "^1.4.1",
     "mdast-util-gfm": "^3.0.0",
     "mdast-util-math": "^3.0.0",
     "mdast-util-to-markdown": "^2.1.0",
@@ -171,26 +131,28 @@ var package_default = {
   devDependencies: {
     "@cloudflare/puppeteer": "^0.0.6",
     "@tsconfig/recommended": "^1.0.3",
+    "fastest-levenshtein": "^1.0.16",
     "@types/hast": "^3.0.2",
     "@types/mdast": "^4.0.2",
-    "@types/node": "^20.11.0",
+    "@types/node": "^20.14.10",
     arg: "^5.0.2",
     esbuild: "^0.19.11",
     glob: "^10.3.10",
     ncp: "^2.0.0",
     np: "^9.2.0",
-    playwright: "^1.40.1",
+    "playwright-core": "^1.40.1",
     puppeteer: "^23.2.2",
     rimraf: "^5.0.5",
-    tsx: "^4.7.0",
+    tsx: "^4.19.1",
     typescript: "^5.4.5"
   }
 };
 
-// src/cli/commands/handlers/index.ts
-import fs5 from "node:fs";
+// src/cli/commands/webforai/index.ts
+import fs5 from "node:fs/promises";
 import path2 from "node:path";
-import { outro } from "@clack/prompts";
+import { intro, outro, spinner } from "@clack/prompts";
+import pc2 from "picocolors";
 
 // src/html-to-mdast.ts
 import { fromHtml } from "hast-util-from-html";
@@ -215,19 +177,6 @@ var isStrInclude = (value, match) => {
   }
   return false;
 };
-var hasAncestors = (element, tagNames, depth = 3) => {
-  let parent = element.parent;
-  for (let i = 0; i < depth; i++) {
-    if (!parent || parent.type !== "element") {
-      return false;
-    }
-    if (tagNames.includes(parent.tagName)) {
-      return true;
-    }
-    parent = parent.parent;
-  }
-  return false;
-};
 
 // src/extract-hast/readability.ts
 var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
@@ -235,35 +184,24 @@ var REGEXPS = {
   hidden: /hidden|invisible|fallback-image/i,
   byline: /byline|author|dateline|writtenby|p-author/i,
   specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
-  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
   okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
 };
-var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
-var PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
+var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
+var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
 var BASE_MINIMAL_LENGTH = {
   ja: 200,
   en: 500
 };
-var tryGetLang = (node) => {
-  if (node.type !== "element") {
-    return;
-  }
-  const element = node;
-  if (element.tagName !== "html") {
-    return;
-  }
-  const langAttr = element.properties.lang || element.properties["xml:lang"];
-  if (langAttr) {
-    return langAttr;
-  }
+var metadataFilter = (node) => {
+  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
 };
-var metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
 var universalElementFilter = (node) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (["script", "aside"].includes(element.tagName)) {
+  if (["aside", "nav"].includes(element.tagName)) {
     return false;
   }
   if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
@@ -294,10 +232,7 @@ var unlikelyElementFilter = (node) => {
     return true;
   }
   const element = node;
-  if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
-    return true;
-  }
-  if (hasAncestors(element, ["table", "code"], 3)) {
+  if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
     return true;
   }
   const match = matchString(element);
@@ -309,55 +244,36 @@ var unlikelyElementFilter = (node) => {
   }
   return true;
 };
-var isImageLink = (element) => {
-  const a = select("a", element);
-  const img = select("img", a);
-  if (!(a && img)) {
-    return false;
-  }
-  const imgFilename = img.properties.src?.toString().split("/").pop();
-  const hrefFilename = a.properties.href?.toString().split("/").pop();
-  return imgFilename === hrefFilename && imgFilename;
-};
-var removeEmptyFilter = (node) => {
+var removeEmptyFilter = (node, _lang) => {
   if (node.type !== "element") {
     return true;
   }
   const element = node;
-  if (!PARAGRAPH_TAGS.includes(element.tagName)) {
+  if (PARAGRAPH_TAGS.includes(element.tagName)) {
     return true;
   }
-  if (isImageLink(element)) {
-    return true;
-  }
-  const text3 = hastToString(element);
-  if (text3.length < 10) {
+  if (element.tagName === "img" && !element.properties.src) {
     return false;
   }
   return true;
 };
-var readabilityExtractHast = (hast) => {
-  const lang = String(select("html", hast)?.properties.lang || tryGetLang(hast) || "en");
+var readabilityExtractHast = (params) => {
+  const { hast, lang = "en" } = params;
   const body = select("body", hast) ?? hast;
-  const bodyText = hastToString(body);
-  const baseFilterd = filter(body, (node) => {
-    if (!metadataFilter(node)) {
-      return false;
-    }
-    if (!universalElementFilter(node)) {
-      return false;
-    }
-    return true;
-  });
-  if (!baseFilterd) {
-    return { type: "root", children: [] };
-  }
-  const baseFilterdText = hastToString(baseFilterd);
-  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
+  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
+  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
+  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+    return body;
+  }
+  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
+  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
+  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
   let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
   if (baseText.length < minimalLength) {
     minimalLength = Math.max(0, baseText.length - 200);
   }
+  let extractedTree = baseTree;
+  let extractedText = baseText;
   for (const selector of BODY_SELECTORS) {
     const content = { type: "root", children: selectAll(selector, baseFilterd) };
     const contentText = hastToString(content);
@@ -366,18 +282,18 @@ var readabilityExtractHast = (hast) => {
     }
     const links = selectAll("a", content);
     const linkText = links.map((link) => hastToString(link)).join("");
-    const linkDensity = linkText.length / bodyText.length;
+    const linkDensity = linkText.length / contentText.length;
     if (linkDensity > 0.4) {
       continue;
     }
     if (contentText.length > minimalLength) {
-      baseTree = content;
-      baseText = contentText;
+      extractedTree = content;
+      extractedText = contentText;
       break;
     }
   }
-  const finalFilteredTree = filter(baseTree, (node) => {
-    if (!removeEmptyFilter(node)) {
+  const finalFilteredTree = filter(extractedTree, (node) => {
+    if (!removeEmptyFilter(node, lang)) {
       return false;
     }
     if (!unlikelyElementFilter(node)) {
@@ -385,7 +301,7 @@ var readabilityExtractHast = (hast) => {
     }
     return true;
   });
-  const finalTree = hastToString(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
+  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
   return finalTree;
 };
 
@@ -393,7 +309,8 @@ var readabilityExtractHast = (hast) => {
 import { select as select2 } from "hast-util-select";
 import { filter as filter2 } from "unist-util-filter";
 var REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
-var stronglyExtractHast = (hast) => {
+var stronglyExtractHast = (params) => {
+  const { hast } = params;
   const selectors = ["article", "#article", "#content", "main", "div.container"];
   let selectedElement = hast;
   for (const selector of selectors) {
@@ -426,17 +343,18 @@ var PRESET_EXTRACT_HAST = {
   readability: readabilityExtractHast
 };
 var DEFAULT_EXTRACT_HAST = ["readability"];
-var extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
+var extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+  const { hast, lang } = params;
   const _extractors = Array.isArray(extractors) ? extractors : [extractors];
   const extracted = _extractors.reduce((acc, extractor) => {
     if (extractor === false) {
       return acc;
     }
     if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
-      return PRESET_EXTRACT_HAST[extractor](acc);
+      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
     }
     if (typeof extractor === "function") {
-      return extractor(acc);
+      return extractor({ hast: acc, lang });
     }
     throw new Error(`Invalid extractor: ${extractor}`);
   }, hast) || hast;
@@ -625,10 +543,6 @@ var customTableHandler = (options) => (state, node) => {
   return defaultHandlers4.table(state, node);
 };
 
-// src/mdast-handlers/empty-handler.ts
-var emptyHandler = () => {
-};
-
 // src/mdast-handlers/math-handler.ts
 import { toHtml } from "hast-util-to-html";
 import { MathMLToLaTeX } from "mathml-to-latex";
@@ -640,11 +554,54 @@ var mathHandler = (state, node) => {
   return result;
 };
 
+// src/utils/hast-utils.ts
+import { select as select4, selectAll as selectAll2 } from "hast-util-select";
+var getLangFromHast = (node) => {
+  const html = select4("html", node);
+  if (html && typeof html.properties.lang === "string") {
+    return html.properties.lang;
+  }
+  if (node.type !== "element") {
+    return;
+  }
+  const element = node;
+  if (element.tagName !== "html") {
+    return;
+  }
+  const langAttr = element.properties.lang || element.properties["xml:lang"];
+  if (langAttr) {
+    return langAttr;
+  }
+  return void 0;
+};
+var getLangFromStr = (str) => {
+  const match = str.match(/lang=["']([^"']+)["']/);
+  if (match) {
+    return match[1];
+  }
+  return void 0;
+};
+var getUrlFromHast = (node) => {
+  if (node.type !== "element") {
+    return void 0;
+  }
+  const metaTagAttributes = ["og:url", "twitter:url"];
+  const metaTags = selectAll2("meta", node);
+  for (const meta of metaTags) {
+    const property = meta.properties.property || meta.properties.name;
+    if (typeof property === "string" && metaTagAttributes.includes(property)) {
+      return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
+    }
+  }
+  return void 0;
+};
+
 // src/html-to-mdast.ts
 var htmlToMdast = (htmlOrHast, options) => {
-  const { extractors } = options || {};
-  const hast = typeof htmlOrHast === "string" ? fromHtml(htmlOrHast, { fragment: true }) : htmlOrHast;
-  const extractedHast = extractHast(hast, extractors);
+  const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
+  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
+  const url = defaultUrl || getUrlFromHast(hast);
+  const extractedHast = extractHast({ hast, lang, url }, extractors);
   const mdast = toMdast(extractedHast, {
     handlers: {
       math: mathHandler,
@@ -652,8 +609,7 @@ var htmlToMdast = (htmlOrHast, options) => {
       pre: customCodeHandler,
       a: customAHandler({ asText: options?.linkAsText }),
       img: customImgHandler({ hideImage: options?.hideImage }),
-      table: customTableHandler({ asText: options?.tableAsText }),
-      br: emptyHandler
+      table: customTableHandler({ asText: options?.tableAsText })
     }
   });
   const extractedMdast = extractMdast(mdast);
@@ -666,13 +622,18 @@ import { mathToMarkdown } from "mdast-util-math";
 import { toMarkdown } from "mdast-util-to-markdown";
 
 // src/link-replacer.ts
-var linkReplacer = (markdown, link) => {
-  const regex = /(!?\[.*?\]\()(.+?)(\))/g;
-  return markdown.replace(regex, (match, pre, thisUrl, post) => {
-    if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
+var linkReplacer = (markdown, base) => {
+  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
+  return markdown.replace(regex, (match, pre, url, post) => {
+    if (/^(https?:|#)/.test(url)) {
+      return match;
+    }
+    try {
+      const absoluteUrl = new URL(url, base).href;
+      return `${pre}${absoluteUrl}${post}`;
+    } catch {
       return match;
     }
-    return `${pre}${new URL(thisUrl, link).href}${post}`;
   });
 };
 
@@ -691,7 +652,7 @@ var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
 };
 var mdastToMarkdown = (mdast, options) => {
   const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
-  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions);
+  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
   if (baseUrl) {
     markdown = linkReplacer(markdown, baseUrl);
   }
@@ -722,71 +683,38 @@ var isUrl = (maybeUrl) => {
   }
 };
 function changeFileExtension(filePath, newExtension) {
-  const parsedPath = filePath.split("/");
-  const fileName = parsedPath[parsedPath.length - 1];
-  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
-  if (fileName.startsWith(".")) {
-    const parts = fileName.split(".");
-    if (parts.length === 2) {
-      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
-    }
-    parts[parts.length - 1] = newExtension.replace(/^\./, "");
-    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
-  }
-  const lastDotIndex = fileName.lastIndexOf(".");
-  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
-  const newFileName = `${baseName}${formattedNewExtension}`;
-  parsedPath[parsedPath.length - 1] = newFileName;
-  return parsedPath.join("/");
+  const parsed = path.parse(filePath);
+  const ext = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  return path.format({ ...parsed, ext });
 }
 function urlToFilename(url) {
   try {
     const urlObj = new URL(url);
-    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
-      if (index === 0) {
-        return acc;
-      }
-      if (acc.length >= 2) {
-        return acc;
-      }
-      if (part === "www") {
-        return acc;
-      }
-      return [part, ...acc];
-    }, []);
-    const domainString = domainParts.reverse().join("-");
+    const hostname = urlObj.hostname.startsWith("www.") ? urlObj.hostname.slice(4) : urlObj.hostname;
+    const domainParts = hostname.split(".").slice(-2).join("-");
     const pathParts = urlObj.pathname.split("/").filter(Boolean);
-    const relevantPathParts = pathParts.slice(-2);
-    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
-    let filename = [domainString, pathString].filter(Boolean).join("-");
+    const relevantPathParts = pathParts.slice(-2).map(decodeURIComponent).join("-");
+    let filename = [domainParts, relevantPathParts].filter(Boolean).join("-");
     filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
     return filename || "output";
   } catch {
     return "output";
   }
 }
-var sourcePathToOutputPath = (sourcePath) => {
-  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
-};
+var sourcePathToOutputPath = (sourcePath) => isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
 function getNextAvailableFilePath(filePath) {
-  const parsedPath = path.parse(filePath);
-  const directory = parsedPath.dir;
-  const fullName = parsedPath.base;
-  const [firstPart, ...restParts] = fullName.split(".");
-  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
-  const baseName = firstPart.replace(/_\d+$/, "");
-  let counter = 1;
-  let nextFilePath = filePath;
-  while (fs.existsSync(nextFilePath)) {
-    const match = firstPart.match(/_(\d+)$/);
-    if (match) {
-      counter = Number.parseInt(match[1], 10) + 1;
-    }
-    const newName = `${baseName}_${counter}${restName}`;
-    nextFilePath = path.join(directory, newName);
+  const parsed = path.parse(filePath);
+  const dir = parsed.dir;
+  const ext = parsed.ext;
+  const baseName = parsed.name.replace(/_\d+$/, "");
+  const match = parsed.name.match(/_(\d+)$/);
+  let counter = match ? Number(match[1]) + 1 : 1;
+  let newPath = path.join(dir, `${baseName}_${counter}${ext}`);
+  while (fs.existsSync(newPath)) {
     counter++;
+    newPath = path.join(dir, `${baseName}_${counter}${ext}`);
   }
-  return nextFilePath;
+  return newPath;
 }
 
 // src/cli/helpers/assertContinue.ts
@@ -882,9 +810,9 @@ var inputSourcePath = async () => {
 };
 
 // src/cli/helpers/selectExtractMode.ts
-import { select as select4 } from "@clack/prompts";
+import { select as select5 } from "@clack/prompts";
 var selectExtractMode = async () => {
-  const result = await select4({
+  const result = await select5({
     message: "Select processing mode:",
     options: MODES.map((mode) => ({ value: mode, label: mode })),
     initialValue: MODES[0]
@@ -894,14 +822,14 @@ var selectExtractMode = async () => {
 };
 
 // src/cli/helpers/selectLoader.ts
-import { select as select5 } from "@clack/prompts";
+import { select as select6 } from "@clack/prompts";
 var loadersHint = {
   fetch: "Fetch HTML content from the given URL",
   playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
   puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
 };
 var selectLoader = async () => {
-  const result = await select5({
+  const result = await select6({
     message: "Select loader:",
     initialValue: "fetch",
     options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
@@ -910,140 +838,155 @@ var selectLoader = async () => {
   return result;
 };
 
-// src/cli/commands/handlers/loadHtml.ts
+// src/cli/commands/webforai/loadHtml.ts
 import fs4 from "node:fs/promises";
-import { cancel as cancel2, spinner as clackSpinner, confirm as confirm2, spinner } from "@clack/prompts";
 import pc from "picocolors";
-import { $ } from "zx";
-var loadHtml3 = async (sourcePath, loader) => {
-  const loadSpinner = clackSpinner();
-  try {
-    switch (loader.type) {
-      case "local":
-        return await loadLocalHtml(sourcePath, loadSpinner);
-      case "web":
-        switch (loader.loader) {
-          case "fetch":
-            return await loadFetchHtml(sourcePath, loadSpinner);
-          case "playwright":
-            return await loadPlaywrightHtml(sourcePath, loadSpinner);
-          default:
-            throw new Error("Invalid loader");
-        }
-      default:
-        throw new Error("Invalid loader type");
-    }
-  } catch (error) {
-    loadSpinner.stop(pc.red("Content loading failed!"));
-    console.error(error);
-    process.exit(1);
-  }
-};
-var loadLocalHtml = async (sourcePath, loadSpinner) => {
-  loadSpinner.start("Retrieving content.");
-  try {
-    await fs4.access(sourcePath);
-  } catch {
-    throw new Error("The file does not exist");
-  }
-  const stats = await fs4.stat(sourcePath);
-  if (!stats.isFile()) {
-    throw new Error("The source must be a file");
-  }
-  const content = await fs4.readFile(sourcePath, "utf-8");
-  loadSpinner.stop("Content retrieval is complete!");
-  return content;
-};
-var loadFetchHtml = async (sourcePath, loadSpinner) => {
-  loadSpinner.start("Downloading content.");
-  const { loadHtml: loadHtml4 } = await Promise.resolve().then(() => (init_fetch(), fetch_exports));
-  const content = await loadHtml4(sourcePath);
-  loadSpinner.stop("Content download is complete!");
-  return content;
-};
-var loadPlaywrightHtml = async (sourcePath, loadSpinner) => {
-  const launchSpinner = clackSpinner();
-  launchSpinner.start("Launching playwright.");
-  const playwright = await import("playwright");
-  let browser;
-  try {
-    browser = await playwright.chromium.launch();
-    launchSpinner.stop("Playwright launched!");
-  } catch (error) {
-    await handlePlaywrightLaunchError(error);
-    browser = await playwright.chromium.launch();
-  }
-  loadSpinner.start("Loading content.");
-  const { loadHtml: loadHtml4 } = await Promise.resolve().then(() => (init_playwright(), playwright_exports));
-  const content = await loadHtml4(sourcePath, browser);
-  loadSpinner.stop("Content download is complete!");
-  await browser.close();
-  return content;
-};
-var handlePlaywrightLaunchError = async (error) => {
-  if (!error.message.includes("npx playwright install")) {
-    throw error;
-  }
-  const isInstall = await confirm2({
-    message: "To continue, you will need to download a browser. Do you wish to continue?"
-  });
-  if (!isInstall) {
-    cancel2("End of process.");
-    process.exit(1);
-  }
-  const installDepsSpinner = spinner();
-  installDepsSpinner.start("Installing browser dependencies.");
-  try {
-    await $`npx playwright install`;
-    installDepsSpinner.stop("Browser dependencies installed!");
-  } catch (error2) {
-    installDepsSpinner.stop(pc.red("Browser dependencies failed to install!"));
-    throw error2;
-  }
+import { chromium as chromium2 } from "playwright-core";
+import playwrightPackageJson from "playwright-core/package.json";
+
+// src/loaders/fetch.ts
+var USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+var loadHtml = async (url, userAgent = USER_AGENT) => {
+  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
+  return response.text();
 };
 
-// src/cli/commands/handlers/index.ts
+// src/loaders/playwright.ts
+import { chromium, devices } from "playwright-core";
+var SUPER_BYPASS_DEVICE = {
+  userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+  viewport: { width: 1920, height: 1080 },
+  deviceScaleFactor: 1,
+  hasTouch: false,
+  isMobile: false,
+  javaScriptEnabled: true,
+  locale: "en-US",
+  timezoneId: "America/New_York"
+};
+var loadHtml2 = async (url, options) => {
+  const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
+  const _browser = browser ?? await chromium.launch({ headless: true });
+  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : devices["Desktop Chrome"]);
+  if (superBypassMode) {
+    await context.addInitScript(() => {
+      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+    });
+  }
+  const page = await context.newPage();
+  if (superBypassMode) {
+    await page.route("**/*.js", (route) => {
+      if (route.request().url().includes("captcha-delivery")) {
+        return route.abort();
+      }
+      return route.continue();
+    });
+  }
+  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
+  const html = await page.content();
+  await page.close();
+  if (browser) {
+    await context.close();
+  } else {
+    await _browser.close();
+  }
+  return html;
+};
+
+// src/cli/commands/webforai/loadHtml.ts
+var checkPlaywrightAvailable = async () => {
+  const path3 = chromium2.executablePath();
+  const isAvailable = await fs4.access(path3).catch(() => false);
+  return !!isAvailable;
+};
+var loadHtml3 = async (sourcePath, loader, options) => {
+  if (loader === "local") {
+    options.debug && console.debug(`Loading HTML from local file: ${sourcePath}`);
+    const content = await fs4.readFile(sourcePath, "utf-8");
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "fetch") {
+    options.debug && console.debug(`Loading HTML from URL: ${sourcePath}`);
+    const content = await loadHtml(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "playwright") {
+    options.debug && console.debug(`Loading HTML from playwright: ${sourcePath}`);
+    const isPlaywrightAvailable = await checkPlaywrightAvailable();
+    options.debug && console.debug(`Playwright available: ${isPlaywrightAvailable}`);
+    if (!isPlaywrightAvailable) {
+      const message = [
+        "To use playwright loader, you need to install playwright.",
+        "You can install it by running:",
+        "",
+        `	${pc.bold(`npx playwright@${playwrightPackageJson.version} install chromium`)}	`,
+        "",
+        "Hint1: If you get warning like this:",
+        "WARNING: It looks like you are running 'npx playwright install' without first installing your project's dependencies. ",
+        `${pc.bold("Ignore this warning.")}`,
+        "",
+        "Hint2: If you get message like this:",
+        "Host system is missing dependencies to run browsers.",
+        "You should install dependencies by running:",
+        "",
+        `	${pc.bold(`sudo npx playwright@${playwrightPackageJson.version} install-deps`)}	`,
+        "",
+        pc.gray("Note: Good luck with that.")
+      ];
+      console.info(message.join("\n"));
+      throw new Error("Playwright is not available");
+    }
+    const content = await loadHtml2(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  throw new Error(`Unsupported loader: ${loader}`);
+};
+
+// src/cli/commands/webforai/index.ts
 var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
 var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
-var webforaiCommand = async (initialPath, initialOutputPath, options) => {
+var webforaiCommand = async (initialPath, options) => {
+  intro(pc2.bold(pc2.green(`webforai CLI version ${package_default.version}`)));
   const sourcePath = initialPath ?? await inputSourcePath();
-  if (options.debug) {
-    console.debug(`sourcePath: ${sourcePath}`);
-  }
-  const loader = isUrl(sourcePath) ? { type: "web", loader: options.loader ?? await selectLoader() } : { type: "local", loader: null };
-  if (options.debug) {
-    console.debug(`loader: ${loader}`);
-  }
-  const outputPath = initialOutputPath ?? await inputOutputPath(sourcePath);
-  if (options.debug) {
-    console.debug(`outputPath: ${outputPath}`);
-  }
+  options.debug && console.debug(`sourcePath: ${sourcePath}`);
+  const loader = isUrl(sourcePath) ? options.loader ?? await selectLoader() : "local";
+  options.debug && console.debug(`loader: ${loader}`);
+  const outputPath = options.output ?? await inputOutputPath(sourcePath);
+  options.debug && console.debug(`outputPath: ${outputPath}`);
   const mode = options.mode ?? await selectExtractMode();
-  if (options.debug) {
-    console.debug(`mode: ${mode}`);
-  }
-  const content = await loadHtml3(sourcePath, loader);
-  if (options.debug) {
-    console.debug(`content: ${content}`);
+  options.debug && console.debug(`mode: ${mode}`);
+  let html;
+  const s = spinner();
+  try {
+    s.start("Loading content...");
+    html = await loadHtml3(sourcePath, loader, { debug: options.debug });
+    s.stop(pc2.green("Content loaded!"));
+  } catch (error) {
+    s.stop(pc2.red("Content loading failed!"));
+    console.error(error);
+    process.exit(1);
   }
-  const markdown = htmlToMarkdown(content, {
-    baseUrl: isUrl(sourcePath) ? sourcePath : options.baseUrl,
+  options.debug && console.debug(`html: ${html}`);
+  const markdown = htmlToMarkdown(html, {
+    baseUrl: isUrl(sourcePath) ? sourcePath : void 0,
     ...mode === "ai" ? aiModeOptions : readabilityModeOptions
   });
-  if (options.debug) {
-    console.debug(`markdown: ${markdown}`);
-  }
+  options.debug && console.debug(`markdown: ${markdown}`);
   const directory = path2.dirname(outputPath);
-  if (!fs5.existsSync(directory)) {
-    fs5.mkdirSync(directory, { recursive: true });
+  const isDirectoryExists = await fs5.stat(directory).then((stat) => stat.isDirectory());
+  if (!isDirectoryExists) {
+    await fs5.mkdir(directory, { recursive: true });
   }
-  fs5.writeFileSync(outputPath, markdown);
-  outro("Markdown conversion is complete!");
+  await fs5.writeFile(outputPath, markdown);
+  outro(pc2.green(`${pc2.bold("Done!")} Markdown saved to ${outputPath}`));
 };
 
 // src/cli/bin.ts
-program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(package_default.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
-  process.env.DEBUG = "true";
-}).action(webforaiCommand);
-intro(pc2.inverse(`webforai CLI version ${package_default.version}`));
+program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(package_default.version, "-v, --version", "output the current version");
+program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(webforaiCommand);
+intro2(pc3.inverse(`webforai CLI version ${package_default.version}`));
 program.parse();