webforai 1.5.0 → 1.6.0

package/dist/bin.js

@@ -1,233 +1,992 @@
 #!/usr/bin/env node
 
-// src/bin.ts
-import fs from "node:fs";
-import path from "node:path";
-import { cancel, confirm, intro, isCancel, outro, select, spinner, text } from "@clack/prompts";
+// src/cli/bin.ts
+import { intro as intro2 } from "@clack/prompts";
 import { program } from "commander";
-import pc from "picocolors";
-import packageInfo from "../package.json" assert { type: "json" };
-import { htmlToMarkdown } from "./html-to-markdown.js";
-import { loadHtml as playwrightLoadHtml } from "./loaders/playwright.js";
-import { loadHtml as puppeteerLoadHtml } from "./loaders/puppeteer.js";
-import { changeFileExtension, getNextAvailableFilePath, isUrl, urlToFilename } from "./utils/bin-utils.js";
-var MODES = ["default", "ai"];
-var LOADERS = ["fetch", "playwright", "puppeteer"];
-var loadersHint = {
-  fetch: "Fetch HTML content from the given URL",
-  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
-  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
+import pc3 from "picocolors";
+
+// package.json
+var package_default = {
+  name: "webforai",
+  version: "1.6.0",
+  description: "A library that provides a web interface for AI",
+  author: "inaridiy",
+  license: "MIT",
+  keywords: [
+    "web",
+    "ai",
+    "html",
+    "html2md",
+    "markdown",
+    "mdast",
+    "hast"
+  ],
+  repository: {
+    type: "git",
+    url: "https://github.com/inaridiy/webforai.git"
+  },
+  homepage: "https://github.com/inaridiy/webforai.git",
+  scripts: {
+    "copy:package.cjs.json": "pnpm ncp ./package.cjs.json ./dist/cjs/package.json && pnpm ncp ./package.cjs.json ./dist/types/package.json ",
+    clean: "rimraf dist",
+    build: "pnpm clean && tsx build.ts && pnpm copy:package.cjs.json",
+    test: 'echo "Error: no test specified"',
+    prerelease: "pnpm build",
+    release: "np"
+  },
+  files: [
+    "dist",
+    "!dist/types/**/*.js"
+  ],
+  main: "dist/cjs/index.js",
+  type: "module",
+  module: "dist/index.js",
+  types: "dist/types/index.d.ts",
+  bin: "dist/bin.js",
+  exports: {
+    ".": {
+      types: "./dist/types/index.d.ts",
+      import: "./dist/index.js",
+      require: "./dist/cjs/index.js"
+    },
+    "./types": {
+      types: "./dist/types/index.d.ts",
+      import: "./dist/index.js",
+      require: "./dist/cjs/index.js"
+    },
+    "./loaders/playwright": {
+      types: "./dist/types/loaders/playwright.d.ts",
+      import: "./dist/loaders/playwright.js",
+      require: "./dist/cjs/loaders/playwright.js"
+    },
+    "./loaders/fetch": {
+      types: "./dist/types/loaders/fetch.d.ts",
+      import: "./dist/loaders/fetch.js",
+      require: "./dist/cjs/loaders/fetch.js"
+    },
+    "./loaders/cf-puppeteer": {
+      types: "./dist/types/loaders/cf-puppeteer.d.ts",
+      import: "./dist/loaders/cf-puppeteer.js",
+      require: "./dist/cjs/loaders/cf-puppeteer.js"
+    },
+    "./loaders/puppeteer": {
+      types: "./dist/types/loaders/puppeteer.d.ts",
+      import: "./dist/loaders/puppeteer.js",
+      require: "./dist/cjs/loaders/puppeteer.js"
+    }
+  },
+  typesVersions: {
+    "*": {
+      types: [
+        "./dist/types/index.d.ts"
+      ],
+      "loaders/playwright": [
+        "./dist/types/loaders/playwright.d.ts"
+      ],
+      "loaders/cf-puppeteer": [
+        "./dist/types/loaders/cf-puppeteer.d.ts"
+      ],
+      "loaders/fetch": [
+        "./dist/types/loaders/fetch.d.ts"
+      ],
+      "loaders/puppeteer": [
+        "./dist/types/loaders/puppeteer.d.ts"
+      ]
+    }
+  },
+  peerDependencies: {
+    "@cloudflare/puppeteer": ">=0.0.6",
+    "playwright-core": ">=1.4",
+    puppeteer: ">=22"
+  },
+  peerDependenciesMeta: {
+    "@cloudflare/puppeteer": {
+      optional: true
+    },
+    "playwright-core": {
+      optional: false
+    },
+    puppeteer: {
+      optional: true
+    }
+  },
+  dependencies: {
+    "@clack/prompts": "^0.7.0",
+    commander: "^12.1.0",
+    "hast-util-from-html": "^2.0.3",
+    "hast-util-select": "^6.0.2",
+    "hast-util-to-html": "^9.0.3",
+    "hast-util-to-mdast": "^10.1.0",
+    "hast-util-to-string": "^3.0.0",
+    "hast-util-to-text": "^4.0.0",
+    "mathml-to-latex": "^1.4.1",
+    "mdast-util-gfm": "^3.0.0",
+    "mdast-util-math": "^3.0.0",
+    "mdast-util-to-markdown": "^2.1.0",
+    picocolors: "^1.0.1",
+    "trim-trailing-lines": "^2.1.0",
+    "unist-util-filter": "^5.0.1",
+    zx: "^8.1.5"
+  },
+  devDependencies: {
+    "@cloudflare/puppeteer": "^0.0.6",
+    "@tsconfig/recommended": "^1.0.3",
+    "fastest-levenshtein": "^1.0.16",
+    "@types/hast": "^3.0.2",
+    "@types/mdast": "^4.0.2",
+    "@types/node": "^20.14.10",
+    arg: "^5.0.2",
+    esbuild: "^0.19.11",
+    glob: "^10.3.10",
+    ncp: "^2.0.0",
+    np: "^9.2.0",
+    "playwright-core": "^1.40.1",
+    puppeteer: "^23.2.2",
+    rimraf: "^5.0.5",
+    tsx: "^4.19.1",
+    typescript: "^5.4.5"
+  }
 };
-var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
-var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
-program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`, MODES[0]).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`, LOADERS[0]).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
-  process.env.DEBUG = "true";
-}).action(
-  async (source, outputPath, options) => {
-    let finalSource;
-    let sourceIsUrl;
-    let finalOutputPath;
-    let finalMode;
-    let finalLoader = "fetch";
-    const isOutputFile = !options.stdout;
-    if (process.env.DEBUG) {
-      console.info("Arguments and options:");
-      console.table({
-        source,
-        outputPath,
-        ...options
-      });
-      console.info("isOutputFile:", isOutputFile);
+
+// src/cli/commands/webforai/index.ts
+import fs5 from "node:fs/promises";
+import path2 from "node:path";
+import { intro, outro, spinner } from "@clack/prompts";
+import pc2 from "picocolors";
+
+// src/html-to-mdast.ts
+import { fromHtml } from "hast-util-from-html";
+import { toMdast } from "hast-util-to-mdast";
+
+// src/extract-hast/readability.ts
+import { select, selectAll } from "hast-util-select";
+import { toString as hastToString } from "hast-util-to-string";
+import { filter } from "unist-util-filter";
+
+// src/extract-hast/utils.ts
+var matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")}`;
+var classnames = (element) => {
+  if (Array.isArray(element.properties.className)) {
+    return element.properties.className;
+  }
+  return [];
+};
+var isStrInclude = (value, match) => {
+  if (typeof value === "string") {
+    return value.includes(match);
+  }
+  return false;
+};
+
+// src/extract-hast/readability.ts
+var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
+var REGEXPS = {
+  hidden: /hidden|invisible|fallback-image/i,
+  byline: /byline|author|dateline|writtenby|p-author/i,
+  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
+  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
+};
+var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
+var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
+var BASE_MINIMAL_LENGTH = {
+  ja: 200,
+  en: 500
+};
+var metadataFilter = (node) => {
+  return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
+};
+var universalElementFilter = (node) => {
+  if (node.type !== "element") {
+    return true;
+  }
+  const element = node;
+  if (["aside", "nav"].includes(element.tagName)) {
+    return false;
+  }
+  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
+    return false;
+  }
+  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
+    return false;
+  }
+  if (element.tagName === "dialog") {
+    return false;
+  }
+  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
+    return false;
+  }
+  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
+    return false;
+  }
+  if (REGEXPS.byline.test(matchString(element))) {
+    return false;
+  }
+  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
+    return false;
+  }
+  return true;
+};
+var unlikelyElementFilter = (node) => {
+  if (node.type !== "element") {
+    return true;
+  }
+  const element = node;
+  if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
+    return true;
+  }
+  const match = matchString(element);
+  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
+    return false;
+  }
+  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
+    return false;
+  }
+  return true;
+};
+var removeEmptyFilter = (node, _lang) => {
+  if (node.type !== "element") {
+    return true;
+  }
+  const element = node;
+  if (PARAGRAPH_TAGS.includes(element.tagName)) {
+    return true;
+  }
+  if (element.tagName === "img" && !element.properties.src) {
+    return false;
+  }
+  return true;
+};
+var readabilityExtractHast = (params) => {
+  const { hast, lang = "en" } = params;
+  const body = select("body", hast) ?? hast;
+  const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
+  const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
+  if (!(metadataFilteredHast && metadataFilteredHastText)) {
+    return body;
+  }
+  const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
+  const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
+  const [baseTree, baseText] = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [metadataFilteredHast, metadataFilteredHastText];
+  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
+  if (baseText.length < minimalLength) {
+    minimalLength = Math.max(0, baseText.length - 200);
+  }
+  let extractedTree = baseTree;
+  let extractedText = baseText;
+  for (const selector of BODY_SELECTORS) {
+    const content = { type: "root", children: selectAll(selector, baseFilterd) };
+    const contentText = hastToString(content);
+    if (contentText.length < 25) {
+      continue;
+    }
+    const links = selectAll("a", content);
+    const linkText = links.map((link) => hastToString(link)).join("");
+    const linkDensity = linkText.length / contentText.length;
+    if (linkDensity > 0.4) {
+      continue;
+    }
+    if (contentText.length > minimalLength) {
+      extractedTree = content;
+      extractedText = contentText;
+      break;
+    }
+  }
+  const finalFilteredTree = filter(extractedTree, (node) => {
+    if (!removeEmptyFilter(node, lang)) {
+      return false;
+    }
+    if (!unlikelyElementFilter(node)) {
+      return false;
+    }
+    return true;
+  });
+  const finalTree = hastToString(finalFilteredTree).length > extractedText.length / 3 ? finalFilteredTree : extractedTree;
+  return finalTree;
+};
+
+// src/extract-hast/strongly.ts
+import { select as select2 } from "hast-util-select";
+import { filter as filter2 } from "unist-util-filter";
+var REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
+var stronglyExtractHast = (params) => {
+  const { hast } = params;
+  const selectors = ["article", "#article", "#content", "main", "div.container"];
+  let selectedElement = hast;
+  for (const selector of selectors) {
+    const element = select2(selector, hast);
+    if (element) {
+      selectedElement = element;
+      break;
+    }
+  }
+  const removedElements = filter2(selectedElement, (node) => {
+    if (node.type === "comment") {
+      return false;
+    }
+    if (node.type !== "element") {
+      return true;
+    }
+    const element = node;
+    const elementName = `${element.tagName} ${element.properties.id} ${element.properties.className}`;
+    if (REMOVE_REGEX.test(elementName)) {
+      return false;
+    }
+    return true;
+  });
+  return removedElements;
+};
+
+// src/extract-hast/index.ts
+var PRESET_EXTRACT_HAST = {
+  strongly: stronglyExtractHast,
+  readability: readabilityExtractHast
+};
+var DEFAULT_EXTRACT_HAST = ["readability"];
+var extractHast = (params, extractors = DEFAULT_EXTRACT_HAST) => {
+  const { hast, lang } = params;
+  const _extractors = Array.isArray(extractors) ? extractors : [extractors];
+  const extracted = _extractors.reduce((acc, extractor) => {
+    if (extractor === false) {
+      return acc;
+    }
+    if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
+      return PRESET_EXTRACT_HAST[extractor]({ hast: acc, lang });
+    }
+    if (typeof extractor === "function") {
+      return extractor({ hast: acc, lang });
+    }
+    throw new Error(`Invalid extractor: ${extractor}`);
+  }, hast) || hast;
+  return extracted;
+};
+
+// src/extract-mdast.ts
+import { filter as filter3 } from "unist-util-filter";
+var DECLATION_TYPES = ["blockquote", "strong", "emphasis", "delete"];
+var emptyDeclarationFilter = (node) => {
+  if (!DECLATION_TYPES.includes(node.type)) {
+    return true;
+  }
+  if (node.children.length === 0) {
+    return false;
+  }
+  return true;
+};
+var extractMdast = (node) => {
+  const extracted = filter3(node, (node2) => {
+    if (!emptyDeclarationFilter(node2)) {
+      return false;
+    }
+    return true;
+  });
+  return extracted;
+};
+
+// src/mdast-handlers/custom-a-handler.ts
+import { defaultHandlers } from "hast-util-to-mdast";
+import { toString as hastToString2 } from "hast-util-to-string";
+var customAHandler = (options) => (state, node) => {
+  if (options?.asText) {
+    const text3 = hastToString2(node);
+    if (3 >= text3.length) {
+      return void 0;
+    }
+    const link2 = { type: "text", value: text3 };
+    state.patch(node, link2);
+    return link2;
+  }
+  const link = defaultHandlers.a(state, node);
+  if (link.children.length > 0) {
+    return link;
+  }
+  return void 0;
+};
+
+// src/mdast-handlers/custom-code-handler.ts
+import { toText } from "hast-util-to-text";
+import { trimTrailingLines } from "trim-trailing-lines";
+
+// src/utils/detect-code-lang.ts
+var languages = [
+  ["bash", [/#!(\/usr)?\/bin\/bash/g, 500], [/\b(if|elif|then|fi|echo)\b|\$/g, 10]],
+  ["html", [/<\/?[a-z-]+[^\n>]*>/g, 10], [/^\s+<!DOCTYPE\s+html/g, 500]],
+  ["http", [/^(GET|HEAD|POST|PUT|DELETE|PATCH|HTTP)\b/g, 500]],
+  [
+    "ts",
+    [/(import .* from|export \*|export const|const .* = await)/g, 400],
+    [
+      /\b(console|await|async|function|undefined|export|import|this|class|for|let|const|map|join|require|implements|interface|namespace)\b/g,
+      10
+    ]
+  ],
+  [
+    "tsx",
+    [/(import .* from|export \*|export const)/g, 400],
+    [/\b(react|next|FC)\b/g, 200],
+    [
+      /\b(console|await|async|function|export|undefined|import|this|class|for|let|const|map|join|require|implements|interface)\b/g,
+      10
+    ]
+  ],
+  ["py", [/\b(def|print|class|and|or|lambda)\b/g, 10]],
+  ["sql", [/\b(SELECT|INSERT|FROM)\b/g, 50]],
+  ["pl", [/#!(\/usr)?\/bin\/perl/g, 500], [/\b(use|print)\b|\$/g, 10]],
+  ["lua", [/#!(\/usr)?\/bin\/lua/g, 500]],
+  ["make", [/\b(ifneq|endif|if|elif|then|fi|echo|.PHONY|^[a-z]+ ?:$)\b|\$/gm, 10]],
+  ["uri", [/https?:|mailto:|tel:|ftp:/g, 30]],
+  ["css", [/^(@import|@page|@media|(\.|#)[a-z]+)/gm, 20]],
+  ["diff", [/^[+><-]/gm, 10], [/^@@ ?[-+,0-9 ]+ ?@@/gm, 25]],
+  ["md", [/^(>|\t\*|\t\d+.)/gm, 10], [/\[.*\](.*)/g, 10]],
+  ["docker", [/^(FROM|ENTRYPOINT|RUN)/gm, 500]],
+  ["xml", [/<\/?[a-z-]+[^\n>]*>/g, 10], [/^<\?xml/g, 500]],
+  ["c", [/#include\b|\bprintf\s+\(/g, 100]],
+  ["rs", [/^\s+(use|fn|mut|match)\b/gm, 100]],
+  ["go", [/\b(func|fmt|package)\b/g, 100]],
+  ["java", [/^import\s+java/gm, 500]],
+  ["asm", [/^(section|global main|extern|\t(call|mov|ret))/gm, 100]],
+  ["css", [/^(@import|@page|@media|(\.|#)[a-z]+)/gm, 20]],
+  ["json", [/\b(true|false|null|\{})\b|\"[^"]+\":/g, 10]],
+  ["yaml", [/^(\s+)?[a-z][a-z0-9]*:/gim, 10]]
+];
+var detectLanguage = (code) => {
+  return languages.map(
+    ([lang, ...features]) => [lang, features.reduce((acc, [match, score]) => acc + [...code.matchAll(match)].length * score, 0)]
+  ).filter(([_, score]) => score > 20).sort((a, b) => b[1] - a[1])[0]?.[0] || "plain";
+};
+
+// src/mdast-handlers/custom-code-handler.ts
+var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+var customCodeHandler = (state, node) => {
+  const classNames = node.properties?.className || [];
+  const codeValue = trimTrailingLines(toText(node)).trim();
+  const classLang = classNames.map((className) => {
+    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
+    return match?.[1];
+  }).find((className) => className);
+  const lang = classLang || detectLanguage(codeValue) || null;
+  const result = { type: "code", lang, meta: null, value: codeValue };
+  state.patch(node, result);
+  return result;
+};
+
+// src/mdast-handlers/custom-div-handler.ts
+import { select as select3 } from "hast-util-select";
+import { defaultHandlers as defaultHandlers2 } from "hast-util-to-mdast";
+import { toString as hastToString3 } from "hast-util-to-string";
+import { toText as toText2 } from "hast-util-to-text";
+import { trimTrailingLines as trimTrailingLines2 } from "trim-trailing-lines";
+var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
+var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
+var LANGUAGE_MATCH_REGEX2 = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+var findRecursive = (array, condition, maxDepth = 3) => {
+  if (maxDepth <= 0) {
+    return null;
+  }
+  for (const value of array) {
+    const result = condition(value);
+    if (Array.isArray(result)) {
+      return findRecursive(result, condition, maxDepth - 1);
+    }
+    if (result) {
+      return value;
+    }
+  }
+  return null;
+};
+var customDivHandler = (state, node) => {
+  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
+  const codeBlock = findRecursive(node.children, (child) => {
+    if (child.type !== "element") {
+      return false;
+    }
+    if (child.tagName === "pre") {
+      return true;
+    }
+    return child.children.filter((child2) => child2.type === "element");
+  });
+  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
+    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
+    const codeValue = trimTrailingLines2(toText2(codeBlock)).trim();
+    const filenameElement = select3(CODE_FILENAME_SELECTORS, node);
+    const fileLang = filenameElement ? hastToString3(filenameElement).match(/\.(\w+)$/)?.[1] : null;
+    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
+      const match = LANGUAGE_MATCH_REGEX2.map((regex) => className.match(regex)).find((match2) => match2);
+      return match?.[1];
+    }).find((className) => className);
+    const lang = fileLang || classLang || detectLanguage(codeValue) || null;
+    const result = { type: "code", lang, meta: null, value: codeValue };
+    state.patch(node, result);
+    return result;
+  }
+  return defaultHandlers2.div(state, node);
+};
+
+// src/mdast-handlers/custom-img-handler.ts
+import { defaultHandlers as defaultHandlers3 } from "hast-util-to-mdast";
+var customImgHandler = (options) => (state, node) => {
+  if (options?.hideImage) {
+    return void 0;
+  }
+  return defaultHandlers3.image(state, node);
+};
+
+// src/mdast-handlers/custom-table-handler.ts
+import { defaultHandlers as defaultHandlers4 } from "hast-util-to-mdast";
+import { toText as toText3 } from "hast-util-to-text";
+var customTableHandler = (options) => (state, node) => {
+  if (options?.asText) {
+    const paragraph = { type: "paragraph", children: [{ type: "text", value: toText3(node) }] };
+    state.patch(node, paragraph);
+    return paragraph;
+  }
+  return defaultHandlers4.table(state, node);
+};
+
+// src/mdast-handlers/math-handler.ts
+import { toHtml } from "hast-util-to-html";
+import { MathMLToLaTeX } from "mathml-to-latex";
+var mathHandler = (state, node) => {
+  const mathMl = toHtml(node);
+  const latex = MathMLToLaTeX.convert(mathMl);
+  const result = { type: "inlineMath", value: latex };
+  state.patch(node, result);
+  return result;
+};
+
+// src/utils/hast-utils.ts
+import { select as select4, selectAll as selectAll2 } from "hast-util-select";
+var getLangFromHast = (node) => {
+  const html = select4("html", node);
+  if (html && typeof html.properties.lang === "string") {
+    return html.properties.lang;
+  }
+  if (node.type !== "element") {
+    return;
+  }
+  const element = node;
+  if (element.tagName !== "html") {
+    return;
+  }
+  const langAttr = element.properties.lang || element.properties["xml:lang"];
+  if (langAttr) {
+    return langAttr;
+  }
+  return void 0;
+};
+var getLangFromStr = (str) => {
+  const match = str.match(/lang=["']([^"']+)["']/);
+  if (match) {
+    return match[1];
+  }
+  return void 0;
+};
+var getUrlFromHast = (node) => {
+  if (node.type !== "element") {
+    return void 0;
+  }
+  const metaTagAttributes = ["og:url", "twitter:url"];
+  const metaTags = selectAll2("meta", node);
+  for (const meta of metaTags) {
+    const property = meta.properties.property || meta.properties.name;
+    if (typeof property === "string" && metaTagAttributes.includes(property)) {
+      return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
+    }
+  }
+  return void 0;
+};
+
+// src/html-to-mdast.ts
+var htmlToMdast = (htmlOrHast, options) => {
+  const { extractors, url: defaultUrl, lang: defaultLang } = options || {};
+  const [lang, hast] = typeof htmlOrHast === "string" ? [defaultLang || getLangFromStr(htmlOrHast), fromHtml(htmlOrHast, { fragment: true })] : [defaultLang || getLangFromHast(htmlOrHast), htmlOrHast];
+  const url = defaultUrl || getUrlFromHast(hast);
+  const extractedHast = extractHast({ hast, lang, url }, extractors);
+  const mdast = toMdast(extractedHast, {
+    handlers: {
+      math: mathHandler,
+      div: customDivHandler,
+      pre: customCodeHandler,
+      a: customAHandler({ asText: options?.linkAsText }),
+      img: customImgHandler({ hideImage: options?.hideImage }),
+      table: customTableHandler({ asText: options?.tableAsText })
+    }
+  });
+  const extractedMdast = extractMdast(mdast);
+  return extractedMdast;
+};
+
+// src/mdast-to-markdown.ts
+import { gfmToMarkdown } from "mdast-util-gfm";
+import { mathToMarkdown } from "mdast-util-math";
+import { toMarkdown } from "mdast-util-to-markdown";
+
+// src/link-replacer.ts
+var linkReplacer = (markdown, base) => {
+  const regex = /(!?\[.*?\]\()([^)\s]+)(\))/g;
+  return markdown.replace(regex, (match, pre, url, post) => {
+    if (/^(https?:|#)/.test(url)) {
+      return match;
     }
     try {
-      finalSource = await text({
-        message: "Enter the URL or html path to be converted to markdown:",
-        placeholder: "https://example.com",
-        initialValue: source,
-        validate: (value) => {
-          if (value.trim() === "") {
-            return "Source is required";
-          }
-          if (!isUrl(value)) {
-            if (!fs.existsSync(value)) {
-              return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
-            }
-            if (fs.statSync(value).isDirectory()) {
-              return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
-            }
-          }
-        }
-      });
-      if (isCancel(finalSource)) {
-        cancel("Canceled.");
-        return;
-      }
-      sourceIsUrl = isUrl(finalSource);
-      if (sourceIsUrl) {
-        finalLoader = await select({
-          message: "Select loader:",
-          options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" })),
-          initialValue: options.loader && LOADERS.includes(options.loader) ? options.loader : "fetch"
-        });
-        if (isCancel(finalLoader)) {
-          cancel("Canceled.");
-          return;
-        }
-      }
-      if (isOutputFile) {
-        finalOutputPath = await text({
-          message: "Enter the output file path:",
-          placeholder: "output.md",
-          initialValue: outputPath ? outputPath : sourceIsUrl ? `${urlToFilename(finalSource)}.md` : changeFileExtension(finalSource, "md"),
-          validate: (value) => {
-            if (value.trim() === "") {
-              return "Output path is required";
-            }
-            if (fs.existsSync(value) && fs.statSync(value).isDirectory()) {
-              return "No directory can be specified.";
-            }
-            if (!fs.existsSync(value) && value.endsWith("/")) {
-              return "No directory can be specified.";
-            }
-          }
-        });
-        if (isCancel(finalOutputPath)) {
-          cancel("Canceled.");
-          return;
-        }
-        if (fs.existsSync(finalOutputPath)) {
-          const isOutputFileOverwrite = await confirm({
-            message: "The file already exists. Overwrite?",
-            initialValue: false
-          });
-          if (isCancel(isOutputFileOverwrite)) {
-            cancel("Canceled.");
-            return;
-          }
-          if (isOutputFileOverwrite === false) {
-            finalOutputPath = await text({
-              message: "Enter the output file path:",
-              placeholder: "output.md",
-              initialValue: getNextAvailableFilePath(finalOutputPath),
-              validate: (value) => {
-                if (value.trim() === "") {
-                  return "Output path is required";
-                }
-                if (fs.existsSync(value)) {
-                  return "The file already exists";
-                }
-                if (!fs.existsSync(value) && value.endsWith("/")) {
-                  return "No directory can be specified.";
-                }
-              }
-            });
-            if (isCancel(finalOutputPath)) {
-              cancel("Canceled.");
-              return;
-            }
-          }
-        }
-      } else {
-        finalOutputPath = "";
+      const absoluteUrl = new URL(url, base).href;
+      return `${pre}${absoluteUrl}${post}`;
+    } catch {
+      return match;
+    }
+  });
+};
+
+// src/utils/mdast-utils.ts
+var warpRoot = (mdast) => {
+  if (Array.isArray(mdast)) {
+    return { type: "root", children: mdast };
+  }
+  return mdast;
+};
+
+// src/mdast-to-markdown.ts
+var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
+  extensions: [gfmToMarkdown(), mathToMarkdown()],
+  bullet: "-"
+};
+var mdastToMarkdown = (mdast, options) => {
+  const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
+  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions).replace(/\*\*\*\*/g, "");
+  if (baseUrl) {
+    markdown = linkReplacer(markdown, baseUrl);
+  }
+  return markdown;
+};
+
+// src/html-to-markdown.ts
+var htmlToMarkdown = (htmlOrHast, options) => {
+  const { baseUrl, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
+  const mdast = htmlToMdast(htmlOrHast, toMdastOptions);
+  const markdown = mdastToMarkdown(mdast, { baseUrl, ...toMarkdownOptions });
+  return markdown;
+};
+
+// src/cli/helpers/inputOutputPath.ts
+import fs2 from "node:fs";
+import { confirm, text } from "@clack/prompts";
+
+// src/cli/utils.ts
+import fs from "node:fs";
+import path from "node:path";
+var isUrl = (maybeUrl) => {
+  try {
+    new URL(maybeUrl);
+    return true;
+  } catch {
+    return false;
+  }
+};
+function changeFileExtension(filePath, newExtension) {
+  const parsed = path.parse(filePath);
+  const ext = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  return path.format({ ...parsed, ext });
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const hostname = urlObj.hostname.startsWith("www.") ? urlObj.hostname.slice(4) : urlObj.hostname;
+    const domainParts = hostname.split(".").slice(-2).join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2).map(decodeURIComponent).join("-");
+    let filename = [domainParts, relevantPathParts].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+var sourcePathToOutputPath = (sourcePath) => isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
+function getNextAvailableFilePath(filePath) {
+  const parsed = path.parse(filePath);
+  const dir = parsed.dir;
+  const ext = parsed.ext;
+  const baseName = parsed.name.replace(/_\d+$/, "");
+  const match = parsed.name.match(/_(\d+)$/);
+  let counter = match ? Number(match[1]) + 1 : 1;
+  let newPath = path.join(dir, `${baseName}_${counter}${ext}`);
+  while (fs.existsSync(newPath)) {
+    counter++;
+    newPath = path.join(dir, `${baseName}_${counter}${ext}`);
+  }
+  return newPath;
+}
+
+// src/cli/helpers/assertContinue.ts
+import { cancel, isCancel } from "@clack/prompts";
+function assertContinue(message, cancelMessage = "Canceled.") {
+  if (isCancel(message)) {
+    cancel(cancelMessage);
+    process.exit(1);
+  }
+}
+
+// src/cli/helpers/inputOutputPath.ts
+var inputOutputPath = async (sourcePath) => {
+  const outputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: sourcePathToOutputPath(sourcePath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
       }
-      finalMode = await select({
-        message: "Select processing mode:",
-        options: MODES.map((mode) => ({ value: mode, label: mode })),
-        initialValue: options.mode && MODES.includes(options.mode) ? options.mode : "default"
-      });
-      if (isCancel(finalMode)) {
-        cancel("Canceled.");
-        return;
+      if (fs2.existsSync(value) && fs2.statSync(value).isDirectory()) {
+        return "No directory can be specified.";
       }
-    } catch (e) {
-      const error = e;
-      console.error(pc.red("An error occurred while processing the given value:"), error.message);
-      if (process.env.DEBUG) {
-        console.error(error.stack);
+      if (!fs2.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
       }
-      process.exit(1);
     }
-    if (process.env.DEBUG) {
-      console.info("Processing with the following options:");
-      console.table({ finalSource, sourceIsUrl, isOutputFile, finalOutputPath, finalMode, finalLoader });
-    }
-    let content;
-    const loadSpinner = spinner();
-    try {
-      if (sourceIsUrl) {
-        loadSpinner.start("Downloading content.");
-        if (finalLoader === "playwright") {
-          content = await playwrightLoadHtml(finalSource);
-        } else if (finalLoader === "puppeteer") {
-          content = await puppeteerLoadHtml(finalSource);
-        } else {
-          const response = await fetch(finalSource);
-          content = await response.text();
-        }
-        loadSpinner.stop("Content download is complete!");
-      } else {
-        loadSpinner.start("Retrieving content.");
-        if (!fs.existsSync(finalSource)) {
-          console.error(pc.red("The file does not exist"));
-          process.exit(1);
-        }
-        const stats = fs.statSync(finalSource);
-        if (!stats.isFile()) {
-          console.error(pc.red("The source must be a file"));
-          process.exit(1);
-        }
-        content = fs.readFileSync(finalSource, "utf-8");
-        loadSpinner.stop("Content retrieval is complete!");
+  });
+  assertContinue(outputPath);
+  if (!fs2.existsSync(outputPath)) {
+    return outputPath;
+  }
+  const isOutputFileOverwrite = await confirm({
+    message: "The file already exists. Overwrite?",
+    initialValue: false
+  });
+  assertContinue(isOutputFileOverwrite);
+  if (isOutputFileOverwrite) {
+    return outputPath;
+  }
+  const escapedOutputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: getNextAvailableFilePath(outputPath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
       }
-    } catch (e) {
-      const error = e;
-      loadSpinner.stop(pc.red("An error occurred while retrieving content"));
-      console.error(error.message);
-      if (process.env.DEBUG) {
-        console.error(error.stack);
+      if (fs2.existsSync(value)) {
+        return "The file already exists";
+      }
+      if (!fs2.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
       }
-      process.exit(1);
-    }
-    if (process.env.DEBUG) {
-      console.info("Content:");
-      console.info(content);
     }
-    const convertSpinner = spinner();
-    try {
-      convertSpinner.start("Converting content to markdown.");
-      const markdown = htmlToMarkdown(content, {
-        ...sourceIsUrl || !!options.baseUrl && options.baseUrl.trim() !== "" ? { baseUrl: options.baseUrl || finalSource } : {},
-        ...finalMode === "ai" ? aiModeOptions : readabilityModeOptions
-      });
-      if (process.env.DEBUG) {
-        console.info("Markdown:");
-        console.info(markdown);
+  });
+  assertContinue(escapedOutputPath);
+  return escapedOutputPath;
+};
+
+// src/cli/helpers/inputSourcePath.ts
+import fs3 from "node:fs";
+import { text as text2 } from "@clack/prompts";
+
+// src/cli/constants.ts
+var DEFAULT_PATH = "https://example.com";
+var LOADERS = ["fetch", "playwright"];
+var MODES = ["default", "ai"];
+
+// src/cli/helpers/inputSourcePath.ts
+var inputSourcePath = async () => {
+  const result = await text2({
+    message: "Enter the URL or html path to be converted to markdown:",
+    placeholder: DEFAULT_PATH,
+    initialValue: "",
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Source is required";
       }
-      if (isOutputFile) {
-        const directory = path.dirname(finalOutputPath);
-        if (!fs.existsSync(directory)) {
-          fs.mkdirSync(directory, { recursive: true });
+      if (!isUrl(value)) {
+        if (!fs3.existsSync(value)) {
+          return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
+        }
+        if (fs3.statSync(value).isDirectory()) {
+          return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
         }
-        fs.writeFileSync(finalOutputPath, markdown);
-        convertSpinner.stop("Markdown conversion is complete!");
-        outro(`Converted successfully to markdown! Output saved to: ${path.resolve(finalOutputPath)} \u{1F389}`);
-      } else {
-        convertSpinner.stop("Markdown conversion is complete!");
-        outro("Converted successfully to markdown\u{1F389}");
-        console.log(`
-
-${markdown}`);
       }
-    } catch (e) {
-      const error = e;
-      convertSpinner.stop(pc.red("An error occurred while converting and saving the markdown"));
-      console.error(error.message);
-      if (process.env.DEBUG) {
-        console.error(error.stack);
+    }
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/helpers/selectExtractMode.ts
+import { select as select5 } from "@clack/prompts";
+var selectExtractMode = async () => {
+  const result = await select5({
+    message: "Select processing mode:",
+    options: MODES.map((mode) => ({ value: mode, label: mode })),
+    initialValue: MODES[0]
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/helpers/selectLoader.ts
+import { select as select6 } from "@clack/prompts";
+var loadersHint = {
+  fetch: "Fetch HTML content from the given URL",
+  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
+  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
+};
+var selectLoader = async () => {
+  const result = await select6({
+    message: "Select loader:",
+    initialValue: "fetch",
+    options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/commands/webforai/loadHtml.ts
+import fs4 from "node:fs/promises";
+import pc from "picocolors";
+import { chromium as chromium2 } from "playwright-core";
+import playwrightPackageJson from "playwright-core/package.json";
+
+// src/loaders/fetch.ts
+var USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+var loadHtml = async (url, userAgent = USER_AGENT) => {
+  const response = await fetch(url, { headers: { "User-Agent": userAgent } });
+  return response.text();
+};
+
+// src/loaders/playwright.ts
+import { chromium, devices } from "playwright-core";
+var SUPER_BYPASS_DEVICE = {
+  userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+  viewport: { width: 1920, height: 1080 },
+  deviceScaleFactor: 1,
+  hasTouch: false,
+  isMobile: false,
+  javaScriptEnabled: true,
+  locale: "en-US",
+  timezoneId: "America/New_York"
+};
+var loadHtml2 = async (url, options) => {
+  const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
+  const _browser = browser ?? await chromium.launch({ headless: true });
+  const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : devices["Desktop Chrome"]);
+  if (superBypassMode) {
+    await context.addInitScript(() => {
+      Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
+      Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
+      Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
+    });
+  }
+  const page = await context.newPage();
+  if (superBypassMode) {
+    await page.route("**/*.js", (route) => {
+      if (route.request().url().includes("captcha-delivery")) {
+        return route.abort();
       }
-      process.exit(1);
+      return route.continue();
+    });
+  }
+  await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
+  const html = await page.content();
+  await page.close();
+  if (browser) {
+    await context.close();
+  } else {
+    await _browser.close();
+  }
+  return html;
+};
+
+// src/cli/commands/webforai/loadHtml.ts
+var checkPlaywrightAvailable = async () => {
+  const path3 = chromium2.executablePath();
+  const isAvailable = await fs4.access(path3).catch(() => false);
+  return !!isAvailable;
+};
+var loadHtml3 = async (sourcePath, loader, options) => {
+  if (loader === "local") {
+    options.debug && console.debug(`Loading HTML from local file: ${sourcePath}`);
+    const content = await fs4.readFile(sourcePath, "utf-8");
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "fetch") {
+    options.debug && console.debug(`Loading HTML from URL: ${sourcePath}`);
+    const content = await loadHtml(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  if (loader === "playwright") {
+    options.debug && console.debug(`Loading HTML from playwright: ${sourcePath}`);
+    const isPlaywrightAvailable = await checkPlaywrightAvailable();
+    options.debug && console.debug(`Playwright available: ${isPlaywrightAvailable}`);
+    if (!isPlaywrightAvailable) {
+      const message = [
+        "To use playwright loader, you need to install playwright.",
+        "You can install it by running:",
+        "",
+        `	${pc.bold(`npx playwright@${playwrightPackageJson.version} install chromium`)}	`,
+        "",
+        "Hint1: If you get warning like this:",
+        "WARNING: It looks like you are running 'npx playwright install' without first installing your project's dependencies. ",
+        `${pc.bold("Ignore this warning.")}`,
+        "",
+        "Hint2: If you get message like this:",
+        "Host system is missing dependencies to run browsers.",
+        "You should install dependencies by running:",
+        "",
+        `	${pc.bold(`sudo npx playwright@${playwrightPackageJson.version} install-deps`)}	`,
+        "",
+        pc.gray("Note: Good luck with that.")
+      ];
+      console.info(message.join("\n"));
+      throw new Error("Playwright is not available");
     }
+    const content = await loadHtml2(sourcePath);
+    options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
+    return content;
+  }
+  throw new Error(`Unsupported loader: ${loader}`);
+};
+
+// src/cli/commands/webforai/index.ts
+var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
+var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
+var webforaiCommand = async (initialPath, options) => {
+  intro(pc2.bold(pc2.green(`webforai CLI version ${package_default.version}`)));
+  const sourcePath = initialPath ?? await inputSourcePath();
+  options.debug && console.debug(`sourcePath: ${sourcePath}`);
+  const loader = isUrl(sourcePath) ? options.loader ?? await selectLoader() : "local";
+  options.debug && console.debug(`loader: ${loader}`);
+  const outputPath = options.output ?? await inputOutputPath(sourcePath);
+  options.debug && console.debug(`outputPath: ${outputPath}`);
+  const mode = options.mode ?? await selectExtractMode();
+  options.debug && console.debug(`mode: ${mode}`);
+  let html;
+  const s = spinner();
+  try {
+    s.start("Loading content...");
+    html = await loadHtml3(sourcePath, loader, { debug: options.debug });
+    s.stop(pc2.green("Content loaded!"));
+  } catch (error) {
+    s.stop(pc2.red("Content loading failed!"));
+    console.error(error);
+    process.exit(1);
   }
-);
-intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
+  options.debug && console.debug(`html: ${html}`);
+  const markdown = htmlToMarkdown(html, {
+    baseUrl: isUrl(sourcePath) ? sourcePath : void 0,
+    ...mode === "ai" ? aiModeOptions : readabilityModeOptions
+  });
+  options.debug && console.debug(`markdown: ${markdown}`);
+  const directory = path2.dirname(outputPath);
+  const isDirectoryExists = await fs5.stat(directory).then((stat) => stat.isDirectory());
+  if (!isDirectoryExists) {
+    await fs5.mkdir(directory, { recursive: true });
+  }
+  await fs5.writeFile(outputPath, markdown);
+  outro(pc2.green(`${pc2.bold("Done!")} Markdown saved to ${outputPath}`));
+};
+
+// src/cli/bin.ts
+program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(package_default.version, "-v, --version", "output the current version");
+program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(webforaiCommand);
+intro2(pc3.inverse(`webforai CLI version ${package_default.version}`));
 program.parse();