webforai 1.5.0 → 1.5.1

package/dist/bin.js

@@ -1,233 +1,1049 @@
-#!/usr/bin/env node
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/loaders/fetch.ts
+var fetch_exports = {};
+__export(fetch_exports, {
+  USER_AGENT: () => USER_AGENT,
+  loadHtml: () => loadHtml
+});
+var USER_AGENT, loadHtml;
+var init_fetch = __esm({
+  "src/loaders/fetch.ts"() {
+    "use strict";
+    USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
+    loadHtml = async (url, userAgent = USER_AGENT) => {
+      const response = await fetch(url, { headers: { "User-Agent": userAgent } });
+      return response.text();
+    };
+  }
+});
+
+// src/loaders/playwright.ts
+var playwright_exports = {};
+__export(playwright_exports, {
+  loadHtml: () => loadHtml2
+});
+import { chromium } from "playwright";
+var loadHtml2;
+var init_playwright = __esm({
+  "src/loaders/playwright.ts"() {
+    "use strict";
+    loadHtml2 = async (url, context) => {
+      const _context = context ?? await chromium.launch({ headless: true });
+      const page = await _context.newPage();
+      await page.goto(url, { waitUntil: "load" });
+      const html = await page.content();
+      await page.close();
+      if (!context) {
+        await _context.close();
+      }
+      return html;
+    };
+  }
+});
+
+// src/cli/bin.ts
+import { intro } from "@clack/prompts";
+import { program } from "commander";
+import pc2 from "picocolors";
+
+// package.json
+var package_default = {
+  name: "webforai",
+  version: "1.5.1",
+  description: "A library that provides a web interface for AI",
+  author: "inaridiy",
+  license: "MIT",
+  keywords: [
+    "web",
+    "ai",
+    "html",
+    "html2md",
+    "markdown",
+    "mdast",
+    "hast"
+  ],
+  repository: {
+    type: "git",
+    url: "https://github.com/inaridiy/webforai.git"
+  },
+  homepage: "https://github.com/inaridiy/webforai.git",
+  scripts: {
+    "copy:package.cjs.json": "pnpm ncp ./package.cjs.json ./dist/cjs/package.json && pnpm ncp ./package.cjs.json ./dist/types/package.json ",
+    clean: "rimraf dist",
+    build: "pnpm clean && tsx build.ts && pnpm copy:package.cjs.json",
+    test: 'echo "Error: no test specified"',
+    prerelease: "pnpm build",
+    release: "np"
+  },
+  files: [
+    "dist",
+    "!dist/types/**/*.js"
+  ],
+  main: "dist/cjs/index.js",
+  type: "module",
+  module: "dist/index.js",
+  types: "dist/types/index.d.ts",
+  bin: "dist/bin.js",
+  exports: {
+    ".": {
+      types: "./dist/types/index.d.ts",
+      import: "./dist/index.js",
+      require: "./dist/cjs/index.js"
+    },
+    "./types": {
+      types: "./dist/types/index.d.ts",
+      import: "./dist/index.js",
+      require: "./dist/cjs/index.js"
+    },
+    "./loaders/playwright": {
+      types: "./dist/types/loaders/playwright.d.ts",
+      import: "./dist/loaders/playwright.js",
+      require: "./dist/cjs/loaders/playwright.js"
+    },
+    "./loaders/fetch": {
+      types: "./dist/types/loaders/fetch.d.ts",
+      import: "./dist/loaders/fetch.js",
+      require: "./dist/cjs/loaders/fetch.js"
+    },
+    "./loaders/cf-puppeteer": {
+      types: "./dist/types/loaders/cf-puppeteer.d.ts",
+      import: "./dist/loaders/cf-puppeteer.js",
+      require: "./dist/cjs/loaders/cf-puppeteer.js"
+    },
+    "./loaders/puppeteer": {
+      types: "./dist/types/loaders/puppeteer.d.ts",
+      import: "./dist/loaders/puppeteer.js",
+      require: "./dist/cjs/loaders/puppeteer.js"
+    }
+  },
+  typesVersions: {
+    "*": {
+      types: [
+        "./dist/types/index.d.ts"
+      ],
+      "loaders/playwright": [
+        "./dist/types/loaders/playwright.d.ts"
+      ],
+      "loaders/cf-puppeteer": [
+        "./dist/types/loaders/cf-puppeteer.d.ts"
+      ],
+      "loaders/fetch": [
+        "./dist/types/loaders/fetch.d.ts"
+      ],
+      "loaders/puppeteer": [
+        "./dist/types/loaders/puppeteer.d.ts"
+      ]
+    }
+  },
+  peerDependencies: {
+    "@cloudflare/puppeteer": ">=0.0.6",
+    playwright: ">=1.4",
+    puppeteer: ">=22"
+  },
+  dependencies: {
+    "@clack/prompts": "^0.7.0",
+    commander: "^12.1.0",
+    "fastest-levenshtein": "^1.0.16",
+    "hast-util-from-html": "^2.0.1",
+    "hast-util-select": "^6.0.1",
+    "hast-util-to-html": "^9.0.0",
+    "hast-util-to-mdast": "^10.1.0",
+    "hast-util-to-string": "^3.0.0",
+    "hast-util-to-text": "^4.0.0",
+    "mathml-to-latex": "^1.3.0",
+    "mdast-util-gfm": "^3.0.0",
+    "mdast-util-math": "^3.0.0",
+    "mdast-util-to-markdown": "^2.1.0",
+    picocolors: "^1.0.1",
+    "trim-trailing-lines": "^2.1.0",
+    "unist-util-filter": "^5.0.1",
+    zx: "^8.1.5"
+  },
+  devDependencies: {
+    "@cloudflare/puppeteer": "^0.0.6",
+    "@tsconfig/recommended": "^1.0.3",
+    "@types/hast": "^3.0.2",
+    "@types/mdast": "^4.0.2",
+    "@types/node": "^20.11.0",
+    arg: "^5.0.2",
+    esbuild: "^0.19.11",
+    glob: "^10.3.10",
+    ncp: "^2.0.0",
+    np: "^9.2.0",
+    playwright: "^1.40.1",
+    puppeteer: "^23.2.2",
+    rimraf: "^5.0.5",
+    tsx: "^4.7.0",
+    typescript: "^5.4.5"
+  }
+};
+
+// src/cli/commands/handlers/index.ts
+import fs5 from "node:fs";
+import path2 from "node:path";
+import { outro } from "@clack/prompts";
+
+// src/html-to-mdast.ts
+import { fromHtml } from "hast-util-from-html";
+import { toMdast } from "hast-util-to-mdast";
+
+// src/extract-hast/readability.ts
+import { select, selectAll } from "hast-util-select";
+import { toString as hastToString } from "hast-util-to-string";
+import { filter } from "unist-util-filter";
+
+// src/extract-hast/utils.ts
+var matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")}`;
+var classnames = (element) => {
+  if (Array.isArray(element.properties.className)) {
+    return element.properties.className;
+  }
+  return [];
+};
+var isStrInclude = (value, match) => {
+  if (typeof value === "string") {
+    return value.includes(match);
+  }
+  return false;
+};
+var hasAncestors = (element, tagNames, depth = 3) => {
+  let parent = element.parent;
+  for (let i = 0; i < depth; i++) {
+    if (!parent || parent.type !== "element") {
+      return false;
+    }
+    if (tagNames.includes(parent.tagName)) {
+      return true;
+    }
+    parent = parent.parent;
+  }
+  return false;
+};
+
+// src/extract-hast/readability.ts
+var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
+var REGEXPS = {
+  hidden: /hidden|invisible|fallback-image/i,
+  byline: /byline|author|dateline|writtenby|p-author/i,
+  specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
+  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
+  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
+};
+var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
+var PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
+var BASE_MINIMAL_LENGTH = {
+  ja: 200,
+  en: 500
+};
+var tryGetLang = (node) => {
+  if (node.type !== "element") {
+    return;
+  }
+  const element = node;
+  if (element.tagName !== "html") {
+    return;
+  }
+  const langAttr = element.properties.lang || element.properties["xml:lang"];
+  if (langAttr) {
+    return langAttr;
+  }
+};
+var metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
+var universalElementFilter = (node) => {
+  if (node.type !== "element") {
+    return true;
+  }
+  const element = node;
+  if (["script", "aside"].includes(element.tagName)) {
+    return false;
+  }
+  if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
+    return false;
+  }
+  if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
+    return false;
+  }
+  if (element.tagName === "dialog") {
+    return false;
+  }
+  if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
+    return false;
+  }
+  if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
+    return false;
+  }
+  if (REGEXPS.byline.test(matchString(element))) {
+    return false;
+  }
+  if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
+    return false;
+  }
+  return true;
+};
+var unlikelyElementFilter = (node) => {
+  if (node.type !== "element") {
+    return true;
+  }
+  const element = node;
+  if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
+    return true;
+  }
+  if (hasAncestors(element, ["table", "code"], 3)) {
+    return true;
+  }
+  const match = matchString(element);
+  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
+    return false;
+  }
+  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
+    return false;
+  }
+  return true;
+};
+var isImageLink = (element) => {
+  const a = select("a", element);
+  const img = select("img", a);
+  if (!(a && img)) {
+    return false;
+  }
+  const imgFilename = img.properties.src?.toString().split("/").pop();
+  const hrefFilename = a.properties.href?.toString().split("/").pop();
+  return imgFilename === hrefFilename && imgFilename;
+};
+var removeEmptyFilter = (node) => {
+  if (node.type !== "element") {
+    return true;
+  }
+  const element = node;
+  if (!PARAGRAPH_TAGS.includes(element.tagName)) {
+    return true;
+  }
+  if (isImageLink(element)) {
+    return true;
+  }
+  const text3 = hastToString(element);
+  if (text3.length < 10) {
+    return false;
+  }
+  return true;
+};
+var readabilityExtractHast = (hast) => {
+  const lang = String(select("html", hast)?.properties.lang || tryGetLang(hast) || "en");
+  const body = select("body", hast) ?? hast;
+  const bodyText = hastToString(body);
+  const baseFilterd = filter(body, (node) => {
+    if (!metadataFilter(node)) {
+      return false;
+    }
+    if (!universalElementFilter(node)) {
+      return false;
+    }
+    return true;
+  });
+  if (!baseFilterd) {
+    return { type: "root", children: [] };
+  }
+  const baseFilterdText = hastToString(baseFilterd);
+  let [baseTree, baseText] = baseFilterdText.length > bodyText.length / 3 || baseFilterdText.length > 5e3 ? [baseFilterd, baseFilterdText] : [body, bodyText];
+  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
+  if (baseText.length < minimalLength) {
+    minimalLength = Math.max(0, baseText.length - 200);
+  }
+  for (const selector of BODY_SELECTORS) {
+    const content = { type: "root", children: selectAll(selector, baseFilterd) };
+    const contentText = hastToString(content);
+    if (contentText.length < 25) {
+      continue;
+    }
+    const links = selectAll("a", content);
+    const linkText = links.map((link) => hastToString(link)).join("");
+    const linkDensity = linkText.length / bodyText.length;
+    if (linkDensity > 0.4) {
+      continue;
+    }
+    if (contentText.length > minimalLength) {
+      baseTree = content;
+      baseText = contentText;
+      break;
+    }
+  }
+  const finalFilteredTree = filter(baseTree, (node) => {
+    if (!removeEmptyFilter(node)) {
+      return false;
+    }
+    if (!unlikelyElementFilter(node)) {
+      return false;
+    }
+    return true;
+  });
+  const finalTree = hastToString(finalFilteredTree).length > baseText.length / 3 ? finalFilteredTree : baseTree;
+  return finalTree;
+};
+
+// src/extract-hast/strongly.ts
+import { select as select2 } from "hast-util-select";
+import { filter as filter2 } from "unist-util-filter";
+var REMOVE_REGEX = /fallback|icon|header|nav|speechify-ignore|assist|-ad-|banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|assist|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i;
+var stronglyExtractHast = (hast) => {
+  const selectors = ["article", "#article", "#content", "main", "div.container"];
+  let selectedElement = hast;
+  for (const selector of selectors) {
+    const element = select2(selector, hast);
+    if (element) {
+      selectedElement = element;
+      break;
+    }
+  }
+  const removedElements = filter2(selectedElement, (node) => {
+    if (node.type === "comment") {
+      return false;
+    }
+    if (node.type !== "element") {
+      return true;
+    }
+    const element = node;
+    const elementName = `${element.tagName} ${element.properties.id} ${element.properties.className}`;
+    if (REMOVE_REGEX.test(elementName)) {
+      return false;
+    }
+    return true;
+  });
+  return removedElements;
+};
+
+// src/extract-hast/index.ts
+var PRESET_EXTRACT_HAST = {
+  strongly: stronglyExtractHast,
+  readability: readabilityExtractHast
+};
+var DEFAULT_EXTRACT_HAST = ["readability"];
+var extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
+  const _extractors = Array.isArray(extractors) ? extractors : [extractors];
+  const extracted = _extractors.reduce((acc, extractor) => {
+    if (extractor === false) {
+      return acc;
+    }
+    if (typeof extractor === "string" && extractor in PRESET_EXTRACT_HAST) {
+      return PRESET_EXTRACT_HAST[extractor](acc);
+    }
+    if (typeof extractor === "function") {
+      return extractor(acc);
+    }
+    throw new Error(`Invalid extractor: ${extractor}`);
+  }, hast) || hast;
+  return extracted;
+};
+
+// src/extract-mdast.ts
+import { filter as filter3 } from "unist-util-filter";
+var DECLATION_TYPES = ["blockquote", "strong", "emphasis", "delete"];
+var emptyDeclarationFilter = (node) => {
+  if (!DECLATION_TYPES.includes(node.type)) {
+    return true;
+  }
+  if (node.children.length === 0) {
+    return false;
+  }
+  return true;
+};
+var extractMdast = (node) => {
+  const extracted = filter3(node, (node2) => {
+    if (!emptyDeclarationFilter(node2)) {
+      return false;
+    }
+    return true;
+  });
+  return extracted;
+};
+
+// src/mdast-handlers/custom-a-handler.ts
+import { defaultHandlers } from "hast-util-to-mdast";
+import { toString as hastToString2 } from "hast-util-to-string";
+var customAHandler = (options) => (state, node) => {
+  if (options?.asText) {
+    const text3 = hastToString2(node);
+    if (3 >= text3.length) {
+      return void 0;
+    }
+    const link2 = { type: "text", value: text3 };
+    state.patch(node, link2);
+    return link2;
+  }
+  const link = defaultHandlers.a(state, node);
+  if (link.children.length > 0) {
+    return link;
+  }
+  return void 0;
+};
+
+// src/mdast-handlers/custom-code-handler.ts
+import { toText } from "hast-util-to-text";
+import { trimTrailingLines } from "trim-trailing-lines";
+
+// src/utils/detect-code-lang.ts
+var languages = [
+  ["bash", [/#!(\/usr)?\/bin\/bash/g, 500], [/\b(if|elif|then|fi|echo)\b|\$/g, 10]],
+  ["html", [/<\/?[a-z-]+[^\n>]*>/g, 10], [/^\s+<!DOCTYPE\s+html/g, 500]],
+  ["http", [/^(GET|HEAD|POST|PUT|DELETE|PATCH|HTTP)\b/g, 500]],
+  [
+    "ts",
+    [/(import .* from|export \*|export const|const .* = await)/g, 400],
+    [
+      /\b(console|await|async|function|undefined|export|import|this|class|for|let|const|map|join|require|implements|interface|namespace)\b/g,
+      10
+    ]
+  ],
+  [
+    "tsx",
+    [/(import .* from|export \*|export const)/g, 400],
+    [/\b(react|next|FC)\b/g, 200],
+    [
+      /\b(console|await|async|function|export|undefined|import|this|class|for|let|const|map|join|require|implements|interface)\b/g,
+      10
+    ]
+  ],
+  ["py", [/\b(def|print|class|and|or|lambda)\b/g, 10]],
+  ["sql", [/\b(SELECT|INSERT|FROM)\b/g, 50]],
+  ["pl", [/#!(\/usr)?\/bin\/perl/g, 500], [/\b(use|print)\b|\$/g, 10]],
+  ["lua", [/#!(\/usr)?\/bin\/lua/g, 500]],
+  ["make", [/\b(ifneq|endif|if|elif|then|fi|echo|.PHONY|^[a-z]+ ?:$)\b|\$/gm, 10]],
+  ["uri", [/https?:|mailto:|tel:|ftp:/g, 30]],
+  ["css", [/^(@import|@page|@media|(\.|#)[a-z]+)/gm, 20]],
+  ["diff", [/^[+><-]/gm, 10], [/^@@ ?[-+,0-9 ]+ ?@@/gm, 25]],
+  ["md", [/^(>|\t\*|\t\d+.)/gm, 10], [/\[.*\](.*)/g, 10]],
+  ["docker", [/^(FROM|ENTRYPOINT|RUN)/gm, 500]],
+  ["xml", [/<\/?[a-z-]+[^\n>]*>/g, 10], [/^<\?xml/g, 500]],
+  ["c", [/#include\b|\bprintf\s+\(/g, 100]],
+  ["rs", [/^\s+(use|fn|mut|match)\b/gm, 100]],
+  ["go", [/\b(func|fmt|package)\b/g, 100]],
+  ["java", [/^import\s+java/gm, 500]],
+  ["asm", [/^(section|global main|extern|\t(call|mov|ret))/gm, 100]],
+  ["css", [/^(@import|@page|@media|(\.|#)[a-z]+)/gm, 20]],
+  ["json", [/\b(true|false|null|\{})\b|\"[^"]+\":/g, 10]],
+  ["yaml", [/^(\s+)?[a-z][a-z0-9]*:/gim, 10]]
+];
+var detectLanguage = (code) => {
+  return languages.map(
+    ([lang, ...features]) => [lang, features.reduce((acc, [match, score]) => acc + [...code.matchAll(match)].length * score, 0)]
+  ).filter(([_, score]) => score > 20).sort((a, b) => b[1] - a[1])[0]?.[0] || "plain";
+};
+
+// src/mdast-handlers/custom-code-handler.ts
+var LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+var customCodeHandler = (state, node) => {
+  const classNames = node.properties?.className || [];
+  const codeValue = trimTrailingLines(toText(node)).trim();
+  const classLang = classNames.map((className) => {
+    const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
+    return match?.[1];
+  }).find((className) => className);
+  const lang = classLang || detectLanguage(codeValue) || null;
+  const result = { type: "code", lang, meta: null, value: codeValue };
+  state.patch(node, result);
+  return result;
+};
+
+// src/mdast-handlers/custom-div-handler.ts
+import { select as select3 } from "hast-util-select";
+import { defaultHandlers as defaultHandlers2 } from "hast-util-to-mdast";
+import { toString as hastToString3 } from "hast-util-to-string";
+import { toText as toText2 } from "hast-util-to-text";
+import { trimTrailingLines as trimTrailingLines2 } from "trim-trailing-lines";
+var CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
+var CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
+var LANGUAGE_MATCH_REGEX2 = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
+var findRecursive = (array, condition, maxDepth = 3) => {
+  if (maxDepth <= 0) {
+    return null;
+  }
+  for (const value of array) {
+    const result = condition(value);
+    if (Array.isArray(result)) {
+      return findRecursive(result, condition, maxDepth - 1);
+    }
+    if (result) {
+      return value;
+    }
+  }
+  return null;
+};
+var customDivHandler = (state, node) => {
+  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
+  const codeBlock = findRecursive(node.children, (child) => {
+    if (child.type !== "element") {
+      return false;
+    }
+    if (child.tagName === "pre") {
+      return true;
+    }
+    return child.children.filter((child2) => child2.type === "element");
+  });
+  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
+    const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
+    const codeValue = trimTrailingLines2(toText2(codeBlock)).trim();
+    const filenameElement = select3(CODE_FILENAME_SELECTORS, node);
+    const fileLang = filenameElement ? hastToString3(filenameElement).match(/\.(\w+)$/)?.[1] : null;
+    const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
+      const match = LANGUAGE_MATCH_REGEX2.map((regex) => className.match(regex)).find((match2) => match2);
+      return match?.[1];
+    }).find((className) => className);
+    const lang = fileLang || classLang || detectLanguage(codeValue) || null;
+    const result = { type: "code", lang, meta: null, value: codeValue };
+    state.patch(node, result);
+    return result;
+  }
+  return defaultHandlers2.div(state, node);
+};
 
-// src/bin.ts
+// src/mdast-handlers/custom-img-handler.ts
+import { defaultHandlers as defaultHandlers3 } from "hast-util-to-mdast";
+var customImgHandler = (options) => (state, node) => {
+  if (options?.hideImage) {
+    return void 0;
+  }
+  return defaultHandlers3.image(state, node);
+};
+
+// src/mdast-handlers/custom-table-handler.ts
+import { defaultHandlers as defaultHandlers4 } from "hast-util-to-mdast";
+import { toText as toText3 } from "hast-util-to-text";
+var customTableHandler = (options) => (state, node) => {
+  if (options?.asText) {
+    const paragraph = { type: "paragraph", children: [{ type: "text", value: toText3(node) }] };
+    state.patch(node, paragraph);
+    return paragraph;
+  }
+  return defaultHandlers4.table(state, node);
+};
+
+// src/mdast-handlers/empty-handler.ts
+var emptyHandler = () => {
+};
+
+// src/mdast-handlers/math-handler.ts
+import { toHtml } from "hast-util-to-html";
+import { MathMLToLaTeX } from "mathml-to-latex";
+var mathHandler = (state, node) => {
+  const mathMl = toHtml(node);
+  const latex = MathMLToLaTeX.convert(mathMl);
+  const result = { type: "inlineMath", value: latex };
+  state.patch(node, result);
+  return result;
+};
+
+// src/html-to-mdast.ts
+var htmlToMdast = (htmlOrHast, options) => {
+  const { extractors } = options || {};
+  const hast = typeof htmlOrHast === "string" ? fromHtml(htmlOrHast, { fragment: true }) : htmlOrHast;
+  const extractedHast = extractHast(hast, extractors);
+  const mdast = toMdast(extractedHast, {
+    handlers: {
+      math: mathHandler,
+      div: customDivHandler,
+      pre: customCodeHandler,
+      a: customAHandler({ asText: options?.linkAsText }),
+      img: customImgHandler({ hideImage: options?.hideImage }),
+      table: customTableHandler({ asText: options?.tableAsText }),
+      br: emptyHandler
+    }
+  });
+  const extractedMdast = extractMdast(mdast);
+  return extractedMdast;
+};
+
+// src/mdast-to-markdown.ts
+import { gfmToMarkdown } from "mdast-util-gfm";
+import { mathToMarkdown } from "mdast-util-math";
+import { toMarkdown } from "mdast-util-to-markdown";
+
+// src/link-replacer.ts
+var linkReplacer = (markdown, link) => {
+  const regex = /(!?\[.*?\]\()(.+?)(\))/g;
+  return markdown.replace(regex, (match, pre, thisUrl, post) => {
+    if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
+      return match;
+    }
+    return `${pre}${new URL(thisUrl, link).href}${post}`;
+  });
+};
+
+// src/utils/mdast-utils.ts
+var warpRoot = (mdast) => {
+  if (Array.isArray(mdast)) {
+    return { type: "root", children: mdast };
+  }
+  return mdast;
+};
+
+// src/mdast-to-markdown.ts
+var DEFAULT_MDAST_TO_MARKDOWN_OPTIONS = {
+  extensions: [gfmToMarkdown(), mathToMarkdown()],
+  bullet: "-"
+};
+var mdastToMarkdown = (mdast, options) => {
+  const { baseUrl, ...toMarkdownOptions } = { ...DEFAULT_MDAST_TO_MARKDOWN_OPTIONS, ...options };
+  let markdown = toMarkdown(warpRoot(mdast), toMarkdownOptions);
+  if (baseUrl) {
+    markdown = linkReplacer(markdown, baseUrl);
+  }
+  return markdown;
+};
+
+// src/html-to-markdown.ts
+var htmlToMarkdown = (htmlOrHast, options) => {
+  const { baseUrl, formatting: toMarkdownOptions, ...toMdastOptions } = options || {};
+  const mdast = htmlToMdast(htmlOrHast, toMdastOptions);
+  const markdown = mdastToMarkdown(mdast, { baseUrl, ...toMarkdownOptions });
+  return markdown;
+};
+
+// src/cli/helpers/inputOutputPath.ts
+import fs2 from "node:fs";
+import { confirm, text } from "@clack/prompts";
+
+// src/cli/utils.ts
 import fs from "node:fs";
 import path from "node:path";
-import { cancel, confirm, intro, isCancel, outro, select, spinner, text } from "@clack/prompts";
-import { program } from "commander";
-import pc from "picocolors";
-import packageInfo from "../package.json" assert { type: "json" };
-import { htmlToMarkdown } from "./html-to-markdown.js";
-import { loadHtml as playwrightLoadHtml } from "./loaders/playwright.js";
-import { loadHtml as puppeteerLoadHtml } from "./loaders/puppeteer.js";
-import { changeFileExtension, getNextAvailableFilePath, isUrl, urlToFilename } from "./utils/bin-utils.js";
-var MODES = ["default", "ai"];
-var LOADERS = ["fetch", "playwright", "puppeteer"];
-var loadersHint = {
-  fetch: "Fetch HTML content from the given URL",
-  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
-  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
+var isUrl = (maybeUrl) => {
+  try {
+    new URL(maybeUrl);
+    return true;
+  } catch {
+    return false;
+  }
 };
-var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
-var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
-program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`, MODES[0]).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`, LOADERS[0]).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
-  process.env.DEBUG = "true";
-}).action(
-  async (source, outputPath, options) => {
-    let finalSource;
-    let sourceIsUrl;
-    let finalOutputPath;
-    let finalMode;
-    let finalLoader = "fetch";
-    const isOutputFile = !options.stdout;
-    if (process.env.DEBUG) {
-      console.info("Arguments and options:");
-      console.table({
-        source,
-        outputPath,
-        ...options
-      });
-      console.info("isOutputFile:", isOutputFile);
-    }
-    try {
-      finalSource = await text({
-        message: "Enter the URL or html path to be converted to markdown:",
-        placeholder: "https://example.com",
-        initialValue: source,
-        validate: (value) => {
-          if (value.trim() === "") {
-            return "Source is required";
-          }
-          if (!isUrl(value)) {
-            if (!fs.existsSync(value)) {
-              return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
-            }
-            if (fs.statSync(value).isDirectory()) {
-              return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
-            }
-          }
-        }
-      });
-      if (isCancel(finalSource)) {
-        cancel("Canceled.");
-        return;
+function changeFileExtension(filePath, newExtension) {
+  const parsedPath = filePath.split("/");
+  const fileName = parsedPath[parsedPath.length - 1];
+  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  if (fileName.startsWith(".")) {
+    const parts = fileName.split(".");
+    if (parts.length === 2) {
+      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
+    }
+    parts[parts.length - 1] = newExtension.replace(/^\./, "");
+    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+  }
+  const lastDotIndex = fileName.lastIndexOf(".");
+  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
+  const newFileName = `${baseName}${formattedNewExtension}`;
+  parsedPath[parsedPath.length - 1] = newFileName;
+  return parsedPath.join("/");
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
+      if (index === 0) {
+        return acc;
       }
-      sourceIsUrl = isUrl(finalSource);
-      if (sourceIsUrl) {
-        finalLoader = await select({
-          message: "Select loader:",
-          options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" })),
-          initialValue: options.loader && LOADERS.includes(options.loader) ? options.loader : "fetch"
-        });
-        if (isCancel(finalLoader)) {
-          cancel("Canceled.");
-          return;
-        }
+      if (acc.length >= 2) {
+        return acc;
       }
-      if (isOutputFile) {
-        finalOutputPath = await text({
-          message: "Enter the output file path:",
-          placeholder: "output.md",
-          initialValue: outputPath ? outputPath : sourceIsUrl ? `${urlToFilename(finalSource)}.md` : changeFileExtension(finalSource, "md"),
-          validate: (value) => {
-            if (value.trim() === "") {
-              return "Output path is required";
-            }
-            if (fs.existsSync(value) && fs.statSync(value).isDirectory()) {
-              return "No directory can be specified.";
-            }
-            if (!fs.existsSync(value) && value.endsWith("/")) {
-              return "No directory can be specified.";
-            }
-          }
-        });
-        if (isCancel(finalOutputPath)) {
-          cancel("Canceled.");
-          return;
-        }
-        if (fs.existsSync(finalOutputPath)) {
-          const isOutputFileOverwrite = await confirm({
-            message: "The file already exists. Overwrite?",
-            initialValue: false
-          });
-          if (isCancel(isOutputFileOverwrite)) {
-            cancel("Canceled.");
-            return;
-          }
-          if (isOutputFileOverwrite === false) {
-            finalOutputPath = await text({
-              message: "Enter the output file path:",
-              placeholder: "output.md",
-              initialValue: getNextAvailableFilePath(finalOutputPath),
-              validate: (value) => {
-                if (value.trim() === "") {
-                  return "Output path is required";
-                }
-                if (fs.existsSync(value)) {
-                  return "The file already exists";
-                }
-                if (!fs.existsSync(value) && value.endsWith("/")) {
-                  return "No directory can be specified.";
-                }
-              }
-            });
-            if (isCancel(finalOutputPath)) {
-              cancel("Canceled.");
-              return;
-            }
-          }
-        }
-      } else {
-        finalOutputPath = "";
+      if (part === "www") {
+        return acc;
       }
-      finalMode = await select({
-        message: "Select processing mode:",
-        options: MODES.map((mode) => ({ value: mode, label: mode })),
-        initialValue: options.mode && MODES.includes(options.mode) ? options.mode : "default"
-      });
-      if (isCancel(finalMode)) {
-        cancel("Canceled.");
-        return;
+      return [part, ...acc];
+    }, []);
+    const domainString = domainParts.reverse().join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2);
+    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
+    let filename = [domainString, pathString].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+var sourcePathToOutputPath = (sourcePath) => {
+  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
+};
+function getNextAvailableFilePath(filePath) {
+  const parsedPath = path.parse(filePath);
+  const directory = parsedPath.dir;
+  const fullName = parsedPath.base;
+  const [firstPart, ...restParts] = fullName.split(".");
+  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
+  const baseName = firstPart.replace(/_\d+$/, "");
+  let counter = 1;
+  let nextFilePath = filePath;
+  while (fs.existsSync(nextFilePath)) {
+    const match = firstPart.match(/_(\d+)$/);
+    if (match) {
+      counter = Number.parseInt(match[1], 10) + 1;
+    }
+    const newName = `${baseName}_${counter}${restName}`;
+    nextFilePath = path.join(directory, newName);
+    counter++;
+  }
+  return nextFilePath;
+}
+
+// src/cli/helpers/assertContinue.ts
+import { cancel, isCancel } from "@clack/prompts";
+function assertContinue(message, cancelMessage = "Canceled.") {
+  if (isCancel(message)) {
+    cancel(cancelMessage);
+    process.exit(1);
+  }
+}
+
+// src/cli/helpers/inputOutputPath.ts
+var inputOutputPath = async (sourcePath) => {
+  const outputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: sourcePathToOutputPath(sourcePath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
       }
-    } catch (e) {
-      const error = e;
-      console.error(pc.red("An error occurred while processing the given value:"), error.message);
-      if (process.env.DEBUG) {
-        console.error(error.stack);
+      if (fs2.existsSync(value) && fs2.statSync(value).isDirectory()) {
+        return "No directory can be specified.";
       }
-      process.exit(1);
-    }
-    if (process.env.DEBUG) {
-      console.info("Processing with the following options:");
-      console.table({ finalSource, sourceIsUrl, isOutputFile, finalOutputPath, finalMode, finalLoader });
-    }
-    let content;
-    const loadSpinner = spinner();
-    try {
-      if (sourceIsUrl) {
-        loadSpinner.start("Downloading content.");
-        if (finalLoader === "playwright") {
-          content = await playwrightLoadHtml(finalSource);
-        } else if (finalLoader === "puppeteer") {
-          content = await puppeteerLoadHtml(finalSource);
-        } else {
-          const response = await fetch(finalSource);
-          content = await response.text();
-        }
-        loadSpinner.stop("Content download is complete!");
-      } else {
-        loadSpinner.start("Retrieving content.");
-        if (!fs.existsSync(finalSource)) {
-          console.error(pc.red("The file does not exist"));
-          process.exit(1);
-        }
-        const stats = fs.statSync(finalSource);
-        if (!stats.isFile()) {
-          console.error(pc.red("The source must be a file"));
-          process.exit(1);
-        }
-        content = fs.readFileSync(finalSource, "utf-8");
-        loadSpinner.stop("Content retrieval is complete!");
+      if (!fs2.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
       }
-    } catch (e) {
-      const error = e;
-      loadSpinner.stop(pc.red("An error occurred while retrieving content"));
-      console.error(error.message);
-      if (process.env.DEBUG) {
-        console.error(error.stack);
+    }
+  });
+  assertContinue(outputPath);
+  if (!fs2.existsSync(outputPath)) {
+    return outputPath;
+  }
+  const isOutputFileOverwrite = await confirm({
+    message: "The file already exists. Overwrite?",
+    initialValue: false
+  });
+  assertContinue(isOutputFileOverwrite);
+  if (isOutputFileOverwrite) {
+    return outputPath;
+  }
+  const escapedOutputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: getNextAvailableFilePath(outputPath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
       }
-      process.exit(1);
-    }
-    if (process.env.DEBUG) {
-      console.info("Content:");
-      console.info(content);
-    }
-    const convertSpinner = spinner();
-    try {
-      convertSpinner.start("Converting content to markdown.");
-      const markdown = htmlToMarkdown(content, {
-        ...sourceIsUrl || !!options.baseUrl && options.baseUrl.trim() !== "" ? { baseUrl: options.baseUrl || finalSource } : {},
-        ...finalMode === "ai" ? aiModeOptions : readabilityModeOptions
-      });
-      if (process.env.DEBUG) {
-        console.info("Markdown:");
-        console.info(markdown);
+      if (fs2.existsSync(value)) {
+        return "The file already exists";
       }
-      if (isOutputFile) {
-        const directory = path.dirname(finalOutputPath);
-        if (!fs.existsSync(directory)) {
-          fs.mkdirSync(directory, { recursive: true });
-        }
-        fs.writeFileSync(finalOutputPath, markdown);
-        convertSpinner.stop("Markdown conversion is complete!");
-        outro(`Converted successfully to markdown! Output saved to: ${path.resolve(finalOutputPath)} \u{1F389}`);
-      } else {
-        convertSpinner.stop("Markdown conversion is complete!");
-        outro("Converted successfully to markdown\u{1F389}");
-        console.log(`
-
-${markdown}`);
+      if (!fs2.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
       }
-    } catch (e) {
-      const error = e;
-      convertSpinner.stop(pc.red("An error occurred while converting and saving the markdown"));
-      console.error(error.message);
-      if (process.env.DEBUG) {
-        console.error(error.stack);
+    }
+  });
+  assertContinue(escapedOutputPath);
+  return escapedOutputPath;
+};
+
+// src/cli/helpers/inputSourcePath.ts
+import fs3 from "node:fs";
+import { text as text2 } from "@clack/prompts";
+
+// src/cli/constants.ts
+var DEFAULT_PATH = "https://example.com";
+var LOADERS = ["fetch", "playwright"];
+var MODES = ["default", "ai"];
+
+// src/cli/helpers/inputSourcePath.ts
+var inputSourcePath = async () => {
+  const result = await text2({
+    message: "Enter the URL or html path to be converted to markdown:",
+    placeholder: DEFAULT_PATH,
+    initialValue: "",
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Source is required";
+      }
+      if (!isUrl(value)) {
+        if (!fs3.existsSync(value)) {
+          return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
+        }
+        if (fs3.statSync(value).isDirectory()) {
+          return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
+        }
       }
-      process.exit(1);
     }
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/helpers/selectExtractMode.ts
+import { select as select4 } from "@clack/prompts";
+var selectExtractMode = async () => {
+  const result = await select4({
+    message: "Select processing mode:",
+    options: MODES.map((mode) => ({ value: mode, label: mode })),
+    initialValue: MODES[0]
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/helpers/selectLoader.ts
+import { select as select5 } from "@clack/prompts";
+var loadersHint = {
+  fetch: "Fetch HTML content from the given URL",
+  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
+  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
+};
+var selectLoader = async () => {
+  const result = await select5({
+    message: "Select loader:",
+    initialValue: "fetch",
+    options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
+  });
+  assertContinue(result);
+  return result;
+};
+
+// src/cli/commands/handlers/loadHtml.ts
+import fs4 from "node:fs/promises";
+import { cancel as cancel2, spinner as clackSpinner, confirm as confirm2, spinner } from "@clack/prompts";
+import pc from "picocolors";
+import { $ } from "zx";
+var loadHtml3 = async (sourcePath, loader) => {
+  const loadSpinner = clackSpinner();
+  try {
+    switch (loader.type) {
+      case "local":
+        return await loadLocalHtml(sourcePath, loadSpinner);
+      case "web":
+        switch (loader.loader) {
+          case "fetch":
+            return await loadFetchHtml(sourcePath, loadSpinner);
+          case "playwright":
+            return await loadPlaywrightHtml(sourcePath, loadSpinner);
+          default:
+            throw new Error("Invalid loader");
+        }
+      default:
+        throw new Error("Invalid loader type");
+    }
+  } catch (error) {
+    loadSpinner.stop(pc.red("Content loading failed!"));
+    console.error(error);
+    process.exit(1);
+  }
+};
+var loadLocalHtml = async (sourcePath, loadSpinner) => {
+  loadSpinner.start("Retrieving content.");
+  try {
+    await fs4.access(sourcePath);
+  } catch {
+    throw new Error("The file does not exist");
+  }
+  const stats = await fs4.stat(sourcePath);
+  if (!stats.isFile()) {
+    throw new Error("The source must be a file");
+  }
+  const content = await fs4.readFile(sourcePath, "utf-8");
+  loadSpinner.stop("Content retrieval is complete!");
+  return content;
+};
+var loadFetchHtml = async (sourcePath, loadSpinner) => {
+  loadSpinner.start("Downloading content.");
+  const { loadHtml: loadHtml4 } = await Promise.resolve().then(() => (init_fetch(), fetch_exports));
+  const content = await loadHtml4(sourcePath);
+  loadSpinner.stop("Content download is complete!");
+  return content;
+};
+var loadPlaywrightHtml = async (sourcePath, loadSpinner) => {
+  const launchSpinner = clackSpinner();
+  launchSpinner.start("Launching playwright.");
+  const playwright = await import("playwright");
+  let browser;
+  try {
+    browser = await playwright.chromium.launch();
+    launchSpinner.stop("Playwright launched!");
+  } catch (error) {
+    await handlePlaywrightLaunchError(error);
+    browser = await playwright.chromium.launch();
+  }
+  loadSpinner.start("Loading content.");
+  const { loadHtml: loadHtml4 } = await Promise.resolve().then(() => (init_playwright(), playwright_exports));
+  const content = await loadHtml4(sourcePath, browser);
+  loadSpinner.stop("Content download is complete!");
+  await browser.close();
+  return content;
+};
+var handlePlaywrightLaunchError = async (error) => {
+  if (!error.message.includes("npx playwright install")) {
+    throw error;
+  }
+  const isInstall = await confirm2({
+    message: "To continue, you will need to download a browser. Do you wish to continue?"
+  });
+  if (!isInstall) {
+    cancel2("End of process.");
+    process.exit(1);
+  }
+  const installDepsSpinner = spinner();
+  installDepsSpinner.start("Installing browser dependencies.");
+  try {
+    await $`npx playwright install`;
+    installDepsSpinner.stop("Browser dependencies installed!");
+  } catch (error2) {
+    installDepsSpinner.stop(pc.red("Browser dependencies failed to install!"));
+    throw error2;
+  }
+};
+
+// src/cli/commands/handlers/index.ts
+var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
+var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
+var webforaiCommand = async (initialPath, initialOutputPath, options) => {
+  const sourcePath = initialPath ?? await inputSourcePath();
+  if (options.debug) {
+    console.debug(`sourcePath: ${sourcePath}`);
+  }
+  const loader = isUrl(sourcePath) ? { type: "web", loader: options.loader ?? await selectLoader() } : { type: "local", loader: null };
+  if (options.debug) {
+    console.debug(`loader: ${loader}`);
+  }
+  const outputPath = initialOutputPath ?? await inputOutputPath(sourcePath);
+  if (options.debug) {
+    console.debug(`outputPath: ${outputPath}`);
+  }
+  const mode = options.mode ?? await selectExtractMode();
+  if (options.debug) {
+    console.debug(`mode: ${mode}`);
   }
-);
-intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
+  const content = await loadHtml3(sourcePath, loader);
+  if (options.debug) {
+    console.debug(`content: ${content}`);
+  }
+  const markdown = htmlToMarkdown(content, {
+    baseUrl: isUrl(sourcePath) ? sourcePath : options.baseUrl,
+    ...mode === "ai" ? aiModeOptions : readabilityModeOptions
+  });
+  if (options.debug) {
+    console.debug(`markdown: ${markdown}`);
+  }
+  const directory = path2.dirname(outputPath);
+  if (!fs5.existsSync(directory)) {
+    fs5.mkdirSync(directory, { recursive: true });
+  }
+  fs5.writeFileSync(outputPath, markdown);
+  outro("Markdown conversion is complete!");
+};
+
+// src/cli/bin.ts
+program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(package_default.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
+  process.env.DEBUG = "true";
+}).action(webforaiCommand);
+intro(pc2.inverse(`webforai CLI version ${package_default.version}`));
 program.parse();