webforai 1.4.1 → 1.5.1

package/dist/cjs/cli/utils.js

@@ -0,0 +1,122 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var utils_exports = {};
+__export(utils_exports, {
+  changeFileExtension: () => changeFileExtension,
+  getNextAvailableFilePath: () => getNextAvailableFilePath,
+  isUrl: () => isUrl,
+  sourcePathToOutputPath: () => sourcePathToOutputPath,
+  urlToFilename: () => urlToFilename
+});
+module.exports = __toCommonJS(utils_exports);
+var import_node_fs = __toESM(require("node:fs"), 1);
+var import_node_path = __toESM(require("node:path"), 1);
+const isUrl = (maybeUrl) => {
+  try {
+    new URL(maybeUrl);
+    return true;
+  } catch {
+    return false;
+  }
+};
+function changeFileExtension(filePath, newExtension) {
+  const parsedPath = filePath.split("/");
+  const fileName = parsedPath[parsedPath.length - 1];
+  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  if (fileName.startsWith(".")) {
+    const parts = fileName.split(".");
+    if (parts.length === 2) {
+      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
+    }
+    parts[parts.length - 1] = newExtension.replace(/^\./, "");
+    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+  }
+  const lastDotIndex = fileName.lastIndexOf(".");
+  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
+  const newFileName = `${baseName}${formattedNewExtension}`;
+  parsedPath[parsedPath.length - 1] = newFileName;
+  return parsedPath.join("/");
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
+      if (index === 0) {
+        return acc;
+      }
+      if (acc.length >= 2) {
+        return acc;
+      }
+      if (part === "www") {
+        return acc;
+      }
+      return [part, ...acc];
+    }, []);
+    const domainString = domainParts.reverse().join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2);
+    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
+    let filename = [domainString, pathString].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+const sourcePathToOutputPath = (sourcePath) => {
+  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
+};
+function getNextAvailableFilePath(filePath) {
+  const parsedPath = import_node_path.default.parse(filePath);
+  const directory = parsedPath.dir;
+  const fullName = parsedPath.base;
+  const [firstPart, ...restParts] = fullName.split(".");
+  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
+  const baseName = firstPart.replace(/_\d+$/, "");
+  let counter = 1;
+  let nextFilePath = filePath;
+  while (import_node_fs.default.existsSync(nextFilePath)) {
+    const match = firstPart.match(/_(\d+)$/);
+    if (match) {
+      counter = Number.parseInt(match[1], 10) + 1;
+    }
+    const newName = `${baseName}_${counter}${restName}`;
+    nextFilePath = import_node_path.default.join(directory, newName);
+    counter++;
+  }
+  return nextFilePath;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  changeFileExtension,
+  getNextAvailableFilePath,
+  isUrl,
+  sourcePathToOutputPath,
+  urlToFilename
+});

package/dist/cjs/html-to-markdown.test.js

@@ -1,26 +1,4 @@
 "use strict";
-var __create = Object.create;
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getProtoOf = Object.getPrototypeOf;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
-  // If the importer is in node compatibility mode or this is not an ESM
-  // file that has been converted to a CommonJS file using a Babel-
-  // compatible transform (i.e. "__esModule" has not been set), then set
-  // "default" to the CommonJS "module.exports" for node compatibility.
-  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
-  mod
-));
 var import_fastest_levenshtein = require("fastest-levenshtein");
 var import_vitest = require("vitest");
 var import_html_to_markdown = require("./html-to-markdown");
@@ -129,18 +107,12 @@ Cell 1    Cell 2`;
   });
 });
 (0, import_vitest.describe)("htmlToMarkdown E2E", () => {
-  (0, import_vitest.it)("should convert npm README to Markdown ", async () => {
-    const html2 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
-    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2);
-    const original = await import("../README.md?raw");
-    const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
-    (0, import_vitest.expect)(d).lte(200);
-  });
-  (0, import_vitest.it)("should convert GitHub README to Markdown ", async () => {
+  (0, import_vitest.it)("Converting for good", async () => {
+    const html1 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
+    const markdown1 = (0, import_html_to_markdown.htmlToMarkdown)(html1, { linkAsText: true, hideImage: true });
     const html2 = await (0, import_fetch.loadHtml)("https://github.com/inaridiy/webforai");
-    const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html2);
-    const original = await import("../../../README.md?raw");
-    const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
-    (0, import_vitest.expect)(d).lte(200);
+    const markdown2 = (0, import_html_to_markdown.htmlToMarkdown)(html2, { linkAsText: true, hideImage: true });
+    const d = (0, import_fastest_levenshtein.distance)(markdown1, markdown2);
+    (0, import_vitest.expect)(d).lte(2500);
   });
 });

package/dist/cjs/index.js

@@ -18,6 +18,8 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var src_exports = {};
 __export(src_exports, {
+  DEFAULT_EXTRACT_HAST: () => import_extract_hast.DEFAULT_EXTRACT_HAST,
+  PRESET_EXTRACT_HAST: () => import_extract_hast.PRESET_EXTRACT_HAST,
   htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
   htmlToMdast: () => import_html_to_mdast.htmlToMdast,
   mdastSplitter: () => import_md_splitter.mdastSplitter,
@@ -28,8 +30,11 @@ var import_html_to_markdown = require("./html-to-markdown");
 var import_md_splitter = require("./md-splitter");
 var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
+var import_extract_hast = require("./extract-hast/index");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  DEFAULT_EXTRACT_HAST,
+  PRESET_EXTRACT_HAST,
   htmlToMarkdown,
   htmlToMdast,
   mdastSplitter,

package/dist/cli/bin.js

@@ -0,0 +1,12 @@
+// src/cli/bin.ts
+import { intro } from "@clack/prompts";
+import { program } from "commander";
+import pc from "picocolors";
+import packageInfo from "../../package.json";
+import { webforaiCommand } from "./commands/handlers/index.js";
+import { LOADERS, MODES } from "./constants.js";
+program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
+  process.env.DEBUG = "true";
+}).action(webforaiCommand);
+intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
+program.parse();

package/dist/cli/commands/handlers/index.js

@@ -0,0 +1,51 @@
+// src/cli/commands/handlers/index.ts
+import fs from "node:fs";
+import path from "node:path";
+import { outro } from "@clack/prompts";
+import { htmlToMarkdown } from "../../../html-to-markdown.js";
+import { inputOutputPath } from "../../helpers/inputOutputPath.js";
+import { inputSourcePath } from "../../helpers/inputSourcePath.js";
+import { selectExtractMode } from "../../helpers/selectExtractMode.js";
+import { selectLoader } from "../../helpers/selectLoader.js";
+import { isUrl } from "../../utils.js";
+import { loadHtml } from "./loadHtml.js";
+var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
+var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
+var webforaiCommand = async (initialPath, initialOutputPath, options) => {
+  const sourcePath = initialPath ?? await inputSourcePath();
+  if (options.debug) {
+    console.debug(`sourcePath: ${sourcePath}`);
+  }
+  const loader = isUrl(sourcePath) ? { type: "web", loader: options.loader ?? await selectLoader() } : { type: "local", loader: null };
+  if (options.debug) {
+    console.debug(`loader: ${loader}`);
+  }
+  const outputPath = initialOutputPath ?? await inputOutputPath(sourcePath);
+  if (options.debug) {
+    console.debug(`outputPath: ${outputPath}`);
+  }
+  const mode = options.mode ?? await selectExtractMode();
+  if (options.debug) {
+    console.debug(`mode: ${mode}`);
+  }
+  const content = await loadHtml(sourcePath, loader);
+  if (options.debug) {
+    console.debug(`content: ${content}`);
+  }
+  const markdown = htmlToMarkdown(content, {
+    baseUrl: isUrl(sourcePath) ? sourcePath : options.baseUrl,
+    ...mode === "ai" ? aiModeOptions : readabilityModeOptions
+  });
+  if (options.debug) {
+    console.debug(`markdown: ${markdown}`);
+  }
+  const directory = path.dirname(outputPath);
+  if (!fs.existsSync(directory)) {
+    fs.mkdirSync(directory, { recursive: true });
+  }
+  fs.writeFileSync(outputPath, markdown);
+  outro("Markdown conversion is complete!");
+};
+export {
+  webforaiCommand
+};

package/dist/cli/commands/handlers/loadHtml.js

@@ -0,0 +1,94 @@
+// src/cli/commands/handlers/loadHtml.ts
+import fs from "node:fs/promises";
+import { cancel, spinner as clackSpinner, confirm, spinner } from "@clack/prompts";
+import pc from "picocolors";
+import { $ } from "zx";
+var loadHtml = async (sourcePath, loader) => {
+  const loadSpinner = clackSpinner();
+  try {
+    switch (loader.type) {
+      case "local":
+        return await loadLocalHtml(sourcePath, loadSpinner);
+      case "web":
+        switch (loader.loader) {
+          case "fetch":
+            return await loadFetchHtml(sourcePath, loadSpinner);
+          case "playwright":
+            return await loadPlaywrightHtml(sourcePath, loadSpinner);
+          default:
+            throw new Error("Invalid loader");
+        }
+      default:
+        throw new Error("Invalid loader type");
+    }
+  } catch (error) {
+    loadSpinner.stop(pc.red("Content loading failed!"));
+    console.error(error);
+    process.exit(1);
+  }
+};
+var loadLocalHtml = async (sourcePath, loadSpinner) => {
+  loadSpinner.start("Retrieving content.");
+  try {
+    await fs.access(sourcePath);
+  } catch {
+    throw new Error("The file does not exist");
+  }
+  const stats = await fs.stat(sourcePath);
+  if (!stats.isFile()) {
+    throw new Error("The source must be a file");
+  }
+  const content = await fs.readFile(sourcePath, "utf-8");
+  loadSpinner.stop("Content retrieval is complete!");
+  return content;
+};
+var loadFetchHtml = async (sourcePath, loadSpinner) => {
+  loadSpinner.start("Downloading content.");
+  const { loadHtml: loadHtml2 } = await import("../../../loaders/fetch.js");
+  const content = await loadHtml2(sourcePath);
+  loadSpinner.stop("Content download is complete!");
+  return content;
+};
+var loadPlaywrightHtml = async (sourcePath, loadSpinner) => {
+  const launchSpinner = clackSpinner();
+  launchSpinner.start("Launching playwright.");
+  const playwright = await import("playwright");
+  let browser;
+  try {
+    browser = await playwright.chromium.launch();
+    launchSpinner.stop("Playwright launched!");
+  } catch (error) {
+    await handlePlaywrightLaunchError(error);
+    browser = await playwright.chromium.launch();
+  }
+  loadSpinner.start("Loading content.");
+  const { loadHtml: loadHtml2 } = await import("../../../loaders/playwright.js");
+  const content = await loadHtml2(sourcePath, browser);
+  loadSpinner.stop("Content download is complete!");
+  await browser.close();
+  return content;
+};
+var handlePlaywrightLaunchError = async (error) => {
+  if (!error.message.includes("npx playwright install")) {
+    throw error;
+  }
+  const isInstall = await confirm({
+    message: "To continue, you will need to download a browser. Do you wish to continue?"
+  });
+  if (!isInstall) {
+    cancel("End of process.");
+    process.exit(1);
+  }
+  const installDepsSpinner = spinner();
+  installDepsSpinner.start("Installing browser dependencies.");
+  try {
+    await $`npx playwright install`;
+    installDepsSpinner.stop("Browser dependencies installed!");
+  } catch (error2) {
+    installDepsSpinner.stop(pc.red("Browser dependencies failed to install!"));
+    throw error2;
+  }
+};
+export {
+  loadHtml
+};

package/dist/cli/constants.js

@@ -0,0 +1,9 @@
+// src/cli/constants.ts
+var DEFAULT_PATH = "https://example.com";
+var LOADERS = ["fetch", "playwright"];
+var MODES = ["default", "ai"];
+export {
+  DEFAULT_PATH,
+  LOADERS,
+  MODES
+};

package/dist/cli/helpers/assertContinue.js

@@ -0,0 +1,11 @@
+// src/cli/helpers/assertContinue.ts
+import { cancel, isCancel } from "@clack/prompts";
+function assertContinue(message, cancelMessage = "Canceled.") {
+  if (isCancel(message)) {
+    cancel(cancelMessage);
+    process.exit(1);
+  }
+}
+export {
+  assertContinue
+};

package/dist/cli/helpers/inputOutputPath.js

@@ -0,0 +1,56 @@
+// src/cli/helpers/inputOutputPath.ts
+import fs from "node:fs";
+import { confirm, text } from "@clack/prompts";
+import { getNextAvailableFilePath, sourcePathToOutputPath } from "../utils.js";
+import { assertContinue } from "./assertContinue.js";
+var inputOutputPath = async (sourcePath) => {
+  const outputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: sourcePathToOutputPath(sourcePath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
+      }
+      if (fs.existsSync(value) && fs.statSync(value).isDirectory()) {
+        return "No directory can be specified.";
+      }
+      if (!fs.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
+      }
+    }
+  });
+  assertContinue(outputPath);
+  if (!fs.existsSync(outputPath)) {
+    return outputPath;
+  }
+  const isOutputFileOverwrite = await confirm({
+    message: "The file already exists. Overwrite?",
+    initialValue: false
+  });
+  assertContinue(isOutputFileOverwrite);
+  if (isOutputFileOverwrite) {
+    return outputPath;
+  }
+  const escapedOutputPath = await text({
+    message: "Enter the output file path:",
+    placeholder: "output.md",
+    initialValue: getNextAvailableFilePath(outputPath),
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Output path is required";
+      }
+      if (fs.existsSync(value)) {
+        return "The file already exists";
+      }
+      if (!fs.existsSync(value) && value.endsWith("/")) {
+        return "No directory can be specified.";
+      }
+    }
+  });
+  assertContinue(escapedOutputPath);
+  return escapedOutputPath;
+};
+export {
+  inputOutputPath
+};

package/dist/cli/helpers/inputSourcePath.js

@@ -0,0 +1,31 @@
+// src/cli/helpers/inputSourcePath.ts
+import fs from "node:fs";
+import { text } from "@clack/prompts";
+import { DEFAULT_PATH } from "../constants.js";
+import { isUrl } from "../utils.js";
+import { assertContinue } from "./assertContinue.js";
+var inputSourcePath = async () => {
+  const result = await text({
+    message: "Enter the URL or html path to be converted to markdown:",
+    placeholder: DEFAULT_PATH,
+    initialValue: "",
+    validate: (value) => {
+      if (value.trim() === "") {
+        return "Source is required";
+      }
+      if (!isUrl(value)) {
+        if (!fs.existsSync(value)) {
+          return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
+        }
+        if (fs.statSync(value).isDirectory()) {
+          return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
+        }
+      }
+    }
+  });
+  assertContinue(result);
+  return result;
+};
+export {
+  inputSourcePath
+};

package/dist/cli/helpers/selectExtractMode.js

@@ -0,0 +1,16 @@
+// src/cli/helpers/selectExtractMode.ts
+import { select } from "@clack/prompts";
+import { MODES } from "../constants.js";
+import { assertContinue } from "./assertContinue.js";
+var selectExtractMode = async () => {
+  const result = await select({
+    message: "Select processing mode:",
+    options: MODES.map((mode) => ({ value: mode, label: mode })),
+    initialValue: MODES[0]
+  });
+  assertContinue(result);
+  return result;
+};
+export {
+  selectExtractMode
+};

package/dist/cli/helpers/selectLoader.js

@@ -0,0 +1,21 @@
+// src/cli/helpers/selectLoader.ts
+import { select } from "@clack/prompts";
+import { LOADERS } from "../constants.js";
+import { assertContinue } from "./assertContinue.js";
+var loadersHint = {
+  fetch: "Fetch HTML content from the given URL",
+  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
+  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
+};
+var selectLoader = async () => {
+  const result = await select({
+    message: "Select loader:",
+    initialValue: "fetch",
+    options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
+  });
+  assertContinue(result);
+  return result;
+};
+export {
+  selectLoader
+};

package/dist/cli/utils.js

@@ -0,0 +1,85 @@
+// src/cli/utils.ts
+import fs from "node:fs";
+import path from "node:path";
+var isUrl = (maybeUrl) => {
+  try {
+    new URL(maybeUrl);
+    return true;
+  } catch {
+    return false;
+  }
+};
+function changeFileExtension(filePath, newExtension) {
+  const parsedPath = filePath.split("/");
+  const fileName = parsedPath[parsedPath.length - 1];
+  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  if (fileName.startsWith(".")) {
+    const parts = fileName.split(".");
+    if (parts.length === 2) {
+      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
+    }
+    parts[parts.length - 1] = newExtension.replace(/^\./, "");
+    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+  }
+  const lastDotIndex = fileName.lastIndexOf(".");
+  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
+  const newFileName = `${baseName}${formattedNewExtension}`;
+  parsedPath[parsedPath.length - 1] = newFileName;
+  return parsedPath.join("/");
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
+      if (index === 0) {
+        return acc;
+      }
+      if (acc.length >= 2) {
+        return acc;
+      }
+      if (part === "www") {
+        return acc;
+      }
+      return [part, ...acc];
+    }, []);
+    const domainString = domainParts.reverse().join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2);
+    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
+    let filename = [domainString, pathString].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+var sourcePathToOutputPath = (sourcePath) => {
+  return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
+};
+function getNextAvailableFilePath(filePath) {
+  const parsedPath = path.parse(filePath);
+  const directory = parsedPath.dir;
+  const fullName = parsedPath.base;
+  const [firstPart, ...restParts] = fullName.split(".");
+  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
+  const baseName = firstPart.replace(/_\d+$/, "");
+  let counter = 1;
+  let nextFilePath = filePath;
+  while (fs.existsSync(nextFilePath)) {
+    const match = firstPart.match(/_(\d+)$/);
+    if (match) {
+      counter = Number.parseInt(match[1], 10) + 1;
+    }
+    const newName = `${baseName}_${counter}${restName}`;
+    nextFilePath = path.join(directory, newName);
+    counter++;
+  }
+  return nextFilePath;
+}
+export {
+  changeFileExtension,
+  getNextAvailableFilePath,
+  isUrl,
+  sourcePathToOutputPath,
+  urlToFilename
+};

package/dist/html-to-markdown.test.js

@@ -107,18 +107,12 @@ describe("htmlToMarkdown", () => {
   });
 });
 describe("htmlToMarkdown E2E", () => {
-  it("should convert npm README to Markdown ", async () => {
-    const html2 = await loadHtml("https://www.npmjs.com/package/webforai");
-    const markdown = htmlToMarkdown(html2);
-    const original = await import("../README.md?raw");
-    const d = distance(markdown, original.default);
-    expect(d).lte(200);
-  });
-  it("should convert GitHub README to Markdown ", async () => {
+  it("Converting for good", async () => {
+    const html1 = await loadHtml("https://www.npmjs.com/package/webforai");
+    const markdown1 = htmlToMarkdown(html1, { linkAsText: true, hideImage: true });
     const html2 = await loadHtml("https://github.com/inaridiy/webforai");
-    const markdown = htmlToMarkdown(html2);
-    const original = await import("../../../README.md?raw");
-    const d = distance(markdown, original.default);
-    expect(d).lte(200);
+    const markdown2 = htmlToMarkdown(html2, { linkAsText: true, hideImage: true });
+    const d = distance(markdown1, markdown2);
+    expect(d).lte(2500);
   });
 });

package/dist/index.js

@@ -3,7 +3,10 @@ import { htmlToMarkdown } from "./html-to-markdown.js";
 import { mdastSplitter } from "./md-splitter.js";
 import { htmlToMdast } from "./html-to-mdast.js";
 import { mdastToMarkdown } from "./mdast-to-markdown.js";
+import { PRESET_EXTRACT_HAST, DEFAULT_EXTRACT_HAST } from "./extract-hast/index.js";
 export {
+  DEFAULT_EXTRACT_HAST,
+  PRESET_EXTRACT_HAST,
   htmlToMarkdown,
   htmlToMdast,
   mdastSplitter,

package/dist/types/cli/bin.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/cli/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../../../src/cli/bin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,WAAW,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO;KACL,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,wDAAwD,CAAC;KACrE,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,eAAe,EAAE,4BAA4B,CAAC;KAC3E,QAAQ,CAAC,UAAU,EAAE,wBAAwB,CAAC;KAC9C,QAAQ,CAAC,cAAc,EAAE,kCAAkC,CAAC;KAC5D,MAAM,CAAC,eAAe,EAAE,oBAAoB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;KAChE,MAAM,CAAC,mBAAmB,EAAE,kBAAkB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;KACpE,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,CAAC;KAC5D,MAAM,CAAC,cAAc,EAAE,kBAAkB,CAAC;KAC1C,MAAM,CAAC,aAAa,EAAE,oCAAoC,CAAC;KAC3D,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;IACxB,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC;AAC5B,CAAC,CAAC;KACD,MAAM,CAAC,eAAe,CAAC,CAAC;AAE1B,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,wBAAwB,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAEjE,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/types/cli/commands/handlers/index.d.ts

@@ -0,0 +1,13 @@
+export type LoaderOptions = {
+    type: "web";
+    loader: "playwright" | "pw" | "fetch" | string;
+} | {
+    type: "local";
+    loader: null;
+};
+export declare const webforaiCommand: (initialPath: string, initialOutputPath: string, options: {
+    mode?: string;
+    loader: string;
+    debug?: boolean;
+    baseUrl?: string;
+}) => Promise<void>;

package/dist/types/cli/commands/handlers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAM3F,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EACnC,WAAmB,EACnB,iBAAyB,EACzB,OAA6E,EAC5E,EAAE;IACH,MAAM,UAAU,GAAG,WAAW,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,MAAM,GAAkB,KAAK,CAAC,UAAU,CAAC;QAC9C,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,EAAE;QACnE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,UAAU,GAAG,iBAAiB,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE;QACxC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;QACzD,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAC3D,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEvC,KAAK,CAAC,kCAAkC,CAAC,CAAC;AAC3C,CAAC,CAAC"}

package/dist/types/cli/commands/handlers/loadHtml.d.ts

@@ -0,0 +1,2 @@
+import type { LoaderOptions } from "./index";
+export declare const loadHtml: (sourcePath: string, loader: LoaderOptions) => Promise<string>;