webforai 1.4.1 → 1.5.0

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 inaridiy
+Copyright (c) 2023 - present, inaridiy and webforai contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -10,6 +10,7 @@ A library that provides a web interface for AI
 - Ultra-precise Markdown segmentation based on AST
 - Ultra-precise HTML retrieval functions using headless browsers
 - The core functionality is edge-native (runs on Cloudflare Workers!!!)
+- Command-line interface for easy use without coding
 
 ## Demo
 
@@ -18,14 +19,25 @@ There is a demo API for Html2Markdown deployed on CloudflareWorker. Please acces
 - [NPM Package page](https://webforai.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai)
 - [Wikipedia of Cloudflare (AI Mode)](https://webforai.inaridiy.workers.dev/?url=https://en.wikipedia.org/wiki/Cloudflare&mode=ai)
 
-## Quick Start
-
-Just install and execute scripts
+## Installation
 
+### As a library
+To use WebforAI as a library in your project, install it along with playwright:
 ```bash
 pnpm i webforai playwright
 ```
 
+### As a CLI tool
+To use WebforAI as a command-line tool, install it globally:
+```bash
+npm install -g webforai
+```
+
+## Quick Start
+
+### Using as a library
+Just install and execute scripts
+
 ```js
 import { promises as fs } from "fs";
 import { htmlToMarkdown, htmlToMdast } from "webforai";
@@ -41,6 +53,18 @@ await fs.writeFile("output.md", markdown);
 
 other examples are in [examples](./examples/simple/src/index.ts)
 
+### Using as a CLI tool
+After installing globally, you can use the webforai command:
+```bash
+webforai https://www.npmjs.com/package/webforai output.md
+```
+This will fetch the HTML from the specified URL, convert it to Markdown, and save it to output.md.
+
+For more CLI options, run:
+```bash
+webforai --help
+```
+
 ## Examples
 
 - [Simple Example](https://github.com/inaridiy/webforai/tree/main/examples/simple/src/index.ts)
@@ -68,3 +92,37 @@ Convert Mdast to Markdown. If `solveLinks` is specified, the relative links in t
 
 **`loadHtml(url: string, options?: LoadHtmlOptions): Promise<string>`**  
 Load HTML from the specified URL. This function uses Playwright internally.
+
+### CLI Commands
+
+**`webforai <source?> <outputPath?> [options]`**
+Converts the HTML at the specified URL or path to Markdown and saves it to an output file.
+Arguments and options can be specified in the interactive interface even if they are not specified.
+- **`source`**  
+    The URL or path to the HTML file to convert.
+- **`outputPath`**
+    The path to save the output Markdown file. 
+
+#### Options
+- **`--mode <mode>`**  
+    Specify the mode to use for conversion. Options are `default` and `ai`. Default is `default`.
+- **`--loader <loader>`**  
+    Specify the loader to use for fetching HTML. Options are `fetch`, `playwright` and `puppeteer`. Default is `fetch`.
+- **`--baseUrl <baseUrl>`**
+    Specify the base URL to use for relative links in the output Markdown.
+- **`-o --stdout`**  
+    Output the converted Markdown to the console instead of saving it to a file.
+- **`-d --debug`**  
+    Enable debug mode. This will output additional information to the console.
+
+**`webforai --help`**
+Displays help information for the CLI tool.
+
+## Contributing
+1.  Fork it!
+2.  Create your feature branch: `git checkout -b my-new-feature`
+3.  Add your changes: `git add .`
+4.  Commit your changes: `git commit -am 'Add some feature'`
+5.  Add a changelog: `pnpm changeset`
+6.  Push to the branch: `git push origin my-new-feature`
+7.  Submit a pull request :sunglasses:

package/dist/bin.js

@@ -0,0 +1,233 @@
+#!/usr/bin/env node
+
+// src/bin.ts
+import fs from "node:fs";
+import path from "node:path";
+import { cancel, confirm, intro, isCancel, outro, select, spinner, text } from "@clack/prompts";
+import { program } from "commander";
+import pc from "picocolors";
+import packageInfo from "../package.json" assert { type: "json" };
+import { htmlToMarkdown } from "./html-to-markdown.js";
+import { loadHtml as playwrightLoadHtml } from "./loaders/playwright.js";
+import { loadHtml as puppeteerLoadHtml } from "./loaders/puppeteer.js";
+import { changeFileExtension, getNextAvailableFilePath, isUrl, urlToFilename } from "./utils/bin-utils.js";
+var MODES = ["default", "ai"];
+var LOADERS = ["fetch", "playwright", "puppeteer"];
+var loadersHint = {
+  fetch: "Fetch HTML content from the given URL",
+  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
+  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
+};
+var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
+var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
+program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`, MODES[0]).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`, LOADERS[0]).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
+  process.env.DEBUG = "true";
+}).action(
+  async (source, outputPath, options) => {
+    let finalSource;
+    let sourceIsUrl;
+    let finalOutputPath;
+    let finalMode;
+    let finalLoader = "fetch";
+    const isOutputFile = !options.stdout;
+    if (process.env.DEBUG) {
+      console.info("Arguments and options:");
+      console.table({
+        source,
+        outputPath,
+        ...options
+      });
+      console.info("isOutputFile:", isOutputFile);
+    }
+    try {
+      finalSource = await text({
+        message: "Enter the URL or html path to be converted to markdown:",
+        placeholder: "https://example.com",
+        initialValue: source,
+        validate: (value) => {
+          if (value.trim() === "") {
+            return "Source is required";
+          }
+          if (!isUrl(value)) {
+            if (!fs.existsSync(value)) {
+              return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
+            }
+            if (fs.statSync(value).isDirectory()) {
+              return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
+            }
+          }
+        }
+      });
+      if (isCancel(finalSource)) {
+        cancel("Canceled.");
+        return;
+      }
+      sourceIsUrl = isUrl(finalSource);
+      if (sourceIsUrl) {
+        finalLoader = await select({
+          message: "Select loader:",
+          options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" })),
+          initialValue: options.loader && LOADERS.includes(options.loader) ? options.loader : "fetch"
+        });
+        if (isCancel(finalLoader)) {
+          cancel("Canceled.");
+          return;
+        }
+      }
+      if (isOutputFile) {
+        finalOutputPath = await text({
+          message: "Enter the output file path:",
+          placeholder: "output.md",
+          initialValue: outputPath ? outputPath : sourceIsUrl ? `${urlToFilename(finalSource)}.md` : changeFileExtension(finalSource, "md"),
+          validate: (value) => {
+            if (value.trim() === "") {
+              return "Output path is required";
+            }
+            if (fs.existsSync(value) && fs.statSync(value).isDirectory()) {
+              return "No directory can be specified.";
+            }
+            if (!fs.existsSync(value) && value.endsWith("/")) {
+              return "No directory can be specified.";
+            }
+          }
+        });
+        if (isCancel(finalOutputPath)) {
+          cancel("Canceled.");
+          return;
+        }
+        if (fs.existsSync(finalOutputPath)) {
+          const isOutputFileOverwrite = await confirm({
+            message: "The file already exists. Overwrite?",
+            initialValue: false
+          });
+          if (isCancel(isOutputFileOverwrite)) {
+            cancel("Canceled.");
+            return;
+          }
+          if (isOutputFileOverwrite === false) {
+            finalOutputPath = await text({
+              message: "Enter the output file path:",
+              placeholder: "output.md",
+              initialValue: getNextAvailableFilePath(finalOutputPath),
+              validate: (value) => {
+                if (value.trim() === "") {
+                  return "Output path is required";
+                }
+                if (fs.existsSync(value)) {
+                  return "The file already exists";
+                }
+                if (!fs.existsSync(value) && value.endsWith("/")) {
+                  return "No directory can be specified.";
+                }
+              }
+            });
+            if (isCancel(finalOutputPath)) {
+              cancel("Canceled.");
+              return;
+            }
+          }
+        }
+      } else {
+        finalOutputPath = "";
+      }
+      finalMode = await select({
+        message: "Select processing mode:",
+        options: MODES.map((mode) => ({ value: mode, label: mode })),
+        initialValue: options.mode && MODES.includes(options.mode) ? options.mode : "default"
+      });
+      if (isCancel(finalMode)) {
+        cancel("Canceled.");
+        return;
+      }
+    } catch (e) {
+      const error = e;
+      console.error(pc.red("An error occurred while processing the given value:"), error.message);
+      if (process.env.DEBUG) {
+        console.error(error.stack);
+      }
+      process.exit(1);
+    }
+    if (process.env.DEBUG) {
+      console.info("Processing with the following options:");
+      console.table({ finalSource, sourceIsUrl, isOutputFile, finalOutputPath, finalMode, finalLoader });
+    }
+    let content;
+    const loadSpinner = spinner();
+    try {
+      if (sourceIsUrl) {
+        loadSpinner.start("Downloading content.");
+        if (finalLoader === "playwright") {
+          content = await playwrightLoadHtml(finalSource);
+        } else if (finalLoader === "puppeteer") {
+          content = await puppeteerLoadHtml(finalSource);
+        } else {
+          const response = await fetch(finalSource);
+          content = await response.text();
+        }
+        loadSpinner.stop("Content download is complete!");
+      } else {
+        loadSpinner.start("Retrieving content.");
+        if (!fs.existsSync(finalSource)) {
+          console.error(pc.red("The file does not exist"));
+          process.exit(1);
+        }
+        const stats = fs.statSync(finalSource);
+        if (!stats.isFile()) {
+          console.error(pc.red("The source must be a file"));
+          process.exit(1);
+        }
+        content = fs.readFileSync(finalSource, "utf-8");
+        loadSpinner.stop("Content retrieval is complete!");
+      }
+    } catch (e) {
+      const error = e;
+      loadSpinner.stop(pc.red("An error occurred while retrieving content"));
+      console.error(error.message);
+      if (process.env.DEBUG) {
+        console.error(error.stack);
+      }
+      process.exit(1);
+    }
+    if (process.env.DEBUG) {
+      console.info("Content:");
+      console.info(content);
+    }
+    const convertSpinner = spinner();
+    try {
+      convertSpinner.start("Converting content to markdown.");
+      const markdown = htmlToMarkdown(content, {
+        ...sourceIsUrl || !!options.baseUrl && options.baseUrl.trim() !== "" ? { baseUrl: options.baseUrl || finalSource } : {},
+        ...finalMode === "ai" ? aiModeOptions : readabilityModeOptions
+      });
+      if (process.env.DEBUG) {
+        console.info("Markdown:");
+        console.info(markdown);
+      }
+      if (isOutputFile) {
+        const directory = path.dirname(finalOutputPath);
+        if (!fs.existsSync(directory)) {
+          fs.mkdirSync(directory, { recursive: true });
+        }
+        fs.writeFileSync(finalOutputPath, markdown);
+        convertSpinner.stop("Markdown conversion is complete!");
+        outro(`Converted successfully to markdown! Output saved to: ${path.resolve(finalOutputPath)} \u{1F389}`);
+      } else {
+        convertSpinner.stop("Markdown conversion is complete!");
+        outro("Converted successfully to markdown\u{1F389}");
+        console.log(`
+
+${markdown}`);
+      }
+    } catch (e) {
+      const error = e;
+      convertSpinner.stop(pc.red("An error occurred while converting and saving the markdown"));
+      console.error(error.message);
+      if (process.env.DEBUG) {
+        console.error(error.stack);
+      }
+      process.exit(1);
+    }
+  }
+);
+intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
+program.parse();

package/dist/cjs/bin.js

@@ -0,0 +1,254 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var import_node_fs = __toESM(require("node:fs"), 1);
+var import_node_path = __toESM(require("node:path"), 1);
+var import_prompts = require("@clack/prompts");
+var import_commander = require("commander");
+var import_picocolors = __toESM(require("picocolors"), 1);
+var import_package = __toESM(require("../package.json"), 1);
+var import_html_to_markdown = require("./html-to-markdown");
+var import_playwright = require("./loaders/playwright");
+var import_puppeteer = require("./loaders/puppeteer");
+var import_bin_utils = require("./utils/bin-utils");
+const MODES = ["default", "ai"];
+const LOADERS = ["fetch", "playwright", "puppeteer"];
+const loadersHint = {
+  fetch: "Fetch HTML content from the given URL",
+  playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
+  puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
+};
+const aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
+const readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
+import_commander.program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(import_package.default.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`, MODES[0]).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`, LOADERS[0]).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
+  process.env.DEBUG = "true";
+}).action(
+  async (source, outputPath, options) => {
+    let finalSource;
+    let sourceIsUrl;
+    let finalOutputPath;
+    let finalMode;
+    let finalLoader = "fetch";
+    const isOutputFile = !options.stdout;
+    if (process.env.DEBUG) {
+      console.info("Arguments and options:");
+      console.table({
+        source,
+        outputPath,
+        ...options
+      });
+      console.info("isOutputFile:", isOutputFile);
+    }
+    try {
+      finalSource = await (0, import_prompts.text)({
+        message: "Enter the URL or html path to be converted to markdown:",
+        placeholder: "https://example.com",
+        initialValue: source,
+        validate: (value) => {
+          if (value.trim() === "") {
+            return "Source is required";
+          }
+          if (!(0, import_bin_utils.isUrl)(value)) {
+            if (!import_node_fs.default.existsSync(value)) {
+              return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
+            }
+            if (import_node_fs.default.statSync(value).isDirectory()) {
+              return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
+            }
+          }
+        }
+      });
+      if ((0, import_prompts.isCancel)(finalSource)) {
+        (0, import_prompts.cancel)("Canceled.");
+        return;
+      }
+      sourceIsUrl = (0, import_bin_utils.isUrl)(finalSource);
+      if (sourceIsUrl) {
+        finalLoader = await (0, import_prompts.select)({
+          message: "Select loader:",
+          options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" })),
+          initialValue: options.loader && LOADERS.includes(options.loader) ? options.loader : "fetch"
+        });
+        if ((0, import_prompts.isCancel)(finalLoader)) {
+          (0, import_prompts.cancel)("Canceled.");
+          return;
+        }
+      }
+      if (isOutputFile) {
+        finalOutputPath = await (0, import_prompts.text)({
+          message: "Enter the output file path:",
+          placeholder: "output.md",
+          initialValue: outputPath ? outputPath : sourceIsUrl ? `${(0, import_bin_utils.urlToFilename)(finalSource)}.md` : (0, import_bin_utils.changeFileExtension)(finalSource, "md"),
+          validate: (value) => {
+            if (value.trim() === "") {
+              return "Output path is required";
+            }
+            if (import_node_fs.default.existsSync(value) && import_node_fs.default.statSync(value).isDirectory()) {
+              return "No directory can be specified.";
+            }
+            if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
+              return "No directory can be specified.";
+            }
+          }
+        });
+        if ((0, import_prompts.isCancel)(finalOutputPath)) {
+          (0, import_prompts.cancel)("Canceled.");
+          return;
+        }
+        if (import_node_fs.default.existsSync(finalOutputPath)) {
+          const isOutputFileOverwrite = await (0, import_prompts.confirm)({
+            message: "The file already exists. Overwrite?",
+            initialValue: false
+          });
+          if ((0, import_prompts.isCancel)(isOutputFileOverwrite)) {
+            (0, import_prompts.cancel)("Canceled.");
+            return;
+          }
+          if (isOutputFileOverwrite === false) {
+            finalOutputPath = await (0, import_prompts.text)({
+              message: "Enter the output file path:",
+              placeholder: "output.md",
+              initialValue: (0, import_bin_utils.getNextAvailableFilePath)(finalOutputPath),
+              validate: (value) => {
+                if (value.trim() === "") {
+                  return "Output path is required";
+                }
+                if (import_node_fs.default.existsSync(value)) {
+                  return "The file already exists";
+                }
+                if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
+                  return "No directory can be specified.";
+                }
+              }
+            });
+            if ((0, import_prompts.isCancel)(finalOutputPath)) {
+              (0, import_prompts.cancel)("Canceled.");
+              return;
+            }
+          }
+        }
+      } else {
+        finalOutputPath = "";
+      }
+      finalMode = await (0, import_prompts.select)({
+        message: "Select processing mode:",
+        options: MODES.map((mode) => ({ value: mode, label: mode })),
+        initialValue: options.mode && MODES.includes(options.mode) ? options.mode : "default"
+      });
+      if ((0, import_prompts.isCancel)(finalMode)) {
+        (0, import_prompts.cancel)("Canceled.");
+        return;
+      }
+    } catch (e) {
+      const error = e;
+      console.error(import_picocolors.default.red("An error occurred while processing the given value:"), error.message);
+      if (process.env.DEBUG) {
+        console.error(error.stack);
+      }
+      process.exit(1);
+    }
+    if (process.env.DEBUG) {
+      console.info("Processing with the following options:");
+      console.table({ finalSource, sourceIsUrl, isOutputFile, finalOutputPath, finalMode, finalLoader });
+    }
+    let content;
+    const loadSpinner = (0, import_prompts.spinner)();
+    try {
+      if (sourceIsUrl) {
+        loadSpinner.start("Downloading content.");
+        if (finalLoader === "playwright") {
+          content = await (0, import_playwright.loadHtml)(finalSource);
+        } else if (finalLoader === "puppeteer") {
+          content = await (0, import_puppeteer.loadHtml)(finalSource);
+        } else {
+          const response = await fetch(finalSource);
+          content = await response.text();
+        }
+        loadSpinner.stop("Content download is complete!");
+      } else {
+        loadSpinner.start("Retrieving content.");
+        if (!import_node_fs.default.existsSync(finalSource)) {
+          console.error(import_picocolors.default.red("The file does not exist"));
+          process.exit(1);
+        }
+        const stats = import_node_fs.default.statSync(finalSource);
+        if (!stats.isFile()) {
+          console.error(import_picocolors.default.red("The source must be a file"));
+          process.exit(1);
+        }
+        content = import_node_fs.default.readFileSync(finalSource, "utf-8");
+        loadSpinner.stop("Content retrieval is complete!");
+      }
+    } catch (e) {
+      const error = e;
+      loadSpinner.stop(import_picocolors.default.red("An error occurred while retrieving content"));
+      console.error(error.message);
+      if (process.env.DEBUG) {
+        console.error(error.stack);
+      }
+      process.exit(1);
+    }
+    if (process.env.DEBUG) {
+      console.info("Content:");
+      console.info(content);
+    }
+    const convertSpinner = (0, import_prompts.spinner)();
+    try {
+      convertSpinner.start("Converting content to markdown.");
+      const markdown = (0, import_html_to_markdown.htmlToMarkdown)(content, {
+        ...sourceIsUrl || !!options.baseUrl && options.baseUrl.trim() !== "" ? { baseUrl: options.baseUrl || finalSource } : {},
+        ...finalMode === "ai" ? aiModeOptions : readabilityModeOptions
+      });
+      if (process.env.DEBUG) {
+        console.info("Markdown:");
+        console.info(markdown);
+      }
+      if (isOutputFile) {
+        const directory = import_node_path.default.dirname(finalOutputPath);
+        if (!import_node_fs.default.existsSync(directory)) {
+          import_node_fs.default.mkdirSync(directory, { recursive: true });
+        }
+        import_node_fs.default.writeFileSync(finalOutputPath, markdown);
+        convertSpinner.stop("Markdown conversion is complete!");
+        (0, import_prompts.outro)(`Converted successfully to markdown! Output saved to: ${import_node_path.default.resolve(finalOutputPath)} \u{1F389}`);
+      } else {
+        convertSpinner.stop("Markdown conversion is complete!");
+        (0, import_prompts.outro)("Converted successfully to markdown\u{1F389}");
+        console.log(`
+
+${markdown}`);
+      }
+    } catch (e) {
+      const error = e;
+      convertSpinner.stop(import_picocolors.default.red("An error occurred while converting and saving the markdown"));
+      console.error(error.message);
+      if (process.env.DEBUG) {
+        console.error(error.stack);
+      }
+      process.exit(1);
+    }
+  }
+);
+(0, import_prompts.intro)(import_picocolors.default.inverse(`webforai CLI version ${import_package.default.version}`));
+import_commander.program.parse();

package/dist/cjs/index.js

@@ -18,6 +18,8 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var src_exports = {};
 __export(src_exports, {
+  DEFAULT_EXTRACT_HAST: () => import_extract_hast.DEFAULT_EXTRACT_HAST,
+  PRESET_EXTRACT_HAST: () => import_extract_hast.PRESET_EXTRACT_HAST,
   htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
   htmlToMdast: () => import_html_to_mdast.htmlToMdast,
   mdastSplitter: () => import_md_splitter.mdastSplitter,
@@ -28,8 +30,11 @@ var import_html_to_markdown = require("./html-to-markdown");
 var import_md_splitter = require("./md-splitter");
 var import_html_to_mdast = require("./html-to-mdast");
 var import_mdast_to_markdown = require("./mdast-to-markdown");
+var import_extract_hast = require("./extract-hast/index");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  DEFAULT_EXTRACT_HAST,
+  PRESET_EXTRACT_HAST,
   htmlToMarkdown,
   htmlToMdast,
   mdastSplitter,

package/dist/cjs/utils/bin-utils.js

@@ -0,0 +1,124 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var bin_utils_exports = {};
+__export(bin_utils_exports, {
+  changeFileExtension: () => changeFileExtension,
+  getNextAvailableFilePath: () => getNextAvailableFilePath,
+  isUrl: () => isUrl,
+  logError: () => logError,
+  urlToFilename: () => urlToFilename
+});
+module.exports = __toCommonJS(bin_utils_exports);
+var import_node_fs = __toESM(require("node:fs"), 1);
+var import_node_path = __toESM(require("node:path"), 1);
+var import_prompts = require("@clack/prompts");
+function logError(error) {
+  import_prompts.log.error("An error occurred during processing");
+  console.error(error.message);
+}
+function isUrl(str) {
+  try {
+    new URL(str);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function getNextAvailableFilePath(filePath) {
+  const parsedPath = import_node_path.default.parse(filePath);
+  const directory = parsedPath.dir;
+  const fullName = parsedPath.base;
+  const [firstPart, ...restParts] = fullName.split(".");
+  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
+  const baseName = firstPart.replace(/_\d+$/, "");
+  let counter = 1;
+  let nextFilePath = filePath;
+  while (import_node_fs.default.existsSync(nextFilePath)) {
+    const match = firstPart.match(/_(\d+)$/);
+    if (match) {
+      counter = Number.parseInt(match[1], 10) + 1;
+    }
+    const newName = `${baseName}_${counter}${restName}`;
+    nextFilePath = import_node_path.default.join(directory, newName);
+    counter++;
+  }
+  return nextFilePath;
+}
+function changeFileExtension(filePath, newExtension) {
+  const parsedPath = filePath.split("/");
+  const fileName = parsedPath[parsedPath.length - 1];
+  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  if (fileName.startsWith(".")) {
+    const parts = fileName.split(".");
+    if (parts.length === 2) {
+      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
+    }
+    parts[parts.length - 1] = newExtension.replace(/^\./, "");
+    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+  }
+  const lastDotIndex = fileName.lastIndexOf(".");
+  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
+  const newFileName = `${baseName}${formattedNewExtension}`;
+  parsedPath[parsedPath.length - 1] = newFileName;
+  return parsedPath.join("/");
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
+      if (index === 0) {
+        return acc;
+      }
+      if (acc.length >= 2) {
+        return acc;
+      }
+      if (part === "www") {
+        return acc;
+      }
+      return [part, ...acc];
+    }, []);
+    const domainString = domainParts.reverse().join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2);
+    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
+    let filename = [domainString, pathString].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  changeFileExtension,
+  getNextAvailableFilePath,
+  isUrl,
+  logError,
+  urlToFilename
+});

package/dist/index.js

@@ -3,7 +3,10 @@ import { htmlToMarkdown } from "./html-to-markdown.js";
 import { mdastSplitter } from "./md-splitter.js";
 import { htmlToMdast } from "./html-to-mdast.js";
 import { mdastToMarkdown } from "./mdast-to-markdown.js";
+import { PRESET_EXTRACT_HAST, DEFAULT_EXTRACT_HAST } from "./extract-hast/index.js";
 export {
+  DEFAULT_EXTRACT_HAST,
+  PRESET_EXTRACT_HAST,
   htmlToMarkdown,
   htmlToMdast,
   mdastSplitter,

package/dist/types/bin.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../../src/bin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,WAAW,MAAM,iBAAiB,CAAC,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,QAAQ,IAAI,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAE,QAAQ,IAAI,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAExG,MAAM,KAAK,GAAa,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAC1C,MAAM,OAAO,GAAa,CAAC,OAAO,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;AAE/D,MAAM,WAAW,GAA+B;IAC/C,KAAK,EAAE,uCAAuC;IAC9C,UAAU,EAAE,kGAAkG;IAC9G,SAAS,EAAE,gGAAgG;CAC3G,CAAC;AACF,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAE3F,OAAO;KACL,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,wDAAwD,CAAC;KACrE,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,eAAe,EAAE,4BAA4B,CAAC;KAC3E,QAAQ,CAAC,UAAU,EAAE,wBAAwB,CAAC;KAC9C,QAAQ,CAAC,cAAc,EAAE,kCAAkC,CAAC;KAC5D,MAAM,CAAC,eAAe,EAAE,oBAAoB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;KAC1E,MAAM,CAAC,mBAAmB,EAAE,kBAAkB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;KAChF,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,CAAC;KAC5D,MAAM,CAAC,cAAc,EAAE,kBAAkB,CAAC;KAC1C,MAAM,CAAC,aAAa,EAAE,oCAAoC,CAAC;KAC3D,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;IACxB,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC;AAC5B,CAAC,CAAC;KACD,MAAM,CACN,KAAK,EACJ,MAAc,EACd,UAAkB,EAClB,OAAgG,EAE/F,EAAE;IACH,IAAI,WAA4B,CAAC;IACjC,IAAI,WAA6B,CAAC;IAClC,IAAI,eAAgC,CAAC;IACrC,IAAI,SAA0B,CAAC;IAC/B,IAAI,WAAW,GAAoB,OAAO,CAAC;IAC3C,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;IAErC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACvC,OAAO,CAAC,KAAK,CAAC;YACb,MAAM;YACN,UAAU;YACV,GAAG,OAAO;SACV,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,CAAC;QACJ,qCAAqC;QACrC,WAAW,GAAG,MAAM,IAAI,CAAC;YACxB,OAAO,EAAE,yDAAyD;YAClE,WAAW,EAAE,qBAAqB;YAClC,YAAY,EAAE,MAAM;YACpB,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;gBAC3B,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;oBACzB,OAAO,oBAAoB,CAAC;gBAC7B,CAAC;gBACD,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACnB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC3B,OAAO,uIAAuI,CAAC;oBAChJ,CAAC;oBACD,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;wBACtC,OAAO,6HAA6H,CAAC;oBACtI,CAAC;gBACF,CAAC;YACF,CAAC;SACD,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,WAAW,CAAC,CAAC;YACpB,OAAO;QACR,CAAC;QACD,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;QAEjC,gBAAgB;QAChB,IAAI,WAAW,EAAE,CAAC;YACjB,WAAW,GAAG,MAAM,MAAM,CAAC;gBAC1B,OAAO,EAAE,gBAAgB;gBACzB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC7F,YAAY,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;aAC3F,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,WAAW,CAAC,CAAC;gBACpB,OAAO;YACR,CAAC;QACF,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YAClB,wBAAwB;YACxB,eAAe,GAAG,MAAM,IAAI,CAAC;gBAC5B,OAAO,EAAE,6BAA6B;gBACtC,WAAW,EAAE,WAAW;gBACxB,YAAY,EAAE,UAAU;oBACvB,CAAC,CAAC,UAAU;oBACZ,CAAC,CAAC,WAAW;wBACZ,CAAC,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,KAAK;wBACpC,CAAC,CAAC,mBAAmB,CAAC,WAAW,EAAE,IAAI,CAAC;gBAC1C,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;oBAC3B,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;wBACzB,OAAO,yBAAyB,CAAC;oBAClC,CAAC;oBACD,IAAI,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;wBAC9D,OAAO,gCAAgC,CAAC;oBACzC,CAAC;oBACD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAClD,OAAO,gCAAgC,CAAC;oBACzC,CAAC;gBACF,CAAC;aACD,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,WAAW,CAAC,CAAC;gBACpB,OAAO;YACR,CAAC;YAED,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;gBACpC,MAAM,qBAAqB,GAAG,MAAM,OAAO,CAAC;oBAC3C,OAAO,EAAE,qCAAqC;oBAC9C,YAAY,EAAE,KAAK;iBACnB,CAAC,CAAC;gBACH,IAAI,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBACrC,MAAM,CAAC,WAAW,CAAC,CAAC;oBACpB,OAAO;gBACR,CAAC;gBAED,IAAI,qBAAqB,KAAK,KAAK,EAAE,CAAC;oBACrC,eAAe,GAAG,MAAM,IAAI,CAAC;wBAC5B,OAAO,EAAE,6BAA6B;wBACtC,WAAW,EAAE,WAAW;wBACxB,YAAY,EAAE,wBAAwB,CAAC,eAAe,CAAC;wBACvD,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;4BAC3B,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gCACzB,OAAO,yBAAyB,CAAC;4BAClC,CAAC;4BACD,IAAI,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gCAC1B,OAAO,yBAAyB,CAAC;4BAClC,CAAC;4BACD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gCAClD,OAAO,gCAAgC,CAAC;4BACzC,CAAC;wBACF,CAAC;qBACD,CAAC,CAAC;oBACH,IAAI,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;wBAC/B,MAAM,CAAC,WAAW,CAAC,CAAC;wBACpB,OAAO;oBACR,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,eAAe,GAAG,EAAE,CAAC;QACtB,CAAC;QAED,cAAc;QACd,SAAS,GAAG,MAAM,MAAM,CAAC;YACxB,OAAO,EAAE,yBAAyB;YAClC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5D,YAAY,EAAE,OAAO,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;SACrF,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,WAAW,CAAC,CAAC;YACpB,OAAO;QACR,CAAC;IACF,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,CAAU,CAAC;QAEzB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,qDAAqD,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACvD,OAAO,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,eAAe,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;IACpG,CAAC;IAED,IAAI,OAAe,CAAC;IAEpB,MAAM,WAAW,GAAG,OAAO,EAAE,CAAC;IAC9B,IAAI,CAAC;QACJ,IAAI,WAAW,EAAE,CAAC;YACjB,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAE1C,IAAI,WAAW,KAAK,YAAY,EAAE,CAAC;gBAClC,OAAO,GAAG,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC;YACjD,CAAC;iBAAM,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;gBACxC,OAAO,GAAG,MAAM,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC;gBAE1C,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACP,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACzC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBACjC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC;gBACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;YAED,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;YAED,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAChD,WAAW,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;IACF,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,CAAU,CAAC;QAEzB,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC,CAAC;QAEvE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAAE,CAAC;IACjC,IAAI,CAAC;QACJ,cAAc,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAExD,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE;YACxC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;gBACtE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,WAAW,EAAE;gBAC7C,CAAC,CAAC,EAAE,CAAC;YACN,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;SAChE,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YAClB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;YAEhD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,CAAC;YACD,EAAE,CAAC,aAAa,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;YAE5C,cAAc,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAExD,KAAK,CAAC,wDAAwD,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnG,CAAC;aAAM,CAAC;YACP,cAAc,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YACxD,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC9C,2DAA2D;YAC3D,OAAO,CAAC,GAAG,CAAC,OAAO,QAAQ,EAAE,CAAC,CAAC;QAChC,CAAC;IACF,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,CAAU,CAAC;QAEzB,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,4DAA4D,CAAC,CAAC,CAAC;QAE1F,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC,CACD,CAAC;AAEH,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,wBAAwB,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAEjE,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/types/index.d.ts

@@ -2,3 +2,4 @@ export { htmlToMarkdown, type HtmlToMarkdownOptions } from "./html-to-markdown";
 export { mdastSplitter } from "./md-splitter";
 export { htmlToMdast, type HtmlToMdastOptions } from "./html-to-mdast";
 export { mdastToMarkdown } from "./mdast-to-markdown";
+export { PRESET_EXTRACT_HAST, DEFAULT_EXTRACT_HAST } from "./extract-hast/index";

package/dist/types/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EAAE,cAAc,EAA8B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EAAE,cAAc,EAA8B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/types/utils/bin-utils.d.ts

@@ -0,0 +1,5 @@
+export declare function logError(error: Error): void;
+export declare function isUrl(str: string): boolean;
+export declare function getNextAvailableFilePath(filePath: string): string;
+export declare function changeFileExtension(filePath: string, newExtension: string): string;
+export declare function urlToFilename(url: string): string;

package/dist/types/utils/bin-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin-utils.js","sourceRoot":"","sources":["../../../src/utils/bin-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAErC,MAAM,UAAU,QAAQ,CAAC,KAAY;IACpC,GAAG,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACjD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,GAAW;IAChC,IAAI,CAAC;QACJ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACb,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;AACF,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,QAAgB;IACxD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC;IACjC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEvE,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEhD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,QAAQ,CAAC;IAE5B,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,OAAO,GAAG,QAAQ,EAAE,CAAC;QACpD,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,EAAE,CAAC;IACX,CAAC;IAED,OAAO,YAAY,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAE,YAAoB;IACzE,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEnD,MAAM,qBAAqB,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC;IAE/F,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,QAAQ,GAAG,qBAAqB,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxF,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC1D,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAClF,MAAM,WAAW,GAAG,GAAG,QAAQ,GAAG,qBAAqB,EAAE,CAAC;IAE1D,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACxC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ;aACjC,KAAK,CAAC,GAAG,CAAC;aACV,OAAO,EAAE;aACT,MAAM,CAAC,CAAC,GAAa,EAAE,IAAY,EAAE,KAAa,EAAE,EAAE;YACtD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACrB,OAAO,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACpB,OAAO,GAAG,CAAC;YACZ,CAAC;YACD,oEAAoE;YACpE,OAAO,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC;QACvB,CAAC,EAAE,EAAE,CAAC,CAAC;QACR,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEvF,IAAI,QAAQ,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpE,QAAQ,GAAG,QAAQ;aACjB,WAAW,EAAE;YACd,yEAAyE;aACxE,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;aACrC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAE1B,OAAO,QAAQ,IAAI,QAAQ,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;AACF,CAAC"}

package/dist/utils/bin-utils.js

@@ -0,0 +1,87 @@
+// src/utils/bin-utils.ts
+import fs from "node:fs";
+import path from "node:path";
+import { log } from "@clack/prompts";
+function logError(error) {
+  log.error("An error occurred during processing");
+  console.error(error.message);
+}
+function isUrl(str) {
+  try {
+    new URL(str);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function getNextAvailableFilePath(filePath) {
+  const parsedPath = path.parse(filePath);
+  const directory = parsedPath.dir;
+  const fullName = parsedPath.base;
+  const [firstPart, ...restParts] = fullName.split(".");
+  const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
+  const baseName = firstPart.replace(/_\d+$/, "");
+  let counter = 1;
+  let nextFilePath = filePath;
+  while (fs.existsSync(nextFilePath)) {
+    const match = firstPart.match(/_(\d+)$/);
+    if (match) {
+      counter = Number.parseInt(match[1], 10) + 1;
+    }
+    const newName = `${baseName}_${counter}${restName}`;
+    nextFilePath = path.join(directory, newName);
+    counter++;
+  }
+  return nextFilePath;
+}
+function changeFileExtension(filePath, newExtension) {
+  const parsedPath = filePath.split("/");
+  const fileName = parsedPath[parsedPath.length - 1];
+  const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
+  if (fileName.startsWith(".")) {
+    const parts = fileName.split(".");
+    if (parts.length === 2) {
+      return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
+    }
+    parts[parts.length - 1] = newExtension.replace(/^\./, "");
+    return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
+  }
+  const lastDotIndex = fileName.lastIndexOf(".");
+  const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
+  const newFileName = `${baseName}${formattedNewExtension}`;
+  parsedPath[parsedPath.length - 1] = newFileName;
+  return parsedPath.join("/");
+}
+function urlToFilename(url) {
+  try {
+    const urlObj = new URL(url);
+    const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
+      if (index === 0) {
+        return acc;
+      }
+      if (acc.length >= 2) {
+        return acc;
+      }
+      if (part === "www") {
+        return acc;
+      }
+      return [part, ...acc];
+    }, []);
+    const domainString = domainParts.reverse().join("-");
+    const pathParts = urlObj.pathname.split("/").filter(Boolean);
+    const relevantPathParts = pathParts.slice(-2);
+    const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
+    let filename = [domainString, pathString].filter(Boolean).join("-");
+    filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
+    return filename || "output";
+  } catch {
+    return "output";
+  }
+}
+export {
+  changeFileExtension,
+  getNextAvailableFilePath,
+  isUrl,
+  logError,
+  urlToFilename
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webforai",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "A library that provides a web interface for AI",
   "author": "inaridiy",
   "license": "MIT",
@@ -26,6 +26,7 @@
   "type": "module",
   "module": "dist/index.js",
   "types": "dist/types/index.d.ts",
+  "bin": "dist/bin.js",
   "exports": {
     ".": {
       "types": "./dist/types/index.d.ts",
@@ -94,6 +95,8 @@
     }
   },
   "dependencies": {
+    "@clack/prompts": "^0.7.0",
+    "commander": "^12.1.0",
     "fastest-levenshtein": "^1.0.16",
     "hast-util-from-html": "^2.0.1",
     "hast-util-select": "^6.0.1",
@@ -105,13 +108,10 @@
     "mdast-util-gfm": "^3.0.0",
     "mdast-util-math": "^3.0.0",
     "mdast-util-to-markdown": "^2.1.0",
+    "picocolors": "^1.0.1",
     "puppeteer": "^22.10.0",
     "trim-trailing-lines": "^2.1.0",
-    "unist-util-filter": "^5.0.1",
-    "unist-util-parents": "^3.0.0",
-    "unist-util-remove": "^4.0.0",
-    "unist-util-visit": "^5.0.0",
-    "unist-util-visit-parents": "^6.0.1"
+    "unist-util-filter": "^5.0.1"
   },
   "devDependencies": {
     "@cloudflare/puppeteer": "^0.0.6",
@@ -132,7 +132,7 @@
   "scripts": {
     "copy:package.cjs.json": "pnpm ncp ./package.cjs.json ./dist/cjs/package.json && pnpm ncp ./package.cjs.json ./dist/types/package.json ",
     "clean": "rimraf dist",
-    "build": "pnpm clean && tsx ./build.ts && pnpm copy:package.cjs.json",
+    "build": "pnpm clean && tsx build.ts && pnpm copy:package.cjs.json",
     "test": "echo \"Error: no test specified\"",
     "prerelease": "pnpm build",
     "release": "np"