webforai 3.0.0 → 4.1.0

package/dist/utils/parse-html.js

@@ -0,0 +1,87 @@
+// src/utils/parse-html.ts
+import { fromParse5 } from "hast-util-from-parse5";
+import { parse, parseFragment } from "parse5";
+var MAX_ELEMENT_DEPTH = 256;
+var isParent = (node) => "childNodes" in node;
+var LINE_ELEMENTS = /* @__PURE__ */ new Set([
+  "p",
+  "div",
+  "li",
+  "tr",
+  "td",
+  "th",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "br",
+  "section",
+  "article",
+  "pre",
+  "blockquote"
+]);
+var textNode = (value, parentNode) => ({ nodeName: "#text", value, parentNode });
+var coalesceText = (parent) => {
+  const merged = [];
+  for (const child of parent.childNodes) {
+    const previous = merged.at(-1);
+    if (child.nodeName === "#text" && previous?.nodeName === "#text") {
+      previous.value += child.value;
+    } else {
+      merged.push(child);
+    }
+  }
+  parent.childNodes = merged;
+};
+var flattenDeep = (root, maxDepth) => {
+  const stack = [[root, 0]];
+  while (stack.length > 0) {
+    const [parent, depth] = stack.pop();
+    const children = parent.childNodes;
+    let unwrapped = false;
+    for (let index = 0; index < children.length; index++) {
+      const child = children[index];
+      if (!isParent(child)) {
+        continue;
+      }
+      if (depth + 1 > maxDepth && "tagName" in child) {
+        const replacement = [...child.childNodes];
+        if (LINE_ELEMENTS.has(child.tagName)) {
+          replacement.unshift(textNode("\n", parent));
+          replacement.push(textNode("\n", parent));
+        }
+        for (const node of replacement) {
+          node.parentNode = parent;
+        }
+        children.splice(index, 1, ...replacement);
+        index -= 1;
+        unwrapped = true;
+        continue;
+      }
+      stack.push([child, depth + 1]);
+    }
+    if (unwrapped) {
+      coalesceText(parent);
+    }
+  }
+};
+var SCRIPT_OR_STYLE = /<(script|style)\b([^>]*)>[\s\S]*?<\/\1\s*>/gi;
+var LARGE_DOCUMENT_CHARS = 2e6;
+var stripScriptBodies = (html) => html.replace(
+  SCRIPT_OR_STYLE,
+  (match, tag, attributes) => /application\/ld\+json/i.test(attributes) ? match : `<${tag}${attributes}></${tag}>`
+);
+var parseHtml = (html, options = {}) => {
+  const source = html.length > LARGE_DOCUMENT_CHARS ? stripScriptBodies(html) : html;
+  const document = options.fragment ? parseFragment(source, { scriptingEnabled: false }) : parse(source, { scriptingEnabled: false });
+  flattenDeep(document, MAX_ELEMENT_DEPTH);
+  return fromParse5(document);
+};
+export {
+  LARGE_DOCUMENT_CHARS,
+  MAX_ELEMENT_DEPTH,
+  parseHtml,
+  stripScriptBodies
+};

package/dist/utils/pre-text.js

@@ -0,0 +1,81 @@
+// src/utils/pre-text.ts
+var LINE_BLOCK_TAGS = /* @__PURE__ */ new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "tr",
+  "ul"
+]);
+var INLINE_CONTEXT_TAGS = /* @__PURE__ */ new Set(["span", "a", "b", "i", "em", "strong", "mark", "small", "u", "s", "label"]);
+var preText = (node) => {
+  let out = "";
+  const ensureLineStart = () => {
+    if (out.length > 0 && out.charCodeAt(out.length - 1) !== 10) {
+      out += "\n";
+    }
+  };
+  const visitChildren = (parent, inline) => {
+    let cells = 0;
+    for (const child of parent.children) {
+      const isCell = child.type === "element" && (child.tagName === "td" || child.tagName === "th");
+      visit(child, isCell ? cells++ : 0, inline);
+    }
+  };
+  const visitElement = (element, cellIndex, inline) => {
+    const tag = element.tagName;
+    if (tag === "br") {
+      out += "\n";
+      return;
+    }
+    if (cellIndex > 0) {
+      out += "	";
+    }
+    const block = !inline && LINE_BLOCK_TAGS.has(tag);
+    if (block) {
+      ensureLineStart();
+    }
+    visitChildren(element, inline || INLINE_CONTEXT_TAGS.has(tag));
+    if (block) {
+      ensureLineStart();
+    }
+  };
+  const visit = (current, cellIndex, inline) => {
+    if (current.type === "text") {
+      out += current.value;
+    } else if (current.type === "element") {
+      visitElement(current, cellIndex, inline);
+    } else if (current.type === "root") {
+      visitChildren(current, inline);
+    }
+  };
+  visit(node, 0, false);
+  return out;
+};
+export {
+  preText
+};

package/package.json

@@ -1,23 +1,34 @@
 {
   "name": "webforai",
-  "version": "3.0.0",
-  "description": "A library that provides a web interface for AI",
+  "version": "4.1.0",
+  "description": "Convert web pages and local HTML to clean, LLM-ready Markdown: library, CLI (npx webforai) and hosted-API client.",
   "author": "inaridiy",
   "license": "Apache-2.0",
   "keywords": [
-    "web",
+    "html-to-markdown",
+    "markdown",
+    "llm",
     "ai",
+    "scraper",
+    "crawler",
+    "readability",
+    "web-scraping",
+    "cli",
+    "rag",
     "html",
     "html2md",
-    "markdown",
     "mdast",
     "hast"
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/inaridiy/webforai.git"
+    "url": "https://github.com/inaridiy/webforai.git",
+    "directory": "packages/webforai"
   },
   "homepage": "https://webforai.dev/",
+  "bugs": {
+    "url": "https://github.com/inaridiy/webforai/issues"
+  },
   "files": [
     "dist",
     "!dist/types/**/*.js"
@@ -107,6 +118,7 @@
     "boxen": "^8.0.1",
     "commander": "^12.1.0",
     "hast-util-from-html": "^2.0.3",
+    "hast-util-from-parse5": "^8.0.1",
     "hast-util-select": "^6.0.2",
     "hast-util-to-html": "^9.0.3",
     "hast-util-to-mdast": "^10.1.0",
@@ -116,6 +128,7 @@
     "mdast-util-gfm": "^3.0.0",
     "mdast-util-math": "^3.0.0",
     "mdast-util-to-markdown": "^2.1.0",
+    "parse5": "^7.1.2",
     "picocolors": "^1.0.1",
     "trim-trailing-lines": "^2.1.0",
     "unist-util-filter": "^5.0.1"