webforai 0.0.17 → 0.0.19

package/dist/loaders/playwright.js

@@ -1,22 +1,24 @@
-import * as playwright from "playwright";
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-export const loadHtml = async (url, context) => {
-    let _browser = null;
-    let _context;
-    if (!context) {
-        _browser = await playwright.chromium.launch();
-        _context = await _browser.newContext();
-    }
-    else {
-        _context = context;
-    }
-    const page = await _context.newPage();
-    await page.goto(url);
-    await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
-    const html = await page.content();
-    await page.close();
-    if (_browser)
-        await _browser.close();
-    return html;
+// src/loaders/playwright.ts
+import * as playwright from "playwright.js";
+var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+var loadHtml = async (url, context) => {
+  let _browser = null;
+  let _context;
+  if (!context) {
+    _browser = await playwright.chromium.launch();
+    _context = await _browser.newContext();
+  } else {
+    _context = context;
+  }
+  const page = await _context.newPage();
+  await page.goto(url);
+  await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
+  const html = await page.content();
+  await page.close();
+  if (_browser)
+    await _browser.close();
+  return html;
+};
+export {
+  loadHtml
 };
-//# sourceMappingURL=playwright.js.map

package/dist/md-splitter.js

@@ -1,30 +1,31 @@
-import { mdastToMarkdown } from "./mdast-to-markdown";
-import { chunk } from "./utils/common";
-import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
-const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
-const getSplitterGenerator = function* () {
-    for (const splitter of PRIORITY_SPLITTERS)
-        yield splitter;
+// src/md-splitter.ts
+import { mdastToMarkdown } from "./mdast-to-markdown.js";
+import { chunk } from "./utils/common.js";
+import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils.js";
+var PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
+var getSplitterGenerator = function* () {
+  for (const splitter of PRIORITY_SPLITTERS)
+    yield splitter;
 };
-const _mdastSplitter = async (contents, checker, splitterGenerator) => {
-    const markdown = mdastToMarkdown(warpRoot(contents));
-    if ((await checker(markdown)) || 3 > contents.length)
-        return [contents];
-    const splitter = splitterGenerator.next().value;
-    const chunked = splitter
-        ? contents.reduce((acc, content) => {
-            if (internalType(content) === splitter || acc.length === 0) {
-                acc.push([content]);
-                return acc;
-            }
-            acc[acc.length - 1].push(content);
-            return acc;
-        }, [])
-        : chunk(contents, 3);
-    const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
-    return Promise.all(splitting).then((chunks) => chunks.flat());
+var _mdastSplitter = async (contents, checker, splitterGenerator) => {
+  const markdown = mdastToMarkdown(warpRoot(contents));
+  if (await checker(markdown) || 3 > contents.length)
+    return [contents];
+  const splitter = splitterGenerator.next().value;
+  const chunked = splitter ? contents.reduce((acc, content) => {
+    if (internalType(content) === splitter || acc.length === 0) {
+      acc.push([content]);
+      return acc;
+    }
+    acc[acc.length - 1].push(content);
+    return acc;
+  }, []) : chunk(contents, 3);
+  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
+  return Promise.all(splitting).then((chunks) => chunks.flat());
 };
-export const mdastSplitter = async (mdast, checker, options) => {
-    return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
+var mdastSplitter = async (mdast, checker, options) => {
+  return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
+};
+export {
+  mdastSplitter
 };
-//# sourceMappingURL=md-splitter.js.map

package/dist/mdast-handlers/custom-div-handler.js

@@ -1,42 +1,42 @@
+// src/mdast-handlers/custom-div-handler.ts
 import { defaultHandlers } from "hast-util-to-mdast";
 import { toText } from "hast-util-to-text";
 import { trimTrailingLines } from "trim-trailing-lines";
-export const customDivHandler = (state, node) => {
-    const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
-    let highlightLanguage;
-    for (const className of classNames) {
-        if (String(className) === "highlight-text-html-basic") {
-            highlightLanguage = "html";
-            break;
-        }
-        else if (String(className).includes("highlight-source-")) {
-            highlightLanguage = String(className).slice("highlight-source-".length);
-            break;
-        }
-        else if (String(className).includes("language-")) {
-            highlightLanguage = String(className).slice("language-".length);
-            break;
-        }
-        else if (String(className).includes("code-block_wrapper")) {
-            const text = toText(node);
-            if (!text.includes("."))
-                break;
-            const lang = toText(node).split("\n")[0].split(".").slice(-1)[0];
-            highlightLanguage = lang;
-            break;
-        }
+var customDivHandler = (state, node) => {
+  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
+  let highlightLanguage;
+  for (const className of classNames) {
+    if (String(className) === "highlight-text-html-basic") {
+      highlightLanguage = "html";
+      break;
+    } else if (String(className).includes("highlight-source-")) {
+      highlightLanguage = String(className).slice("highlight-source-".length);
+      break;
+    } else if (String(className).includes("language-")) {
+      highlightLanguage = String(className).slice("language-".length);
+      break;
+    } else if (String(className).includes("code-block_wrapper")) {
+      const text = toText(node);
+      if (!text.includes("."))
+        break;
+      const lang = toText(node).split("\n")[0].split(".").slice(-1)[0];
+      highlightLanguage = lang;
+      break;
     }
-    if (highlightLanguage !== undefined) {
-        const code = node.children.find((child) => child.type === "element" && child.tagName === "pre") || node;
-        const result = {
-            type: "code",
-            lang: highlightLanguage,
-            meta: null,
-            value: trimTrailingLines(toText(code)).trim(),
-        };
-        state.patch(node, result);
-        return result;
-    }
-    return defaultHandlers.div(state, node);
+  }
+  if (highlightLanguage !== void 0) {
+    const code = node.children.find((child) => child.type === "element" && child.tagName === "pre") || node;
+    const result = {
+      type: "code",
+      lang: highlightLanguage,
+      meta: null,
+      value: trimTrailingLines(toText(code)).trim()
+    };
+    state.patch(node, result);
+    return result;
+  }
+  return defaultHandlers.div(state, node);
+};
+export {
+  customDivHandler
 };
-//# sourceMappingURL=custom-div-handler.js.map

package/dist/mdast-handlers/math-handler.js

@@ -1,10 +1,13 @@
+// src/mdast-handlers/math-handler.ts
 import { toHtml } from "hast-util-to-html";
 import { MathMLToLaTeX } from "mathml-to-latex";
-export const mathHandler = (state, node) => {
-    const mathMl = toHtml(node);
-    const latex = MathMLToLaTeX.convert(mathMl);
-    const result = { type: "inlineMath", value: latex };
-    state.patch(node, result);
-    return result;
+var mathHandler = (state, node) => {
+  const mathMl = toHtml(node);
+  const latex = MathMLToLaTeX.convert(mathMl);
+  const result = { type: "inlineMath", value: latex };
+  state.patch(node, result);
+  return result;
+};
+export {
+  mathHandler
 };
-//# sourceMappingURL=math-handler.js.map

package/dist/mdast-to-markdown.js

@@ -1,14 +1,17 @@
+// src/mdast-to-markdown.ts
 import { gfmToMarkdown } from "mdast-util-gfm";
 import { mathToMarkdown } from "mdast-util-math";
 import { toMarkdown } from "mdast-util-to-markdown";
-import { linkReplacer } from "./link-replacer";
-import { warpRoot } from "./utils/mdast-utils";
-export const mdastToMarkdown = (mdast, options) => {
-    let markdown = toMarkdown(warpRoot(mdast), {
-        extensions: [gfmToMarkdown(), mathToMarkdown()],
-    });
-    if (options?.solveLinks)
-        markdown = linkReplacer(markdown, options.solveLinks);
-    return markdown;
+import { linkReplacer } from "./link-replacer.js";
+import { warpRoot } from "./utils/mdast-utils.js";
+var mdastToMarkdown = (mdast, options) => {
+  let markdown = toMarkdown(warpRoot(mdast), {
+    extensions: [gfmToMarkdown(), mathToMarkdown()]
+  });
+  if (options?.solveLinks)
+    markdown = linkReplacer(markdown, options.solveLinks);
+  return markdown;
+};
+export {
+  mdastToMarkdown
 };
-//# sourceMappingURL=mdast-to-markdown.js.map

package/dist/types/extract-hast.d.ts

@@ -0,0 +1,2 @@
+import type { Root as Hast } from "hast";
+export declare const extractHast: (hast: Hast) => import("hast").Element | Hast;

package/dist/types/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

package/dist/utils/common.js

@@ -1,10 +1,13 @@
-export const chunk = (array, size) => {
-    return array.reduce((acc, _, index) => {
-        if (index % size === 0) {
-            acc.push(array.slice(index, index + size));
-            return acc;
-        }
-        return acc;
-    }, []);
+// src/utils/common.ts
+var chunk = (array, size) => {
+  return array.reduce((acc, _, index) => {
+    if (index % size === 0) {
+      acc.push(array.slice(index, index + size));
+      return acc;
+    }
+    return acc;
+  }, []);
+};
+export {
+  chunk
 };
-//# sourceMappingURL=common.js.map

package/dist/utils/mdast-utils.js

@@ -1,16 +1,21 @@
-export const unwarpRoot = (mdast) => {
-    if (mdast.type === "root")
-        return mdast.children;
-    return [mdast];
+// src/utils/mdast-utils.ts
+var unwarpRoot = (mdast) => {
+  if (mdast.type === "root")
+    return mdast.children;
+  return [mdast];
 };
-export const warpRoot = (mdast) => {
-    if (Array.isArray(mdast))
-        return { type: "root", children: mdast };
-    return mdast;
+var warpRoot = (mdast) => {
+  if (Array.isArray(mdast))
+    return { type: "root", children: mdast };
+  return mdast;
 };
-export const internalType = (content) => {
-    if (content.type === "heading")
-        return `h${content.depth}`;
-    return content.type;
+var internalType = (content) => {
+  if (content.type === "heading")
+    return `h${content.depth}`;
+  return content.type;
+};
+export {
+  internalType,
+  unwarpRoot,
+  warpRoot
 };
-//# sourceMappingURL=mdast-utils.js.map

package/package.json

@@ -1,10 +1,9 @@
 {
   "name": "webforai",
-  "version": "0.0.17",
+  "version": "0.0.19",
   "description": "A library that provides a web interface for AI",
   "author": "inaridiy",
   "license": "MIT",
-  "type": "module",
   "keywords": [
     "web",
     "ai",
@@ -19,24 +18,48 @@
     "url": "https://github.com/inaridiy/webforai.git"
   },
   "homepage": "https://github.com/inaridiy/webforai.git",
+  "scripts": {
+    "copy:package.cjs.json": "pnpm ncp ./package.cjs.json ./dist/cjs/package.json && pnpm ncp ./package.cjs.json ./dist/types/package.json ",
+    "clean": "rimraf dist",
+    "build": "pnpm clean && tsx ./build.ts && pnpm copy:package.cjs.json",
+    "test": "echo \"Error: no test specified\"",
+    "format": "biome format . --write",
+    "lint": "biome check .",
+    "prerelease": "pnpm build",
+    "release": "np"
+  },
   "files": [
-    "dist",
-    "!dist/**/*.tsbuildinfo",
-    "src/**/*.ts",
-    "!src/**/*.test.ts",
-    "!src/**/*.test-d.ts",
-    "!src/**/*.bench.ts"
+    "dist"
   ],
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "dist/cjs/index.js",
+  "type": "module",
+  "module": "dist/index.js",
+  "types": "dist/types/index.d.ts",
   "exports": {
     ".": {
-      "types": "./dist/index.d.ts",
-      "default": "./dist/index.js"
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/cjs/index.js"
+    },
+    "./types": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/cjs/index.js"
     },
     "./loaders/playwright": {
-      "types": "./dist/loaders/playwright.d.ts",
-      "default": "./dist/loaders/playwright.js"
+      "types": "./dist/types/loaders/playwright.d.ts",
+      "import": "./dist/loaders/playwright.js",
+      "require": "./dist/cjs/loaders/playwright.js"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      "types": [
+        "./dist/types/index.d.ts"
+      ],
+      "loaders/playwright": [
+        "./dist/types/loaders/playwright.d.ts"
+      ]
     }
   },
   "peerDependencies": {
@@ -66,14 +89,14 @@
     "@tsconfig/recommended": "^1.0.3",
     "@types/hast": "^3.0.2",
     "@types/mdast": "^4.0.2",
+    "@types/node": "^20.11.0",
+    "arg": "^5.0.2",
+    "esbuild": "^0.19.11",
+    "glob": "^10.3.10",
+    "ncp": "^2.0.0",
+    "np": "^9.2.0",
     "rimraf": "^5.0.5",
+    "tsx": "^4.7.0",
     "typescript": "^5.2.2"
-  },
-  "scripts": {
-    "clean": "rimraf dist",
-    "build": "pnpm run clean && tsc",
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "format": "biome format . --write",
-    "lint": "biome check ."
   }
-}
+}

package/dist/extract-hast.d.ts

@@ -1,2 +0,0 @@
-import type { Root as Hast } from "hast";
-export declare const extractHast: (hast: Hast) => Hast | import("hast").Element;

package/dist/extract-hast.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"extract-hast.js","sourceRoot":"","sources":["../src/extract-hast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAU,EAAE,EAAE;IACzC,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC;IAExE,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE;QACvC,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE;YAChC,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;SACzD;KACD;IAED,MAAM,SAAS,GAAG,CAAC,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;IACtE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;QACjC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;KAC5B;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}

package/dist/html-to-markdown.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"html-to-markdown.js","sourceRoot":"","sources":["../src/html-to-markdown.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAOtD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAE,OAA+B,EAAU,EAAE;IACvF,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}

package/dist/html-to-mdast.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"html-to-mdast.js","sourceRoot":"","sources":["../src/html-to-mdast.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAM5D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,OAA4B,EAAS,EAAE;IAChF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,aAAa,GAAG,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE;QACpC,QAAQ,EAAE,EAAE,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE;KACpE,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC"}

package/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}

package/dist/link-replacer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"link-replacer.js","sourceRoot":"","sources":["../src/link-replacer.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAgB,EAAE,IAAY,EAAE,EAAE;IAC9D,MAAM,KAAK,GAAG,yBAAyB,CAAC;IACxC,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC5D,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QACxE,OAAO,GAAG,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;IACtD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC"}

package/dist/loaders/playwright.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,YAAY,CAAC;AAEzC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAmC,EAAE,EAAE;IAClF,IAAI,QAAQ,GAA8B,IAAI,CAAC;IAC/C,IAAI,QAAmC,CAAC;IACxC,IAAI,CAAC,OAAO,EAAE;QACb,QAAQ,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC9C,QAAQ,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;KACvC;SAAM;QACN,QAAQ,GAAG,OAAO,CAAC;KACnB;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrB,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACnB,IAAI,QAAQ;QAAE,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACrC,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}

package/dist/md-splitter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"md-splitter.js","sourceRoot":"","sources":["../src/md-splitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAEzE,MAAM,kBAAkB,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AAEzF,MAAM,oBAAoB,GAAG,QAAQ,CAAC;IACrC,KAAK,MAAM,QAAQ,IAAI,kBAAkB;QAAE,MAAM,QAAQ,CAAC;AAC3D,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,EAC3B,QAAuB,EACvB,OAA+C,EAC/C,iBAAoC,EACT,EAAE;IAC7B,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM;QAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;IAEhD,MAAM,OAAO,GAAG,QAAQ;QACvB,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;YAClD,IAAI,YAAY,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;gBAC3D,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpB,OAAO,GAAG,CAAC;aACX;YACD,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClC,OAAO,GAAG,CAAC;QACX,CAAC,EAAE,EAAE,CAAC;QACR,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEtB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAE5F,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EACjC,KAAY,EACZ,OAA+C,EAC/C,OAAkC,EACP,EAAE;IAC7B,OAAO,cAAc,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;AAC3E,CAAC,CAAC"}

package/dist/mdast-handlers/custom-div-handler.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"custom-div-handler.js","sourceRoot":"","sources":["../../src/mdast-handlers/custom-div-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,MAAM,CAAC,MAAM,gBAAgB,GAAW,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACvD,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAE7F,IAAI,iBAAqC,CAAC;IAC1C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;QACnC,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,2BAA2B,EAAE;YACtD,iBAAiB,GAAG,MAAM,CAAC;YAC3B,MAAM;SACN;aAAM,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE;YAC3D,iBAAiB,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACxE,MAAM;SACN;aAAM,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;YACnD,iBAAiB,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM;SACN;aAAM,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE;YAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,MAAM;YAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACjE,iBAAiB,GAAG,IAAI,CAAC;YACzB,MAAM;SACN;KACD;IAED,IAAI,iBAAiB,KAAK,SAAS,EAAE;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC;QAExG,MAAM,MAAM,GAAS;YACpB,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,iBAAiB;YACvB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;SAC7C,CAAC;QACF,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1B,OAAO,MAAM,CAAC;KACd;IAED,OAAO,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AACzC,CAAC,CAAC"}

package/dist/mdast-handlers/math-handler.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"math-handler.js","sourceRoot":"","sources":["../../src/mdast-handlers/math-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,MAAM,CAAC,MAAM,WAAW,GAAW,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAwB,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACzE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}

package/dist/mdast-to-markdown.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"mdast-to-markdown.js","sourceRoot":"","sources":["../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAA4B,EAAE,OAAiC,EAAU,EAAE;IAC1G,IAAI,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;QAC1C,UAAU,EAAE,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;KAC/C,CAAC,CAAC;IAEH,IAAI,OAAO,EAAE,UAAU;QAAE,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAE/E,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}

package/dist/utils/common.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/utils/common.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,KAAK,GAAG,CAAI,KAAU,EAAE,IAAY,EAAS,EAAE;IAC3D,OAAO,KAAK,CAAC,MAAM,CAAQ,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE;QAC5C,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC,EAAE;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC;YAC3C,OAAO,GAAG,CAAC;SACX;QACD,OAAO,GAAG,CAAC;IACZ,CAAC,EAAE,EAAE,CAAC,CAAC;AACR,CAAC,CAAC"}

package/dist/utils/mdast-utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"mdast-utils.js","sourceRoot":"","sources":["../../src/utils/mdast-utils.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAY,EAAiB,EAAE;IACzD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,QAAQ,CAAC;IACjD,OAAO,CAAC,KAAK,CAAC,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,KAA4B,EAAS,EAAE;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACnE,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAAoB,EAAU,EAAE;IAC5D,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;IAC3D,OAAO,OAAO,CAAC,IAAI,CAAC;AACrB,CAAC,CAAC"}

package/src/extract-hast.ts

@@ -1,23 +0,0 @@
-import type { Root as Hast } from "hast";
-import { select, selectAll } from "hast-util-select";
-
-import { remove } from "unist-util-remove";
-
-export const extractHast = (hast: Hast) => {
-	const removeSelectors = ["header", "footer", "nav", "speechify-ignore"];
-
-	for (const selector of removeSelectors) {
-		const matchedNodes = selectAll(selector, hast);
-		for (const node of matchedNodes) {
-			remove(hast, { type: node.type, tagName: node.tagName });
-		}
-	}
-
-	const selectors = ["article", "div#article", "main", "div.container"];
-	for (const selector of selectors) {
-		const element = select(selector, hast);
-		if (element) return element;
-	}
-
-	return hast;
-};

package/src/html-to-markdown.ts

@@ -1,14 +0,0 @@
-import type { Root as Hast } from "hast";
-import { htmlToMdast } from "./html-to-mdast";
-import { mdastToMarkdown } from "./mdast-to-markdown";
-
-export interface HtmlToMarkdownOptions {
-	extractHast?: false | ((hast: Hast) => Hast);
-	solveLinks?: string;
-}
-
-export const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {
-	const mdast = htmlToMdast(html, options);
-	const markdown = mdastToMarkdown(mdast, options);
-	return markdown;
-};

package/src/html-to-mdast.ts

@@ -1,23 +0,0 @@
-import type { Root as Hast } from "hast";
-import type { Nodes as Mdast } from "mdast";
-
-import { fromHtml } from "hast-util-from-html";
-import { toMdast } from "hast-util-to-mdast";
-
-import { extractHast } from "./extract-hast";
-import { customDivHandler } from "./mdast-handlers/custom-div-handler";
-import { mathHandler } from "./mdast-handlers/math-handler";
-
-export type HtmlToMdastOptions = {
-	extractHast?: false | ((hast: Hast) => Hast);
-};
-
-export const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {
-	const hast = fromHtml(html, { fragment: true });
-	const extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);
-	const mdast = toMdast(extractedHast, {
-		handlers: { div: customDivHandler, math: mathHandler, br: () => {} },
-	});
-
-	return mdast;
-};

package/src/index.ts

@@ -1,4 +0,0 @@
-export * from "./html-to-markdown";
-export * from "./md-splitter";
-export * from "./html-to-mdast";
-export * from "./mdast-to-markdown";

package/src/link-replacer.ts

@@ -1,7 +0,0 @@
-export const linkReplacer = (markdown: string, link: string) => {
-	const regex = /(!?\[.*?\]\()(.+?)(\))/g;
-	return markdown.replace(regex, (match, pre, thisUrl, post) => {
-		if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) return match;
-		return `${pre}${new URL(thisUrl, link).href}${post}`;
-	});
-};

package/src/loaders/playwright.ts

@@ -1,22 +0,0 @@
-import * as playwright from "playwright";
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-export const loadHtml = async (url: string, context?: playwright.BrowserContext) => {
-	let _browser: playwright.Browser | null = null;
-	let _context: playwright.BrowserContext;
-	if (!context) {
-		_browser = await playwright.chromium.launch();
-		_context = await _browser.newContext();
-	} else {
-		_context = context;
-	}
-
-	const page = await _context.newPage();
-	await page.goto(url);
-	await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
-	const html = await page.content();
-	await page.close();
-	if (_browser) await _browser.close();
-	return html;
-};

package/src/md-splitter.ts

@@ -1,43 +0,0 @@
-import type { Nodes as Mdast, RootContent } from "mdast";
-import { mdastToMarkdown } from "./mdast-to-markdown";
-import { chunk } from "./utils/common";
-import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
-
-const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
-type SplitterGenerator = Generator<string, void, unknown>;
-const getSplitterGenerator = function* () {
-	for (const splitter of PRIORITY_SPLITTERS) yield splitter;
-};
-
-const _mdastSplitter = async (
-	contents: RootContent[],
-	checker: (markdown: string) => Promise<boolean>,
-	splitterGenerator: SplitterGenerator,
-): Promise<RootContent[][]> => {
-	const markdown = mdastToMarkdown(warpRoot(contents));
-	if ((await checker(markdown)) || 3 > contents.length) return [contents];
-	const splitter = splitterGenerator.next().value;
-
-	const chunked = splitter
-		? contents.reduce<RootContent[][]>((acc, content) => {
-				if (internalType(content) === splitter || acc.length === 0) {
-					acc.push([content]);
-					return acc;
-				}
-				acc[acc.length - 1].push(content);
-				return acc;
-		  }, [])
-		: chunk(contents, 3);
-
-	const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
-
-	return Promise.all(splitting).then((chunks) => chunks.flat());
-};
-
-export const mdastSplitter = async (
-	mdast: Mdast,
-	checker: (markdown: string) => Promise<boolean>,
-	options?: { signal?: AbortSignal }, //TODO
-): Promise<RootContent[][]> => {
-	return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
-};