webforai 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 inaridiy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/extract-hast.d.ts

@@ -0,0 +1,3 @@
+import type { Root as Hast } from "hast";
+export declare const extractHast: (hast: Hast) => Hast | import("hast").Element;
+//# sourceMappingURL=extract-hast.d.ts.map

package/dist/extract-hast.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-hast.d.ts","sourceRoot":"","sources":["../src/extract-hast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAKzC,eAAO,MAAM,WAAW,SAAU,IAAI,kCAiBrC,CAAC"}

package/dist/extract-hast.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractHast = void 0;
+const hast_util_select_1 = require("hast-util-select");
+const unist_util_remove_1 = require("unist-util-remove");
+const extractHast = (hast) => {
+    const removeSelectors = ["header", "footer", "nav"];
+    for (const selector of removeSelectors) {
+        const matchedNodes = (0, hast_util_select_1.selectAll)(selector, hast);
+        for (const node of matchedNodes) {
+            (0, unist_util_remove_1.remove)(hast, { type: node.type, tagName: node.tagName });
+        }
+    }
+    const selectors = ["article", "div#article", "main", "div.container"];
+    for (const selector of selectors) {
+        const element = (0, hast_util_select_1.select)(selector, hast);
+        if (element)
+            return element;
+    }
+    return hast;
+};
+exports.extractHast = extractHast;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZXh0cmFjdC1oYXN0LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2V4dHJhY3QtaGFzdC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFDQSx1REFBcUQ7QUFFckQseURBQTJDO0FBRXBDLE1BQU0sV0FBVyxHQUFHLENBQUMsSUFBVSxFQUFFLEVBQUU7SUFDekMsTUFBTSxlQUFlLEdBQUcsQ0FBQyxRQUFRLEVBQUUsUUFBUSxFQUFFLEtBQUssQ0FBQyxDQUFDO0lBRXBELEtBQUssTUFBTSxRQUFRLElBQUksZUFBZSxFQUFFO1FBQ3ZDLE1BQU0sWUFBWSxHQUFHLElBQUEsNEJBQVMsRUFBQyxRQUFRLEVBQUUsSUFBSSxDQUFDLENBQUM7UUFDL0MsS0FBSyxNQUFNLElBQUksSUFBSSxZQUFZLEVBQUU7WUFDaEMsSUFBQSwwQkFBTSxFQUFDLElBQUksRUFBRSxFQUFFLElBQUksRUFBRSxJQUFJLENBQUMsSUFBSSxFQUFFLE9BQU8sRUFBRSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUMsQ0FBQztTQUN6RDtLQUNEO0lBRUQsTUFBTSxTQUFTLEdBQUcsQ0FBQyxTQUFTLEVBQUUsYUFBYSxFQUFFLE1BQU0sRUFBRSxlQUFlLENBQUMsQ0FBQztJQUN0RSxLQUFLLE1BQU0sUUFBUSxJQUFJLFNBQVMsRUFBRTtRQUNqQyxNQUFNLE9BQU8sR0FBRyxJQUFBLHlCQUFNLEVBQUMsUUFBUSxFQUFFLElBQUksQ0FBQyxDQUFDO1FBQ3ZDLElBQUksT0FBTztZQUFFLE9BQU8sT0FBTyxDQUFDO0tBQzVCO0lBRUQsT0FBTyxJQUFJLENBQUM7QUFDYixDQUFDLENBQUM7QUFqQlcsUUFBQSxXQUFXLGVBaUJ0QiJ9

package/dist/html-to-markdown.d.ts

@@ -0,0 +1,7 @@
+import type { Root as Hast } from "hast";
+export interface HtmlToMarkdownOptions {
+    extractHast?: false | ((hast: Hast) => Hast);
+    solveLinks?: string;
+}
+export declare const htmlToMarkdown: (html: string, options?: HtmlToMarkdownOptions) => string;
+//# sourceMappingURL=html-to-markdown.d.ts.map

package/dist/html-to-markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-markdown.d.ts","sourceRoot":"","sources":["../src/html-to-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAIzC,MAAM,WAAW,qBAAqB;IACrC,WAAW,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,cAAc,SAAU,MAAM,YAAY,qBAAqB,KAAG,MAI9E,CAAC"}

package/dist/html-to-markdown.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.htmlToMarkdown = void 0;
+const html_to_mdast_1 = require("./html-to-mdast");
+const mdast_to_markdown_1 = require("./mdast-to-markdown");
+const htmlToMarkdown = (html, options) => {
+    const mdast = (0, html_to_mdast_1.htmlToMdast)(html, options);
+    const markdown = (0, mdast_to_markdown_1.mdastToMarkdown)(mdast, options);
+    return markdown;
+};
+exports.htmlToMarkdown = htmlToMarkdown;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHRtbC10by1tYXJrZG93bi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9odG1sLXRvLW1hcmtkb3duLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUNBLG1EQUE4QztBQUM5QywyREFBc0Q7QUFPL0MsTUFBTSxjQUFjLEdBQUcsQ0FBQyxJQUFZLEVBQUUsT0FBK0IsRUFBVSxFQUFFO0lBQ3ZGLE1BQU0sS0FBSyxHQUFHLElBQUEsMkJBQVcsRUFBQyxJQUFJLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFDekMsTUFBTSxRQUFRLEdBQUcsSUFBQSxtQ0FBZSxFQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQztJQUNqRCxPQUFPLFFBQVEsQ0FBQztBQUNqQixDQUFDLENBQUM7QUFKVyxRQUFBLGNBQWMsa0JBSXpCIn0=

package/dist/html-to-mdast.d.ts

@@ -0,0 +1,7 @@
+import type { Root as Hast } from "hast";
+import type { Nodes as Mdast } from "mdast";
+export type HtmlToMdastOptions = {
+    extractHast?: false | ((hast: Hast) => Hast);
+};
+export declare const htmlToMdast: (html: string, options?: HtmlToMdastOptions) => Mdast;
+//# sourceMappingURL=html-to-mdast.d.ts.map

package/dist/html-to-mdast.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-to-mdast.d.ts","sourceRoot":"","sources":["../src/html-to-mdast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AACzC,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,MAAM,OAAO,CAAC;AAS5C,MAAM,MAAM,kBAAkB,GAAG;IAChC,WAAW,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,WAAW,SAAU,MAAM,YAAY,kBAAkB,KAAG,KAQxE,CAAC"}

package/dist/html-to-mdast.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.htmlToMdast = void 0;
+const hast_util_from_html_1 = require("hast-util-from-html");
+const hast_util_to_mdast_1 = require("hast-util-to-mdast");
+const extract_hast_1 = require("./extract-hast");
+const custom_div_handler_1 = require("./mdast-handlers/custom-div-handler");
+const math_handler_1 = require("./mdast-handlers/math-handler");
+const htmlToMdast = (html, options) => {
+    const hast = (0, hast_util_from_html_1.fromHtml)(html, { fragment: true });
+    const extractedHast = options?.extractHast ? options.extractHast(hast) : (0, extract_hast_1.extractHast)(hast);
+    const mdast = (0, hast_util_to_mdast_1.toMdast)(extractedHast, {
+        handlers: { div: custom_div_handler_1.customDivHandler, math: math_handler_1.mathHandler, br: () => { } },
+    });
+    return mdast;
+};
+exports.htmlToMdast = htmlToMdast;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHRtbC10by1tZGFzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9odG1sLXRvLW1kYXN0LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUdBLDZEQUErQztBQUMvQywyREFBNkM7QUFFN0MsaURBQTZDO0FBQzdDLDRFQUF1RTtBQUN2RSxnRUFBNEQ7QUFNckQsTUFBTSxXQUFXLEdBQUcsQ0FBQyxJQUFZLEVBQUUsT0FBNEIsRUFBUyxFQUFFO0lBQ2hGLE1BQU0sSUFBSSxHQUFHLElBQUEsOEJBQVEsRUFBQyxJQUFJLEVBQUUsRUFBRSxRQUFRLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztJQUNoRCxNQUFNLGFBQWEsR0FBRyxPQUFPLEVBQUUsV0FBVyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFBLDBCQUFXLEVBQUMsSUFBSSxDQUFDLENBQUM7SUFDM0YsTUFBTSxLQUFLLEdBQUcsSUFBQSw0QkFBTyxFQUFDLGFBQWEsRUFBRTtRQUNwQyxRQUFRLEVBQUUsRUFBRSxHQUFHLEVBQUUscUNBQWdCLEVBQUUsSUFBSSxFQUFFLDBCQUFXLEVBQUUsRUFBRSxFQUFFLEdBQUcsRUFBRSxHQUFFLENBQUMsRUFBRTtLQUNwRSxDQUFDLENBQUM7SUFFSCxPQUFPLEtBQUssQ0FBQztBQUNkLENBQUMsQ0FBQztBQVJXLFFBQUEsV0FBVyxlQVF0QiJ9

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./html-to-markdown";
+export * from "./md-splitter";
+export * from "./html-to-mdast";
+export * from "./mdast-to-markdown";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}

package/dist/index.js

@@ -0,0 +1,21 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./html-to-markdown"), exports);
+__exportStar(require("./md-splitter"), exports);
+__exportStar(require("./html-to-mdast"), exports);
+__exportStar(require("./mdast-to-markdown"), exports);
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7Ozs7Ozs7Ozs7OztBQUFBLHFEQUFtQztBQUNuQyxnREFBOEI7QUFDOUIsa0RBQWdDO0FBQ2hDLHNEQUFvQyJ9

package/dist/link-replacer.d.ts

@@ -0,0 +1,2 @@
+export declare const linkReplacer: (markdown: string, link: string) => string;
+//# sourceMappingURL=link-replacer.d.ts.map

package/dist/link-replacer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"link-replacer.d.ts","sourceRoot":"","sources":["../src/link-replacer.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,aAAc,MAAM,QAAQ,MAAM,WAM1D,CAAC"}

package/dist/link-replacer.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.linkReplacer = void 0;
+const linkReplacer = (markdown, link) => {
+    const regex = /(!?\[.*?\]\()(.+?)(\))/g;
+    return markdown.replace(regex, (match, pre, thisUrl, post) => {
+        if (thisUrl.startsWith("http") || thisUrl.startsWith("#"))
+            return match;
+        return `${pre}${new URL(thisUrl, link).href}${post}`;
+    });
+};
+exports.linkReplacer = linkReplacer;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibGluay1yZXBsYWNlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9saW5rLXJlcGxhY2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUFPLE1BQU0sWUFBWSxHQUFHLENBQUMsUUFBZ0IsRUFBRSxJQUFZLEVBQUUsRUFBRTtJQUM5RCxNQUFNLEtBQUssR0FBRyx5QkFBeUIsQ0FBQztJQUN4QyxPQUFPLFFBQVEsQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUMsS0FBSyxFQUFFLEdBQUcsRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLEVBQUU7UUFDNUQsSUFBSSxPQUFPLENBQUMsVUFBVSxDQUFDLE1BQU0sQ0FBQyxJQUFJLE9BQU8sQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDO1lBQUUsT0FBTyxLQUFLLENBQUM7UUFDeEUsT0FBTyxHQUFHLEdBQUcsR0FBRyxJQUFJLEdBQUcsQ0FBQyxPQUFPLEVBQUUsSUFBSSxDQUFDLENBQUMsSUFBSSxHQUFHLElBQUksRUFBRSxDQUFDO0lBQ3RELENBQUMsQ0FBQyxDQUFDO0FBQ0osQ0FBQyxDQUFDO0FBTlcsUUFBQSxZQUFZLGdCQU12QiJ9

package/dist/loaders/playwright.d.ts

@@ -0,0 +1,3 @@
+import playwright from "playwright";
+export declare const loadHtml: (url: string, context?: playwright.BrowserContext) => Promise<string>;
+//# sourceMappingURL=playwright.d.ts.map

package/dist/loaders/playwright.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"playwright.d.ts","sourceRoot":"","sources":["../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAC;AAIpC,eAAO,MAAM,QAAQ,QAAe,MAAM,YAAY,WAAW,cAAc,oBAiB9E,CAAC"}

package/dist/loaders/playwright.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadHtml = void 0;
+const playwright_1 = __importDefault(require("playwright"));
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const loadHtml = async (url, context) => {
+    let _browser = null;
+    let _context;
+    if (!context) {
+        _browser = await playwright_1.default.chromium.launch();
+        _context = await _browser.newContext();
+    }
+    else {
+        _context = context;
+    }
+    const page = await _context.newPage();
+    await page.goto(url);
+    await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
+    const html = await page.content();
+    await page.close();
+    if (_browser)
+        await _browser.close();
+    return html;
+};
+exports.loadHtml = loadHtml;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGxheXdyaWdodC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9sb2FkZXJzL3BsYXl3cmlnaHQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7O0FBQUEsNERBQW9DO0FBRXBDLE1BQU0sS0FBSyxHQUFHLENBQUMsRUFBVSxFQUFFLEVBQUUsQ0FBQyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsVUFBVSxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDO0FBRXpFLE1BQU0sUUFBUSxHQUFHLEtBQUssRUFBRSxHQUFXLEVBQUUsT0FBbUMsRUFBRSxFQUFFO0lBQ2xGLElBQUksUUFBUSxHQUE4QixJQUFJLENBQUM7SUFDL0MsSUFBSSxRQUFtQyxDQUFDO0lBQ3hDLElBQUksQ0FBQyxPQUFPLEVBQUU7UUFDYixRQUFRLEdBQUcsTUFBTSxvQkFBVSxDQUFDLFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUM5QyxRQUFRLEdBQUcsTUFBTSxRQUFRLENBQUMsVUFBVSxFQUFFLENBQUM7S0FDdkM7U0FBTTtRQUNOLFFBQVEsR0FBRyxPQUFPLENBQUM7S0FDbkI7SUFFRCxNQUFNLElBQUksR0FBRyxNQUFNLFFBQVEsQ0FBQyxPQUFPLEVBQUUsQ0FBQztJQUN0QyxNQUFNLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDckIsTUFBTSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxDQUFDLGdCQUFnQixDQUFDLGFBQWEsQ0FBQyxFQUFFLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDekUsTUFBTSxJQUFJLEdBQUcsTUFBTSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7SUFDbEMsTUFBTSxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDbkIsSUFBSSxRQUFRO1FBQUUsTUFBTSxRQUFRLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDckMsT0FBTyxJQUFJLENBQUM7QUFDYixDQUFDLENBQUM7QUFqQlcsUUFBQSxRQUFRLFlBaUJuQiJ9

package/dist/md-splitter.d.ts

@@ -0,0 +1,5 @@
+import type { Nodes as Mdast, RootContent } from "mdast";
+export declare const mdastSplitter: (mdast: Mdast, checker: (markdown: string) => Promise<boolean>, options?: {
+    signal?: AbortSignal;
+}) => Promise<RootContent[][]>;
+//# sourceMappingURL=md-splitter.d.ts.map

package/dist/md-splitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"md-splitter.d.ts","sourceRoot":"","sources":["../src/md-splitter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAoCzD,eAAO,MAAM,aAAa,UAClB,KAAK,sBACQ,MAAM,KAAK,QAAQ,OAAO,CAAC,YACrC;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,KAChC,QAAQ,WAAW,EAAE,EAAE,CAEzB,CAAC"}

package/dist/md-splitter.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mdastSplitter = void 0;
+const mdast_to_markdown_1 = require("./mdast-to-markdown");
+const common_1 = require("./utils/common");
+const mdast_utils_1 = require("./utils/mdast-utils");
+const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
+const getSplitterGenerator = function* () {
+    for (const splitter of PRIORITY_SPLITTERS)
+        yield splitter;
+};
+const _mdastSplitter = async (contents, checker, splitterGenerator) => {
+    const markdown = (0, mdast_to_markdown_1.mdastToMarkdown)((0, mdast_utils_1.warpRoot)(contents));
+    if ((await checker(markdown)) || 3 > contents.length)
+        return [contents];
+    const splitter = splitterGenerator.next().value;
+    const chunked = splitter
+        ? contents.reduce((acc, content) => {
+            if ((0, mdast_utils_1.internalType)(content) === splitter || acc.length === 0) {
+                acc.push([content]);
+                return acc;
+            }
+            acc[acc.length - 1].push(content);
+            return acc;
+        }, [])
+        : (0, common_1.chunk)(contents, 3);
+    const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
+    return Promise.all(splitting).then((chunks) => chunks.flat());
+};
+const mdastSplitter = async (mdast, checker, options) => {
+    return _mdastSplitter((0, mdast_utils_1.unwarpRoot)(mdast), checker, getSplitterGenerator());
+};
+exports.mdastSplitter = mdastSplitter;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWQtc3BsaXR0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWQtc3BsaXR0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQ0EsMkRBQXNEO0FBQ3RELDJDQUF1QztBQUN2QyxxREFBeUU7QUFFekUsTUFBTSxrQkFBa0IsR0FBRyxDQUFDLElBQUksRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUM7QUFFekYsTUFBTSxvQkFBb0IsR0FBRyxRQUFRLENBQUM7SUFDckMsS0FBSyxNQUFNLFFBQVEsSUFBSSxrQkFBa0I7UUFBRSxNQUFNLFFBQVEsQ0FBQztBQUMzRCxDQUFDLENBQUM7QUFFRixNQUFNLGNBQWMsR0FBRyxLQUFLLEVBQzNCLFFBQXVCLEVBQ3ZCLE9BQStDLEVBQy9DLGlCQUFvQyxFQUNULEVBQUU7SUFDN0IsTUFBTSxRQUFRLEdBQUcsSUFBQSxtQ0FBZSxFQUFDLElBQUEsc0JBQVEsRUFBQyxRQUFRLENBQUMsQ0FBQyxDQUFDO0lBQ3JELElBQUksQ0FBQyxNQUFNLE9BQU8sQ0FBQyxRQUFRLENBQUMsQ0FBQyxJQUFJLENBQUMsR0FBRyxRQUFRLENBQUMsTUFBTTtRQUFFLE9BQU8sQ0FBQyxRQUFRLENBQUMsQ0FBQztJQUN4RSxNQUFNLFFBQVEsR0FBRyxpQkFBaUIsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxLQUFLLENBQUM7SUFFaEQsTUFBTSxPQUFPLEdBQUcsUUFBUTtRQUN2QixDQUFDLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBa0IsQ0FBQyxHQUFHLEVBQUUsT0FBTyxFQUFFLEVBQUU7WUFDbEQsSUFBSSxJQUFBLDBCQUFZLEVBQUMsT0FBTyxDQUFDLEtBQUssUUFBUSxJQUFJLEdBQUcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFO2dCQUMzRCxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQztnQkFDcEIsT0FBTyxHQUFHLENBQUM7YUFDWDtZQUNELEdBQUcsQ0FBQyxHQUFHLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUNsQyxPQUFPLEdBQUcsQ0FBQztRQUNYLENBQUMsRUFBRSxFQUFFLENBQUM7UUFDUixDQUFDLENBQUMsSUFBQSxjQUFLLEVBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxDQUFDO0lBRXRCLE1BQU0sU0FBUyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLGNBQWMsQ0FBQyxLQUFLLEVBQUUsT0FBTyxFQUFFLGlCQUFpQixDQUFDLENBQUMsQ0FBQztJQUU1RixPQUFPLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBUyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBTSxFQUFFLEVBQUUsQ0FBQyxNQUFNLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztBQUMvRCxDQUFDLENBQUM7QUFFSyxNQUFNLGFBQWEsR0FBRyxLQUFLLEVBQ2pDLEtBQVksRUFDWixPQUErQyxFQUMvQyxPQUFrQyxFQUNQLEVBQUU7SUFDN0IsT0FBTyxjQUFjLENBQUMsSUFBQSx3QkFBVSxFQUFDLEtBQUssQ0FBQyxFQUFFLE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxDQUFDLENBQUM7QUFDM0UsQ0FBQyxDQUFDO0FBTlcsUUFBQSxhQUFhLGlCQU14QiJ9

package/dist/mdast-handlers/custom-div-handler.d.ts

@@ -0,0 +1,3 @@
+import { Handle } from "hast-util-to-mdast";
+export declare const customDivHandler: Handle;
+//# sourceMappingURL=custom-div-handler.d.ts.map

package/dist/mdast-handlers/custom-div-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom-div-handler.d.ts","sourceRoot":"","sources":["../../src/mdast-handlers/custom-div-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAmB,MAAM,oBAAoB,CAAC;AAK7D,eAAO,MAAM,gBAAgB,EAAE,MAsC9B,CAAC"}

package/dist/mdast-handlers/custom-div-handler.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.customDivHandler = void 0;
+const hast_util_to_mdast_1 = require("hast-util-to-mdast");
+const hast_util_to_text_1 = require("hast-util-to-text");
+const trim_trailing_lines_1 = require("trim-trailing-lines");
+const customDivHandler = (state, node) => {
+    const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
+    let highlightLanguage;
+    for (const className of classNames) {
+        if (String(className) === "highlight-text-html-basic") {
+            highlightLanguage = "html";
+            break;
+        }
+        else if (String(className).includes("highlight-source-")) {
+            highlightLanguage = String(className).slice("highlight-source-".length);
+            break;
+        }
+        else if (String(className).includes("language-")) {
+            highlightLanguage = String(className).slice("language-".length);
+            break;
+        }
+        else if (String(className).includes("code-block_wrapper")) {
+            const text = (0, hast_util_to_text_1.toText)(node);
+            if (!text.includes("."))
+                break;
+            const lang = (0, hast_util_to_text_1.toText)(node).split("\n")[0].split(".").slice(-1)[0];
+            highlightLanguage = lang;
+            break;
+        }
+    }
+    if (highlightLanguage !== undefined) {
+        const code = node.children.find((child) => child.type === "element" && child.tagName === "pre") || node;
+        const result = {
+            type: "code",
+            lang: highlightLanguage,
+            meta: null,
+            value: (0, trim_trailing_lines_1.trimTrailingLines)((0, hast_util_to_text_1.toText)(code)).trim(),
+        };
+        state.patch(node, result);
+        return result;
+    }
+    return hast_util_to_mdast_1.defaultHandlers.div(state, node);
+};
+exports.customDivHandler = customDivHandler;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3VzdG9tLWRpdi1oYW5kbGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL21kYXN0LWhhbmRsZXJzL2N1c3RvbS1kaXYtaGFuZGxlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFBQSwyREFBNkQ7QUFDN0QseURBQTJDO0FBRTNDLDZEQUF3RDtBQUVqRCxNQUFNLGdCQUFnQixHQUFXLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO0lBQ3RELE1BQU0sVUFBVSxHQUFHLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUU3RixJQUFJLGlCQUFxQyxDQUFDO0lBQzFDLEtBQUssTUFBTSxTQUFTLElBQUksVUFBVSxFQUFFO1FBQ2xDLElBQUksTUFBTSxDQUFDLFNBQVMsQ0FBQyxLQUFLLDJCQUEyQixFQUFFO1lBQ3JELGlCQUFpQixHQUFHLE1BQU0sQ0FBQztZQUMzQixNQUFNO1NBQ1A7YUFBTSxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQyxRQUFRLENBQUMsbUJBQW1CLENBQUMsRUFBRTtZQUMxRCxpQkFBaUIsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUMsS0FBSyxDQUFDLG1CQUFtQixDQUFDLE1BQU0sQ0FBQyxDQUFDO1lBQ3hFLE1BQU07U0FDUDthQUFNLElBQUksTUFBTSxDQUFDLFNBQVMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxXQUFXLENBQUMsRUFBRTtZQUNsRCxpQkFBaUIsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUMsS0FBSyxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUNoRSxNQUFNO1NBQ1A7YUFBTSxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQyxRQUFRLENBQUMsb0JBQW9CLENBQUMsRUFBRTtZQUMzRCxNQUFNLElBQUksR0FBRyxJQUFBLDBCQUFNLEVBQUMsSUFBSSxDQUFDLENBQUM7WUFDMUIsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDO2dCQUFFLE1BQU07WUFDL0IsTUFBTSxJQUFJLEdBQUcsSUFBQSwwQkFBTSxFQUFDLElBQUksQ0FBQyxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDakUsaUJBQWlCLEdBQUcsSUFBSSxDQUFDO1lBQ3pCLE1BQU07U0FDUDtLQUNGO0lBRUQsSUFBSSxpQkFBaUIsS0FBSyxTQUFTLEVBQUU7UUFDbkMsTUFBTSxJQUFJLEdBQ1IsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLEtBQUssQ0FBQyxJQUFJLEtBQUssU0FBUyxJQUFJLEtBQUssQ0FBQyxPQUFPLEtBQUssS0FBSyxDQUFDLElBQUksSUFBSSxDQUFDO1FBRTdGLE1BQU0sTUFBTSxHQUFTO1lBQ25CLElBQUksRUFBRSxNQUFNO1lBQ1osSUFBSSxFQUFFLGlCQUFpQjtZQUN2QixJQUFJLEVBQUUsSUFBSTtZQUNWLEtBQUssRUFBRSxJQUFBLHVDQUFpQixFQUFDLElBQUEsMEJBQU0sRUFBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLElBQUksRUFBRTtTQUM5QyxDQUFDO1FBQ0YsS0FBSyxDQUFDLEtBQUssQ0FBQyxJQUFJLEVBQUUsTUFBTSxDQUFDLENBQUM7UUFDMUIsT0FBTyxNQUFNLENBQUM7S0FDZjtJQUVELE9BQU8sb0NBQWUsQ0FBQyxHQUFHLENBQUMsS0FBSyxFQUFFLElBQUksQ0FBQyxDQUFDO0FBQzFDLENBQUMsQ0FBQztBQXRDVyxRQUFBLGdCQUFnQixvQkFzQzNCIn0=

package/dist/mdast-handlers/math-handler.d.ts

@@ -0,0 +1,3 @@
+import { Handle } from "hast-util-to-mdast";
+export declare const mathHandler: Handle;
+//# sourceMappingURL=math-handler.d.ts.map

package/dist/mdast-handlers/math-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"math-handler.d.ts","sourceRoot":"","sources":["../../src/mdast-handlers/math-handler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAI5C,eAAO,MAAM,WAAW,EAAE,MAMzB,CAAC"}

package/dist/mdast-handlers/math-handler.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mathHandler = void 0;
+const hast_util_to_html_1 = require("hast-util-to-html");
+const mathml_to_latex_1 = require("mathml-to-latex");
+const mathHandler = (state, node) => {
+    const mathMl = (0, hast_util_to_html_1.toHtml)(node);
+    const latex = mathml_to_latex_1.MathMLToLaTeX.convert(mathMl);
+    const result = { type: "inlineMath", value: latex };
+    state.patch(node, result);
+    return result;
+};
+exports.mathHandler = mathHandler;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWF0aC1oYW5kbGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL21kYXN0LWhhbmRsZXJzL21hdGgtaGFuZGxlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFBQSx5REFBMkM7QUFFM0MscURBQWdEO0FBR3pDLE1BQU0sV0FBVyxHQUFXLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO0lBQ2pELE1BQU0sTUFBTSxHQUFHLElBQUEsMEJBQU0sRUFBQyxJQUFJLENBQUMsQ0FBQztJQUM1QixNQUFNLEtBQUssR0FBRywrQkFBYSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUM1QyxNQUFNLE1BQU0sR0FBc0IsRUFBRSxJQUFJLEVBQUUsWUFBWSxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsQ0FBQztJQUN2RSxLQUFLLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxNQUFNLENBQUMsQ0FBQztJQUMxQixPQUFPLE1BQU0sQ0FBQztBQUNoQixDQUFDLENBQUM7QUFOVyxRQUFBLFdBQVcsZUFNdEIifQ==

package/dist/mdast-to-markdown.d.ts

@@ -0,0 +1,5 @@
+import type { Nodes as Mdast } from "mdast";
+export declare const mdastToMarkdown: (mdast: Mdast, options?: {
+    solveLinks?: string;
+}) => string;
+//# sourceMappingURL=mdast-to-markdown.d.ts.map

package/dist/mdast-to-markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mdast-to-markdown.d.ts","sourceRoot":"","sources":["../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,MAAM,OAAO,CAAC;AAQ5C,eAAO,MAAM,eAAe,UAAW,KAAK,YAAY;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,KAAG,MAQjF,CAAC"}

package/dist/mdast-to-markdown.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mdastToMarkdown = void 0;
+const mdast_util_gfm_1 = require("mdast-util-gfm");
+const mdast_util_math_1 = require("mdast-util-math");
+const mdast_util_to_markdown_1 = require("mdast-util-to-markdown");
+const link_replacer_1 = require("./link-replacer");
+const mdastToMarkdown = (mdast, options) => {
+    let markdown = (0, mdast_util_to_markdown_1.toMarkdown)(mdast, {
+        extensions: [(0, mdast_util_gfm_1.gfmToMarkdown)(), (0, mdast_util_math_1.mathToMarkdown)()],
+    });
+    if (options?.solveLinks)
+        markdown = (0, link_replacer_1.linkReplacer)(markdown, options.solveLinks);
+    return markdown;
+};
+exports.mdastToMarkdown = mdastToMarkdown;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdG8tbWFya2Rvd24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWRhc3QtdG8tbWFya2Rvd24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBRUEsbURBQStDO0FBQy9DLHFEQUFpRDtBQUNqRCxtRUFBb0Q7QUFFcEQsbURBQStDO0FBRXhDLE1BQU0sZUFBZSxHQUFHLENBQUMsS0FBWSxFQUFFLE9BQWlDLEVBQVUsRUFBRTtJQUMxRixJQUFJLFFBQVEsR0FBRyxJQUFBLG1DQUFVLEVBQUMsS0FBSyxFQUFFO1FBQ2hDLFVBQVUsRUFBRSxDQUFDLElBQUEsOEJBQWEsR0FBRSxFQUFFLElBQUEsZ0NBQWMsR0FBRSxDQUFDO0tBQy9DLENBQUMsQ0FBQztJQUVILElBQUksT0FBTyxFQUFFLFVBQVU7UUFBRSxRQUFRLEdBQUcsSUFBQSw0QkFBWSxFQUFDLFFBQVEsRUFBRSxPQUFPLENBQUMsVUFBVSxDQUFDLENBQUM7SUFFL0UsT0FBTyxRQUFRLENBQUM7QUFDakIsQ0FBQyxDQUFDO0FBUlcsUUFBQSxlQUFlLG1CQVExQiJ9

package/dist/utils/common.d.ts

@@ -0,0 +1,2 @@
+export declare const chunk: <T>(array: T[], size: number) => T[][];
+//# sourceMappingURL=common.d.ts.map

package/dist/utils/common.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/utils/common.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,KAAK,wBAAyB,MAAM,UAQhD,CAAC"}

package/dist/utils/common.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunk = void 0;
+const chunk = (array, size) => {
+    return array.reduce((acc, _, index) => {
+        if (index % size === 0) {
+            acc.push(array.slice(index, index + size));
+            return acc;
+        }
+        return acc;
+    }, []);
+};
+exports.chunk = chunk;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29tbW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3V0aWxzL2NvbW1vbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFBTyxNQUFNLEtBQUssR0FBRyxDQUFJLEtBQVUsRUFBRSxJQUFZLEVBQVMsRUFBRTtJQUMzRCxPQUFPLEtBQUssQ0FBQyxNQUFNLENBQVEsQ0FBQyxHQUFHLEVBQUUsQ0FBQyxFQUFFLEtBQUssRUFBRSxFQUFFO1FBQzVDLElBQUksS0FBSyxHQUFHLElBQUksS0FBSyxDQUFDLEVBQUU7WUFDdkIsR0FBRyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLEtBQUssRUFBRSxLQUFLLEdBQUcsSUFBSSxDQUFDLENBQUMsQ0FBQztZQUMzQyxPQUFPLEdBQUcsQ0FBQztTQUNYO1FBQ0QsT0FBTyxHQUFHLENBQUM7SUFDWixDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7QUFDUixDQUFDLENBQUM7QUFSVyxRQUFBLEtBQUssU0FRaEIifQ==

package/dist/utils/mdast-utils.d.ts

@@ -0,0 +1,5 @@
+import type { Nodes as Mdast, RootContent } from "mdast";
+export declare const unwarpRoot: (mdast: Mdast) => RootContent[];
+export declare const warpRoot: (mdast: RootContent[]) => Mdast;
+export declare const internalType: (content: RootContent) => string;
+//# sourceMappingURL=mdast-utils.d.ts.map

package/dist/utils/mdast-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mdast-utils.d.ts","sourceRoot":"","sources":["../../src/utils/mdast-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzD,eAAO,MAAM,UAAU,UAAW,KAAK,KAAG,WAAW,EAGpD,CAAC;AAEF,eAAO,MAAM,QAAQ,UAAW,WAAW,EAAE,KAAG,KAE/C,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,WAAW,KAAG,MAGnD,CAAC"}

package/dist/utils/mdast-utils.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.internalType = exports.warpRoot = exports.unwarpRoot = void 0;
+const unwarpRoot = (mdast) => {
+    if (mdast.type === "root")
+        return mdast.children;
+    return [mdast];
+};
+exports.unwarpRoot = unwarpRoot;
+const warpRoot = (mdast) => {
+    return { type: "root", children: mdast };
+};
+exports.warpRoot = warpRoot;
+const internalType = (content) => {
+    if (content.type === "heading")
+        return `h${content.depth}`;
+    return content.type;
+};
+exports.internalType = internalType;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdXRpbHMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvdXRpbHMvbWRhc3QtdXRpbHMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBRU8sTUFBTSxVQUFVLEdBQUcsQ0FBQyxLQUFZLEVBQWlCLEVBQUU7SUFDekQsSUFBSSxLQUFLLENBQUMsSUFBSSxLQUFLLE1BQU07UUFBRSxPQUFPLEtBQUssQ0FBQyxRQUFRLENBQUM7SUFDakQsT0FBTyxDQUFDLEtBQUssQ0FBQyxDQUFDO0FBQ2hCLENBQUMsQ0FBQztBQUhXLFFBQUEsVUFBVSxjQUdyQjtBQUVLLE1BQU0sUUFBUSxHQUFHLENBQUMsS0FBb0IsRUFBUyxFQUFFO0lBQ3ZELE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLFFBQVEsRUFBRSxLQUFLLEVBQUUsQ0FBQztBQUMxQyxDQUFDLENBQUM7QUFGVyxRQUFBLFFBQVEsWUFFbkI7QUFFSyxNQUFNLFlBQVksR0FBRyxDQUFDLE9BQW9CLEVBQVUsRUFBRTtJQUM1RCxJQUFJLE9BQU8sQ0FBQyxJQUFJLEtBQUssU0FBUztRQUFFLE9BQU8sSUFBSSxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDM0QsT0FBTyxPQUFPLENBQUMsSUFBSSxDQUFDO0FBQ3JCLENBQUMsQ0FBQztBQUhXLFFBQUEsWUFBWSxnQkFHdkIifQ==

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "webforai",
+  "version": "0.0.1",
+  "type": "module",
+  "description": "",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/inaridiy/webforai.git"
+  },
+  "homepage": "https://github.com/inaridiy/webforai.git",
+  "files": [
+    "dist",
+    "!dist/**/*.tsbuildinfo",
+    "src/**/*.ts",
+    "!src/**/*.test.ts",
+    "!src/**/*.test-d.ts",
+    "!src/**/*.bench.ts"
+  ],
+  "keywords": [],
+  "author": "inaridiy",
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "default": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./loaders/playwright": {
+      "default": "./dist/loaders/playwright.js",
+      "types": "./dist/loaders/playwright.d.ts"
+    }
+  },
+  "peerDependencies": {
+    "playwright": "^1.39.0"
+  },
+  "peerDependenciesMeta": {
+    "playwright": {
+      "optional": true
+    }
+  },
+  "dependencies": {
+    "hast-util-from-html": "^2.0.1",
+    "hast-util-select": "^6.0.1",
+    "hast-util-to-html": "^9.0.0",
+    "hast-util-to-mdast": "^10.1.0",
+    "hast-util-to-text": "^4.0.0",
+    "mathjax": "^3.2.2",
+    "mathml-to-latex": "^1.3.0",
+    "mdast-util-gfm": "^3.0.0",
+    "mdast-util-math": "^3.0.0",
+    "mdast-util-to-markdown": "^2.1.0",
+    "trim-trailing-lines": "^2.1.0",
+    "unist-util-remove": "^4.0.0"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "1.3.3",
+    "@tsconfig/recommended": "^1.0.3",
+    "@types/hast": "^3.0.2",
+    "@types/mdast": "^4.0.2",
+    "rimraf": "^5.0.5",
+    "typescript": "^5.2.2"
+  },
+  "scripts": {
+    "clean": "rimraf dist",
+    "build": "pnpm run clean && tsc ",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "format": "biome format . --write",
+    "lint": "biome check ."
+  }
+}

package/src/extract-hast.ts

@@ -0,0 +1,23 @@
+import type { Root as Hast } from "hast";
+import { select, selectAll } from "hast-util-select";
+
+import { remove } from "unist-util-remove";
+
+export const extractHast = (hast: Hast) => {
+	const removeSelectors = ["header", "footer", "nav"];
+
+	for (const selector of removeSelectors) {
+		const matchedNodes = selectAll(selector, hast);
+		for (const node of matchedNodes) {
+			remove(hast, { type: node.type, tagName: node.tagName });
+		}
+	}
+
+	const selectors = ["article", "div#article", "main", "div.container"];
+	for (const selector of selectors) {
+		const element = select(selector, hast);
+		if (element) return element;
+	}
+
+	return hast;
+};

package/src/html-to-markdown.ts

@@ -0,0 +1,14 @@
+import type { Root as Hast } from "hast";
+import { htmlToMdast } from "./html-to-mdast";
+import { mdastToMarkdown } from "./mdast-to-markdown";
+
+export interface HtmlToMarkdownOptions {
+	extractHast?: false | ((hast: Hast) => Hast);
+	solveLinks?: string;
+}
+
+export const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {
+	const mdast = htmlToMdast(html, options);
+	const markdown = mdastToMarkdown(mdast, options);
+	return markdown;
+};

package/src/html-to-mdast.ts

@@ -0,0 +1,23 @@
+import type { Root as Hast } from "hast";
+import type { Nodes as Mdast } from "mdast";
+
+import { fromHtml } from "hast-util-from-html";
+import { toMdast } from "hast-util-to-mdast";
+
+import { extractHast } from "./extract-hast";
+import { customDivHandler } from "./mdast-handlers/custom-div-handler";
+import { mathHandler } from "./mdast-handlers/math-handler";
+
+export type HtmlToMdastOptions = {
+	extractHast?: false | ((hast: Hast) => Hast);
+};
+
+export const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {
+	const hast = fromHtml(html, { fragment: true });
+	const extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);
+	const mdast = toMdast(extractedHast, {
+		handlers: { div: customDivHandler, math: mathHandler, br: () => {} },
+	});
+
+	return mdast;
+};

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./html-to-markdown";
+export * from "./md-splitter";
+export * from "./html-to-mdast";
+export * from "./mdast-to-markdown";

package/src/link-replacer.ts

@@ -0,0 +1,7 @@
+export const linkReplacer = (markdown: string, link: string) => {
+	const regex = /(!?\[.*?\]\()(.+?)(\))/g;
+	return markdown.replace(regex, (match, pre, thisUrl, post) => {
+		if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) return match;
+		return `${pre}${new URL(thisUrl, link).href}${post}`;
+	});
+};

package/src/loaders/playwright.ts

@@ -0,0 +1,22 @@
+import playwright from "playwright";
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export const loadHtml = async (url: string, context?: playwright.BrowserContext) => {
+	let _browser: playwright.Browser | null = null;
+	let _context: playwright.BrowserContext;
+	if (!context) {
+		_browser = await playwright.chromium.launch();
+		_context = await _browser.newContext();
+	} else {
+		_context = context;
+	}
+
+	const page = await _context.newPage();
+	await page.goto(url);
+	await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
+	const html = await page.content();
+	await page.close();
+	if (_browser) await _browser.close();
+	return html;
+};

package/src/md-splitter.ts

@@ -0,0 +1,43 @@
+import type { Nodes as Mdast, RootContent } from "mdast";
+import { mdastToMarkdown } from "./mdast-to-markdown";
+import { chunk } from "./utils/common";
+import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
+
+const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
+type SplitterGenerator = Generator<string, void, unknown>;
+const getSplitterGenerator = function* () {
+	for (const splitter of PRIORITY_SPLITTERS) yield splitter;
+};
+
+const _mdastSplitter = async (
+	contents: RootContent[],
+	checker: (markdown: string) => Promise<boolean>,
+	splitterGenerator: SplitterGenerator,
+): Promise<RootContent[][]> => {
+	const markdown = mdastToMarkdown(warpRoot(contents));
+	if ((await checker(markdown)) || 3 > contents.length) return [contents];
+	const splitter = splitterGenerator.next().value;
+
+	const chunked = splitter
+		? contents.reduce<RootContent[][]>((acc, content) => {
+				if (internalType(content) === splitter || acc.length === 0) {
+					acc.push([content]);
+					return acc;
+				}
+				acc[acc.length - 1].push(content);
+				return acc;
+		  }, [])
+		: chunk(contents, 3);
+
+	const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
+
+	return Promise.all(splitting).then((chunks) => chunks.flat());
+};
+
+export const mdastSplitter = async (
+	mdast: Mdast,
+	checker: (markdown: string) => Promise<boolean>,
+	options?: { signal?: AbortSignal }, //TODO
+): Promise<RootContent[][]> => {
+	return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
+};

package/src/mdast-handlers/custom-div-handler.ts

@@ -0,0 +1,44 @@
+import { Handle, defaultHandlers } from "hast-util-to-mdast";
+import { toText } from "hast-util-to-text";
+import { Code } from "mdast";
+import { trimTrailingLines } from "trim-trailing-lines";
+
+export const customDivHandler: Handle = (state, node) => {
+  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
+
+  let highlightLanguage: string | undefined;
+  for (const className of classNames) {
+    if (String(className) === "highlight-text-html-basic") {
+      highlightLanguage = "html";
+      break;
+    } else if (String(className).includes("highlight-source-")) {
+      highlightLanguage = String(className).slice("highlight-source-".length);
+      break;
+    } else if (String(className).includes("language-")) {
+      highlightLanguage = String(className).slice("language-".length);
+      break;
+    } else if (String(className).includes("code-block_wrapper")) {
+      const text = toText(node);
+      if (!text.includes(".")) break;
+      const lang = toText(node).split("\n")[0].split(".").slice(-1)[0];
+      highlightLanguage = lang;
+      break;
+    }
+  }
+
+  if (highlightLanguage !== undefined) {
+    const code =
+      node.children.find((child) => child.type === "element" && child.tagName === "pre") || node;
+
+    const result: Code = {
+      type: "code",
+      lang: highlightLanguage,
+      meta: null,
+      value: trimTrailingLines(toText(code)).trim(),
+    };
+    state.patch(node, result);
+    return result;
+  }
+
+  return defaultHandlers.div(state, node);
+};

package/src/mdast-handlers/math-handler.ts

@@ -0,0 +1,12 @@
+import { toHtml } from "hast-util-to-html";
+import { Handle } from "hast-util-to-mdast";
+import { MathMLToLaTeX } from "mathml-to-latex";
+import type { InlineMath, Math } from "mdast-util-math";
+
+export const mathHandler: Handle = (state, node) => {
+  const mathMl = toHtml(node);
+  const latex = MathMLToLaTeX.convert(mathMl);
+  const result: InlineMath | Math = { type: "inlineMath", value: latex };
+  state.patch(node, result);
+  return result;
+};

package/src/mdast-to-markdown.ts

@@ -0,0 +1,17 @@
+import type { Nodes as Mdast } from "mdast";
+
+import { gfmToMarkdown } from "mdast-util-gfm";
+import { mathToMarkdown } from "mdast-util-math";
+import { toMarkdown } from "mdast-util-to-markdown";
+
+import { linkReplacer } from "./link-replacer";
+
+export const mdastToMarkdown = (mdast: Mdast, options?: { solveLinks?: string }): string => {
+	let markdown = toMarkdown(mdast, {
+		extensions: [gfmToMarkdown(), mathToMarkdown()],
+	});
+
+	if (options?.solveLinks) markdown = linkReplacer(markdown, options.solveLinks);
+
+	return markdown;
+};

package/src/utils/common.ts

@@ -0,0 +1,9 @@
+export const chunk = <T>(array: T[], size: number): T[][] => {
+	return array.reduce<T[][]>((acc, _, index) => {
+		if (index % size === 0) {
+			acc.push(array.slice(index, index + size));
+			return acc;
+		}
+		return acc;
+	}, []);
+};

package/src/utils/mdast-utils.ts

@@ -0,0 +1,15 @@
+import type { Nodes as Mdast, RootContent } from "mdast";
+
+export const unwarpRoot = (mdast: Mdast): RootContent[] => {
+	if (mdast.type === "root") return mdast.children;
+	return [mdast];
+};
+
+export const warpRoot = (mdast: RootContent[]): Mdast => {
+	return { type: "root", children: mdast };
+};
+
+export const internalType = (content: RootContent): string => {
+	if (content.type === "heading") return `h${content.depth}`;
+	return content.type;
+};