webforai 0.0.11 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +74 -74
- package/dist/extract-hast.js +7 -11
- package/dist/extract-hast.js.map +1 -0
- package/dist/html-to-markdown.js +6 -10
- package/dist/html-to-markdown.js.map +1 -0
- package/dist/html-to-mdast.js +11 -15
- package/dist/html-to-mdast.js.map +1 -0
- package/dist/index.js +5 -21
- package/dist/index.js.map +1 -0
- package/dist/link-replacer.js +2 -6
- package/dist/link-replacer.js.map +1 -0
- package/dist/loaders/playwright.js +4 -11
- package/dist/loaders/playwright.js.map +1 -0
- package/dist/md-splitter.js +9 -13
- package/dist/md-splitter.js.map +1 -0
- package/dist/mdast-handlers/custom-div-handler.js +9 -13
- package/dist/mdast-handlers/custom-div-handler.js.map +1 -0
- package/dist/mdast-handlers/math-handler.js +6 -10
- package/dist/mdast-handlers/math-handler.js.map +1 -0
- package/dist/mdast-to-markdown.js +10 -14
- package/dist/mdast-to-markdown.js.map +1 -0
- package/dist/utils/common.js +2 -6
- package/dist/utils/common.js.map +1 -0
- package/dist/utils/mdast-utils.js +4 -10
- package/dist/utils/mdast-utils.js.map +1 -0
- package/package.json +3 -2
- package/src/extract-hast.ts +23 -23
- package/src/html-to-markdown.ts +14 -14
- package/src/html-to-mdast.ts +23 -23
- package/src/index.ts +4 -4
- package/src/link-replacer.ts +7 -7
- package/src/loaders/playwright.ts +22 -22
- package/src/md-splitter.ts +43 -43
- package/src/mdast-handlers/custom-div-handler.ts +43 -44
- package/src/mdast-handlers/math-handler.ts +12 -12
- package/src/mdast-to-markdown.ts +18 -18
- package/src/utils/common.ts +9 -9
- package/src/utils/mdast-utils.ts +16 -16
- package/dist/extract-hast.d.ts +0 -3
- package/dist/extract-hast.d.ts.map +0 -1
- package/dist/html-to-markdown.d.ts +0 -7
- package/dist/html-to-markdown.d.ts.map +0 -1
- package/dist/html-to-mdast.d.ts +0 -7
- package/dist/html-to-mdast.d.ts.map +0 -1
- package/dist/index.d.ts +0 -5
- package/dist/index.d.ts.map +0 -1
- package/dist/link-replacer.d.ts +0 -2
- package/dist/link-replacer.d.ts.map +0 -1
- package/dist/loaders/playwright.d.ts +0 -3
- package/dist/loaders/playwright.d.ts.map +0 -1
- package/dist/md-splitter.d.ts +0 -5
- package/dist/md-splitter.d.ts.map +0 -1
- package/dist/mdast-handlers/custom-div-handler.d.ts +0 -3
- package/dist/mdast-handlers/custom-div-handler.d.ts.map +0 -1
- package/dist/mdast-handlers/math-handler.d.ts +0 -3
- package/dist/mdast-handlers/math-handler.d.ts.map +0 -1
- package/dist/mdast-to-markdown.d.ts +0 -5
- package/dist/mdast-to-markdown.d.ts.map +0 -1
- package/dist/utils/common.d.ts +0 -2
- package/dist/utils/common.d.ts.map +0 -1
- package/dist/utils/mdast-utils.d.ts +0 -5
- package/dist/utils/mdast-utils.d.ts.map +0 -1
package/package.json
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webforai",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.13",
|
|
4
4
|
"description": "A library that provides a web interface for AI",
|
|
5
5
|
"author": "inaridiy",
|
|
6
6
|
"license": "MIT",
|
|
7
|
+
"type": "module",
|
|
7
8
|
"keywords": [
|
|
8
9
|
"web",
|
|
9
10
|
"ai",
|
|
@@ -70,7 +71,7 @@
|
|
|
70
71
|
},
|
|
71
72
|
"scripts": {
|
|
72
73
|
"clean": "rimraf dist",
|
|
73
|
-
"build": "pnpm run clean && tsc ",
|
|
74
|
+
"build": "pnpm run clean && tsc --module es2015",
|
|
74
75
|
"test": "echo \"Error: no test specified\" && exit 1",
|
|
75
76
|
"format": "biome format . --write",
|
|
76
77
|
"lint": "biome check ."
|
package/src/extract-hast.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
import { select, selectAll } from "hast-util-select";
|
|
3
|
-
|
|
4
|
-
import { remove } from "unist-util-remove";
|
|
5
|
-
|
|
6
|
-
export const extractHast = (hast: Hast) => {
|
|
7
|
-
const removeSelectors = ["header", "footer", "nav"];
|
|
8
|
-
|
|
9
|
-
for (const selector of removeSelectors) {
|
|
10
|
-
const matchedNodes = selectAll(selector, hast);
|
|
11
|
-
for (const node of matchedNodes) {
|
|
12
|
-
remove(hast, { type: node.type, tagName: node.tagName });
|
|
13
|
-
}
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
const selectors = ["article", "div#article", "main", "div.container"];
|
|
17
|
-
for (const selector of selectors) {
|
|
18
|
-
const element = select(selector, hast);
|
|
19
|
-
if (element) return element;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
return hast;
|
|
23
|
-
};
|
|
1
|
+
import type { Root as Hast } from "hast";
|
|
2
|
+
import { select, selectAll } from "hast-util-select";
|
|
3
|
+
|
|
4
|
+
import { remove } from "unist-util-remove";
|
|
5
|
+
|
|
6
|
+
export const extractHast = (hast: Hast) => {
|
|
7
|
+
const removeSelectors = ["header", "footer", "nav"];
|
|
8
|
+
|
|
9
|
+
for (const selector of removeSelectors) {
|
|
10
|
+
const matchedNodes = selectAll(selector, hast);
|
|
11
|
+
for (const node of matchedNodes) {
|
|
12
|
+
remove(hast, { type: node.type, tagName: node.tagName });
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const selectors = ["article", "div#article", "main", "div.container"];
|
|
17
|
+
for (const selector of selectors) {
|
|
18
|
+
const element = select(selector, hast);
|
|
19
|
+
if (element) return element;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
return hast;
|
|
23
|
+
};
|
package/src/html-to-markdown.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
import { htmlToMdast } from "./html-to-mdast";
|
|
3
|
-
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
4
|
-
|
|
5
|
-
export interface HtmlToMarkdownOptions {
|
|
6
|
-
extractHast?: false | ((hast: Hast) => Hast);
|
|
7
|
-
solveLinks?: string;
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
export const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {
|
|
11
|
-
const mdast = htmlToMdast(html, options);
|
|
12
|
-
const markdown = mdastToMarkdown(mdast, options);
|
|
13
|
-
return markdown;
|
|
14
|
-
};
|
|
1
|
+
import type { Root as Hast } from "hast";
|
|
2
|
+
import { htmlToMdast } from "./html-to-mdast";
|
|
3
|
+
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
4
|
+
|
|
5
|
+
export interface HtmlToMarkdownOptions {
|
|
6
|
+
extractHast?: false | ((hast: Hast) => Hast);
|
|
7
|
+
solveLinks?: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {
|
|
11
|
+
const mdast = htmlToMdast(html, options);
|
|
12
|
+
const markdown = mdastToMarkdown(mdast, options);
|
|
13
|
+
return markdown;
|
|
14
|
+
};
|
package/src/html-to-mdast.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
import type { Nodes as Mdast } from "mdast";
|
|
3
|
-
|
|
4
|
-
import { fromHtml } from "hast-util-from-html";
|
|
5
|
-
import { toMdast } from "hast-util-to-mdast";
|
|
6
|
-
|
|
7
|
-
import { extractHast } from "./extract-hast";
|
|
8
|
-
import { customDivHandler } from "./mdast-handlers/custom-div-handler";
|
|
9
|
-
import { mathHandler } from "./mdast-handlers/math-handler";
|
|
10
|
-
|
|
11
|
-
export type HtmlToMdastOptions = {
|
|
12
|
-
extractHast?: false | ((hast: Hast) => Hast);
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
export const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {
|
|
16
|
-
const hast = fromHtml(html, { fragment: true });
|
|
17
|
-
const extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);
|
|
18
|
-
const mdast = toMdast(extractedHast, {
|
|
19
|
-
handlers: { div: customDivHandler, math: mathHandler, br: () => {} },
|
|
20
|
-
});
|
|
21
|
-
|
|
22
|
-
return mdast;
|
|
23
|
-
};
|
|
1
|
+
import type { Root as Hast } from "hast";
|
|
2
|
+
import type { Nodes as Mdast } from "mdast";
|
|
3
|
+
|
|
4
|
+
import { fromHtml } from "hast-util-from-html";
|
|
5
|
+
import { toMdast } from "hast-util-to-mdast";
|
|
6
|
+
|
|
7
|
+
import { extractHast } from "./extract-hast";
|
|
8
|
+
import { customDivHandler } from "./mdast-handlers/custom-div-handler";
|
|
9
|
+
import { mathHandler } from "./mdast-handlers/math-handler";
|
|
10
|
+
|
|
11
|
+
export type HtmlToMdastOptions = {
|
|
12
|
+
extractHast?: false | ((hast: Hast) => Hast);
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {
|
|
16
|
+
const hast = fromHtml(html, { fragment: true });
|
|
17
|
+
const extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);
|
|
18
|
+
const mdast = toMdast(extractedHast, {
|
|
19
|
+
handlers: { div: customDivHandler, math: mathHandler, br: () => {} },
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
return mdast;
|
|
23
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export * from "./html-to-markdown";
|
|
2
|
-
export * from "./md-splitter";
|
|
3
|
-
export * from "./html-to-mdast";
|
|
4
|
-
export * from "./mdast-to-markdown";
|
|
1
|
+
export * from "./html-to-markdown";
|
|
2
|
+
export * from "./md-splitter";
|
|
3
|
+
export * from "./html-to-mdast";
|
|
4
|
+
export * from "./mdast-to-markdown";
|
package/src/link-replacer.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export const linkReplacer = (markdown: string, link: string) => {
|
|
2
|
-
const regex = /(!?\[.*?\]\()(.+?)(\))/g;
|
|
3
|
-
return markdown.replace(regex, (match, pre, thisUrl, post) => {
|
|
4
|
-
if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) return match;
|
|
5
|
-
return `${pre}${new URL(thisUrl, link).href}${post}`;
|
|
6
|
-
});
|
|
7
|
-
};
|
|
1
|
+
export const linkReplacer = (markdown: string, link: string) => {
|
|
2
|
+
const regex = /(!?\[.*?\]\()(.+?)(\))/g;
|
|
3
|
+
return markdown.replace(regex, (match, pre, thisUrl, post) => {
|
|
4
|
+
if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) return match;
|
|
5
|
+
return `${pre}${new URL(thisUrl, link).href}${post}`;
|
|
6
|
+
});
|
|
7
|
+
};
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
import playwright from "playwright";
|
|
2
|
-
|
|
3
|
-
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4
|
-
|
|
5
|
-
export const loadHtml = async (url: string, context?: playwright.BrowserContext) => {
|
|
6
|
-
let _browser: playwright.Browser | null = null;
|
|
7
|
-
let _context: playwright.BrowserContext;
|
|
8
|
-
if (!context) {
|
|
9
|
-
_browser = await playwright.chromium.launch();
|
|
10
|
-
_context = await _browser.newContext();
|
|
11
|
-
} else {
|
|
12
|
-
_context = context;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
const page = await _context.newPage();
|
|
16
|
-
await page.goto(url);
|
|
17
|
-
await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
|
|
18
|
-
const html = await page.content();
|
|
19
|
-
await page.close();
|
|
20
|
-
if (_browser) await _browser.close();
|
|
21
|
-
return html;
|
|
22
|
-
};
|
|
1
|
+
import * as playwright from "playwright";
|
|
2
|
+
|
|
3
|
+
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4
|
+
|
|
5
|
+
export const loadHtml = async (url: string, context?: playwright.BrowserContext) => {
|
|
6
|
+
let _browser: playwright.Browser | null = null;
|
|
7
|
+
let _context: playwright.BrowserContext;
|
|
8
|
+
if (!context) {
|
|
9
|
+
_browser = await playwright.chromium.launch();
|
|
10
|
+
_context = await _browser.newContext();
|
|
11
|
+
} else {
|
|
12
|
+
_context = context;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const page = await _context.newPage();
|
|
16
|
+
await page.goto(url);
|
|
17
|
+
await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
|
|
18
|
+
const html = await page.content();
|
|
19
|
+
await page.close();
|
|
20
|
+
if (_browser) await _browser.close();
|
|
21
|
+
return html;
|
|
22
|
+
};
|
package/src/md-splitter.ts
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
3
|
-
import { chunk } from "./utils/common";
|
|
4
|
-
import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
|
|
5
|
-
|
|
6
|
-
const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
|
|
7
|
-
type SplitterGenerator = Generator<string, void, unknown>;
|
|
8
|
-
const getSplitterGenerator = function* () {
|
|
9
|
-
for (const splitter of PRIORITY_SPLITTERS) yield splitter;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
const _mdastSplitter = async (
|
|
13
|
-
contents: RootContent[],
|
|
14
|
-
checker: (markdown: string) => Promise<boolean>,
|
|
15
|
-
splitterGenerator: SplitterGenerator,
|
|
16
|
-
): Promise<RootContent[][]> => {
|
|
17
|
-
const markdown = mdastToMarkdown(warpRoot(contents));
|
|
18
|
-
if ((await checker(markdown)) || 3 > contents.length) return [contents];
|
|
19
|
-
const splitter = splitterGenerator.next().value;
|
|
20
|
-
|
|
21
|
-
const chunked = splitter
|
|
22
|
-
? contents.reduce<RootContent[][]>((acc, content) => {
|
|
23
|
-
if (internalType(content) === splitter || acc.length === 0) {
|
|
24
|
-
acc.push([content]);
|
|
25
|
-
return acc;
|
|
26
|
-
}
|
|
27
|
-
acc[acc.length - 1].push(content);
|
|
28
|
-
return acc;
|
|
29
|
-
}, [])
|
|
30
|
-
: chunk(contents, 3);
|
|
31
|
-
|
|
32
|
-
const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
|
|
33
|
-
|
|
34
|
-
return Promise.all(splitting).then((chunks) => chunks.flat());
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
export const mdastSplitter = async (
|
|
38
|
-
mdast: Mdast,
|
|
39
|
-
checker: (markdown: string) => Promise<boolean>,
|
|
40
|
-
options?: { signal?: AbortSignal }, //TODO
|
|
41
|
-
): Promise<RootContent[][]> => {
|
|
42
|
-
return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
|
|
43
|
-
};
|
|
1
|
+
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
+
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
3
|
+
import { chunk } from "./utils/common";
|
|
4
|
+
import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
|
|
5
|
+
|
|
6
|
+
const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
|
|
7
|
+
type SplitterGenerator = Generator<string, void, unknown>;
|
|
8
|
+
const getSplitterGenerator = function* () {
|
|
9
|
+
for (const splitter of PRIORITY_SPLITTERS) yield splitter;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
const _mdastSplitter = async (
|
|
13
|
+
contents: RootContent[],
|
|
14
|
+
checker: (markdown: string) => Promise<boolean>,
|
|
15
|
+
splitterGenerator: SplitterGenerator,
|
|
16
|
+
): Promise<RootContent[][]> => {
|
|
17
|
+
const markdown = mdastToMarkdown(warpRoot(contents));
|
|
18
|
+
if ((await checker(markdown)) || 3 > contents.length) return [contents];
|
|
19
|
+
const splitter = splitterGenerator.next().value;
|
|
20
|
+
|
|
21
|
+
const chunked = splitter
|
|
22
|
+
? contents.reduce<RootContent[][]>((acc, content) => {
|
|
23
|
+
if (internalType(content) === splitter || acc.length === 0) {
|
|
24
|
+
acc.push([content]);
|
|
25
|
+
return acc;
|
|
26
|
+
}
|
|
27
|
+
acc[acc.length - 1].push(content);
|
|
28
|
+
return acc;
|
|
29
|
+
}, [])
|
|
30
|
+
: chunk(contents, 3);
|
|
31
|
+
|
|
32
|
+
const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
|
|
33
|
+
|
|
34
|
+
return Promise.all(splitting).then((chunks) => chunks.flat());
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export const mdastSplitter = async (
|
|
38
|
+
mdast: Mdast,
|
|
39
|
+
checker: (markdown: string) => Promise<boolean>,
|
|
40
|
+
options?: { signal?: AbortSignal }, //TODO
|
|
41
|
+
): Promise<RootContent[][]> => {
|
|
42
|
+
return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
|
|
43
|
+
};
|
|
@@ -1,44 +1,43 @@
|
|
|
1
|
-
import { Handle, defaultHandlers } from "hast-util-to-mdast";
|
|
2
|
-
import { toText } from "hast-util-to-text";
|
|
3
|
-
import { Code } from "mdast";
|
|
4
|
-
import { trimTrailingLines } from "trim-trailing-lines";
|
|
5
|
-
|
|
6
|
-
export const customDivHandler: Handle = (state, node) => {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
};
|
|
1
|
+
import { type Handle, defaultHandlers } from "hast-util-to-mdast";
|
|
2
|
+
import { toText } from "hast-util-to-text";
|
|
3
|
+
import type { Code } from "mdast";
|
|
4
|
+
import { trimTrailingLines } from "trim-trailing-lines";
|
|
5
|
+
|
|
6
|
+
export const customDivHandler: Handle = (state, node) => {
|
|
7
|
+
const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
|
|
8
|
+
|
|
9
|
+
let highlightLanguage: string | undefined;
|
|
10
|
+
for (const className of classNames) {
|
|
11
|
+
if (String(className) === "highlight-text-html-basic") {
|
|
12
|
+
highlightLanguage = "html";
|
|
13
|
+
break;
|
|
14
|
+
} else if (String(className).includes("highlight-source-")) {
|
|
15
|
+
highlightLanguage = String(className).slice("highlight-source-".length);
|
|
16
|
+
break;
|
|
17
|
+
} else if (String(className).includes("language-")) {
|
|
18
|
+
highlightLanguage = String(className).slice("language-".length);
|
|
19
|
+
break;
|
|
20
|
+
} else if (String(className).includes("code-block_wrapper")) {
|
|
21
|
+
const text = toText(node);
|
|
22
|
+
if (!text.includes(".")) break;
|
|
23
|
+
const lang = toText(node).split("\n")[0].split(".").slice(-1)[0];
|
|
24
|
+
highlightLanguage = lang;
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (highlightLanguage !== undefined) {
|
|
30
|
+
const code = node.children.find((child) => child.type === "element" && child.tagName === "pre") || node;
|
|
31
|
+
|
|
32
|
+
const result: Code = {
|
|
33
|
+
type: "code",
|
|
34
|
+
lang: highlightLanguage,
|
|
35
|
+
meta: null,
|
|
36
|
+
value: trimTrailingLines(toText(code)).trim(),
|
|
37
|
+
};
|
|
38
|
+
state.patch(node, result);
|
|
39
|
+
return result;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return defaultHandlers.div(state, node);
|
|
43
|
+
};
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { toHtml } from "hast-util-to-html";
|
|
2
|
-
import { Handle } from "hast-util-to-mdast";
|
|
3
|
-
import { MathMLToLaTeX } from "mathml-to-latex";
|
|
4
|
-
import type { InlineMath, Math } from "mdast-util-math";
|
|
5
|
-
|
|
6
|
-
export const mathHandler: Handle = (state, node) => {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
};
|
|
1
|
+
import { toHtml } from "hast-util-to-html";
|
|
2
|
+
import type { Handle } from "hast-util-to-mdast";
|
|
3
|
+
import { MathMLToLaTeX } from "mathml-to-latex";
|
|
4
|
+
import type { InlineMath, Math as MDMath } from "mdast-util-math";
|
|
5
|
+
|
|
6
|
+
export const mathHandler: Handle = (state, node) => {
|
|
7
|
+
const mathMl = toHtml(node);
|
|
8
|
+
const latex = MathMLToLaTeX.convert(mathMl);
|
|
9
|
+
const result: InlineMath | MDMath = { type: "inlineMath", value: latex };
|
|
10
|
+
state.patch(node, result);
|
|
11
|
+
return result;
|
|
12
|
+
};
|
package/src/mdast-to-markdown.ts
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
|
|
3
|
-
import { gfmToMarkdown } from "mdast-util-gfm";
|
|
4
|
-
import { mathToMarkdown } from "mdast-util-math";
|
|
5
|
-
import { toMarkdown } from "mdast-util-to-markdown";
|
|
6
|
-
|
|
7
|
-
import { linkReplacer } from "./link-replacer";
|
|
8
|
-
import { warpRoot } from "./utils/mdast-utils";
|
|
9
|
-
|
|
10
|
-
export const mdastToMarkdown = (mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string => {
|
|
11
|
-
let markdown = toMarkdown(warpRoot(mdast), {
|
|
12
|
-
extensions: [gfmToMarkdown(), mathToMarkdown()],
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
if (options?.solveLinks) markdown = linkReplacer(markdown, options.solveLinks);
|
|
16
|
-
|
|
17
|
-
return markdown;
|
|
18
|
-
};
|
|
1
|
+
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
+
|
|
3
|
+
import { gfmToMarkdown } from "mdast-util-gfm";
|
|
4
|
+
import { mathToMarkdown } from "mdast-util-math";
|
|
5
|
+
import { toMarkdown } from "mdast-util-to-markdown";
|
|
6
|
+
|
|
7
|
+
import { linkReplacer } from "./link-replacer";
|
|
8
|
+
import { warpRoot } from "./utils/mdast-utils";
|
|
9
|
+
|
|
10
|
+
export const mdastToMarkdown = (mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string => {
|
|
11
|
+
let markdown = toMarkdown(warpRoot(mdast), {
|
|
12
|
+
extensions: [gfmToMarkdown(), mathToMarkdown()],
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
if (options?.solveLinks) markdown = linkReplacer(markdown, options.solveLinks);
|
|
16
|
+
|
|
17
|
+
return markdown;
|
|
18
|
+
};
|
package/src/utils/common.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export const chunk = <T>(array: T[], size: number): T[][] => {
|
|
2
|
-
return array.reduce<T[][]>((acc, _, index) => {
|
|
3
|
-
if (index % size === 0) {
|
|
4
|
-
acc.push(array.slice(index, index + size));
|
|
5
|
-
return acc;
|
|
6
|
-
}
|
|
7
|
-
return acc;
|
|
8
|
-
}, []);
|
|
9
|
-
};
|
|
1
|
+
export const chunk = <T>(array: T[], size: number): T[][] => {
|
|
2
|
+
return array.reduce<T[][]>((acc, _, index) => {
|
|
3
|
+
if (index % size === 0) {
|
|
4
|
+
acc.push(array.slice(index, index + size));
|
|
5
|
+
return acc;
|
|
6
|
+
}
|
|
7
|
+
return acc;
|
|
8
|
+
}, []);
|
|
9
|
+
};
|
package/src/utils/mdast-utils.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
|
|
3
|
-
export const unwarpRoot = (mdast: Mdast): RootContent[] => {
|
|
4
|
-
if (mdast.type === "root") return mdast.children;
|
|
5
|
-
return [mdast];
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
export const warpRoot = (mdast: RootContent[] | Mdast): Mdast => {
|
|
9
|
-
if (Array.isArray(mdast)) return { type: "root", children: mdast };
|
|
10
|
-
return mdast;
|
|
11
|
-
};
|
|
12
|
-
|
|
13
|
-
export const internalType = (content: RootContent): string => {
|
|
14
|
-
if (content.type === "heading") return `h${content.depth}`;
|
|
15
|
-
return content.type;
|
|
16
|
-
};
|
|
1
|
+
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
+
|
|
3
|
+
export const unwarpRoot = (mdast: Mdast): RootContent[] => {
|
|
4
|
+
if (mdast.type === "root") return mdast.children;
|
|
5
|
+
return [mdast];
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export const warpRoot = (mdast: RootContent[] | Mdast): Mdast => {
|
|
9
|
+
if (Array.isArray(mdast)) return { type: "root", children: mdast };
|
|
10
|
+
return mdast;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export const internalType = (content: RootContent): string => {
|
|
14
|
+
if (content.type === "heading") return `h${content.depth}`;
|
|
15
|
+
return content.type;
|
|
16
|
+
};
|
package/dist/extract-hast.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"extract-hast.d.ts","sourceRoot":"","sources":["../src/extract-hast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAKzC,eAAO,MAAM,WAAW,SAAU,IAAI,kCAiBrC,CAAC"}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
export interface HtmlToMarkdownOptions {
|
|
3
|
-
extractHast?: false | ((hast: Hast) => Hast);
|
|
4
|
-
solveLinks?: string;
|
|
5
|
-
}
|
|
6
|
-
export declare const htmlToMarkdown: (html: string, options?: HtmlToMarkdownOptions) => string;
|
|
7
|
-
//# sourceMappingURL=html-to-markdown.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"html-to-markdown.d.ts","sourceRoot":"","sources":["../src/html-to-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AAIzC,MAAM,WAAW,qBAAqB;IACrC,WAAW,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,cAAc,SAAU,MAAM,YAAY,qBAAqB,KAAG,MAI9E,CAAC"}
|
package/dist/html-to-mdast.d.ts
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
import type { Nodes as Mdast } from "mdast";
|
|
3
|
-
export type HtmlToMdastOptions = {
|
|
4
|
-
extractHast?: false | ((hast: Hast) => Hast);
|
|
5
|
-
};
|
|
6
|
-
export declare const htmlToMdast: (html: string, options?: HtmlToMdastOptions) => Mdast;
|
|
7
|
-
//# sourceMappingURL=html-to-mdast.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"html-to-mdast.d.ts","sourceRoot":"","sources":["../src/html-to-mdast.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,MAAM,CAAC;AACzC,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,MAAM,OAAO,CAAC;AAS5C,MAAM,MAAM,kBAAkB,GAAG;IAChC,WAAW,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,WAAW,SAAU,MAAM,YAAY,kBAAkB,KAAG,KAQxE,CAAC"}
|
package/dist/index.d.ts
DELETED
package/dist/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}
|
package/dist/link-replacer.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"link-replacer.d.ts","sourceRoot":"","sources":["../src/link-replacer.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,aAAc,MAAM,QAAQ,MAAM,WAM1D,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.d.ts","sourceRoot":"","sources":["../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAC;AAIpC,eAAO,MAAM,QAAQ,QAAe,MAAM,YAAY,WAAW,cAAc,oBAiB9E,CAAC"}
|
package/dist/md-splitter.d.ts
DELETED
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
export declare const mdastSplitter: (mdast: Mdast, checker: (markdown: string) => Promise<boolean>, options?: {
|
|
3
|
-
signal?: AbortSignal;
|
|
4
|
-
}) => Promise<RootContent[][]>;
|
|
5
|
-
//# sourceMappingURL=md-splitter.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"md-splitter.d.ts","sourceRoot":"","sources":["../src/md-splitter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAoCzD,eAAO,MAAM,aAAa,UAClB,KAAK,sBACQ,MAAM,KAAK,QAAQ,OAAO,CAAC,YACrC;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,KAChC,QAAQ,WAAW,EAAE,EAAE,CAEzB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"custom-div-handler.d.ts","sourceRoot":"","sources":["../../src/mdast-handlers/custom-div-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAmB,MAAM,oBAAoB,CAAC;AAK7D,eAAO,MAAM,gBAAgB,EAAE,MAsC9B,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"math-handler.d.ts","sourceRoot":"","sources":["../../src/mdast-handlers/math-handler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAI5C,eAAO,MAAM,WAAW,EAAE,MAMzB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"mdast-to-markdown.d.ts","sourceRoot":"","sources":["../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AASzD,eAAO,MAAM,eAAe,UAAW,KAAK,GAAG,WAAW,EAAE,YAAY;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,KAAG,MAQjG,CAAC"}
|
package/dist/utils/common.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/utils/common.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,KAAK,wBAAyB,MAAM,UAQhD,CAAC"}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
export declare const unwarpRoot: (mdast: Mdast) => RootContent[];
|
|
3
|
-
export declare const warpRoot: (mdast: RootContent[] | Mdast) => Mdast;
|
|
4
|
-
export declare const internalType: (content: RootContent) => string;
|
|
5
|
-
//# sourceMappingURL=mdast-utils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"mdast-utils.d.ts","sourceRoot":"","sources":["../../src/utils/mdast-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzD,eAAO,MAAM,UAAU,UAAW,KAAK,KAAG,WAAW,EAGpD,CAAC;AAEF,eAAO,MAAM,QAAQ,UAAW,WAAW,EAAE,GAAG,KAAK,KAAG,KAGvD,CAAC;AAEF,eAAO,MAAM,YAAY,YAAa,WAAW,KAAG,MAGnD,CAAC"}
|