webforai 0.0.10 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +74 -74
- package/dist/extract-hast.js +7 -11
- package/dist/html-to-markdown.js +6 -10
- package/dist/html-to-mdast.js +11 -15
- package/dist/index.js +5 -21
- package/dist/link-replacer.js +2 -6
- package/dist/loaders/playwright.js +4 -11
- package/dist/md-splitter.js +9 -13
- package/dist/mdast-handlers/custom-div-handler.js +9 -13
- package/dist/mdast-handlers/math-handler.js +6 -10
- package/dist/mdast-to-markdown.js +10 -14
- package/dist/utils/common.js +2 -6
- package/dist/utils/mdast-utils.js +4 -10
- package/package.json +2 -1
- package/src/extract-hast.ts +23 -23
- package/src/html-to-markdown.ts +14 -14
- package/src/html-to-mdast.ts +23 -23
- package/src/index.ts +4 -4
- package/src/link-replacer.ts +7 -7
- package/src/loaders/playwright.ts +22 -22
- package/src/md-splitter.ts +43 -43
- package/src/mdast-handlers/custom-div-handler.ts +44 -44
- package/src/mdast-handlers/math-handler.ts +12 -12
- package/src/mdast-to-markdown.ts +18 -18
- package/src/utils/common.ts +9 -9
- package/src/utils/mdast-utils.ts +16 -16
package/LICENSE
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2023 inaridiy
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 inaridiy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,74 +1,74 @@
|
|
|
1
|
-
# Web for AI
|
|
2
|
-
|
|
3
|
-

|
|
13
|
-
|
|
14
|
-
## Demo
|
|
15
|
-
|
|
16
|
-
There is a demo API for Html2Markdown deployed on CloudflareWorker. Please access the following link
|
|
17
|
-
|
|
18
|
-
- [https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai](https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai)
|
|
19
|
-
- [https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E](https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E)
|
|
20
|
-
|
|
21
|
-
## Quick Start
|
|
22
|
-
|
|
23
|
-
Just install and execute scripts
|
|
24
|
-
|
|
25
|
-
```bash
|
|
26
|
-
pnpm i webforai playwright
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
```js
|
|
30
|
-
import { promises as fs } from "fs";
|
|
31
|
-
import { htmlToMarkdown, htmlToMdast } from "webforai";
|
|
32
|
-
import { loadHtml } from "webforai/loaders/playwright";
|
|
33
|
-
|
|
34
|
-
const url = "https://www.npmjs.com/package/webforai";
|
|
35
|
-
const html = await loadHtml(url);
|
|
36
|
-
|
|
37
|
-
const markdown = htmlToMarkdown(html, { solveLinks: url });
|
|
38
|
-
|
|
39
|
-
await fs.writeFile("output.md", markdown);
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
other examples are in [examples](./examples/src/index.ts)
|
|
43
|
-
|
|
44
|
-
## Usage
|
|
45
|
-
|
|
46
|
-
### Main Functions
|
|
47
|
-
|
|
48
|
-
**`htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string`**
|
|
49
|
-
Convert HTML to Markdown. By default, unnecessary HTML is excluded and processed.
|
|
50
|
-
If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
51
|
-
This function just calls htmlToMdast and mdastToMarkdown in that order internally.
|
|
52
|
-
|
|
53
|
-
**`htmlToMdast(html: string, options?: HtmlToMdastOptions): Mdast`**
|
|
54
|
-
This project uses Hast and Mdast as defined by syntax-tree internally.
|
|
55
|
-
This function converts HTML to Mdast, an intermediate representation, which is required when using `mdastSplitter`, etc.
|
|
56
|
-
|
|
57
|
-
**`mdastToMarkdown(mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string`**
|
|
58
|
-
Convert Mdast to Markdown. If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
59
|
-
|
|
60
|
-
### Loader Functions
|
|
61
|
-
|
|
62
|
-
**`loadHtml(url: string, options?: LoadHtmlOptions): Promise<string>`**
|
|
63
|
-
Load HTML from the specified URL. This function uses Playwright internally.
|
|
64
|
-
|
|
65
|
-
## Early Users
|
|
66
|
-
|
|
67
|
-
### Voice Genius - Explanatory video generation by AI
|
|
68
|
-
|
|
69
|
-
<img src="
|
|
70
|
-
|
|
71
|
-
by [@moons14](https://twitter.com/moons_dev)
|
|
72
|
-
|
|
73
|
-
AI can be used to automatically generate highly accurate commentary videos.
|
|
74
|
-
Supports conversations and presentations using synthesized voice, embedding of images in videos, etc.
|
|
1
|
+
# Web for AI
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+
|
|
5
|
+
A library that provides a web interface for AI
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- Ultra-precise HTML2Markdown conversion
|
|
10
|
+
- Ultra-precise Markdown segmentation based on AST
|
|
11
|
+
- Ultra-precise HTML retrieval functions using headless browsers
|
|
12
|
+
- The core functionality is edge-native (runs on Cloudflare Workers!!!)
|
|
13
|
+
|
|
14
|
+
## Demo
|
|
15
|
+
|
|
16
|
+
There is a demo API for Html2Markdown deployed on CloudflareWorker. Please access the following link
|
|
17
|
+
|
|
18
|
+
- [https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai](https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai)
|
|
19
|
+
- [https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E](https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E)
|
|
20
|
+
|
|
21
|
+
## Quick Start
|
|
22
|
+
|
|
23
|
+
Just install and execute scripts
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pnpm i webforai playwright
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```js
|
|
30
|
+
import { promises as fs } from "fs";
|
|
31
|
+
import { htmlToMarkdown, htmlToMdast } from "webforai";
|
|
32
|
+
import { loadHtml } from "webforai/loaders/playwright";
|
|
33
|
+
|
|
34
|
+
const url = "https://www.npmjs.com/package/webforai";
|
|
35
|
+
const html = await loadHtml(url);
|
|
36
|
+
|
|
37
|
+
const markdown = htmlToMarkdown(html, { solveLinks: url });
|
|
38
|
+
|
|
39
|
+
await fs.writeFile("output.md", markdown);
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
other examples are in [examples](./examples/src/index.ts)
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
### Main Functions
|
|
47
|
+
|
|
48
|
+
**`htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string`**
|
|
49
|
+
Convert HTML to Markdown. By default, unnecessary HTML is excluded and processed.
|
|
50
|
+
If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
51
|
+
This function just calls htmlToMdast and mdastToMarkdown in that order internally.
|
|
52
|
+
|
|
53
|
+
**`htmlToMdast(html: string, options?: HtmlToMdastOptions): Mdast`**
|
|
54
|
+
This project uses Hast and Mdast as defined by syntax-tree internally.
|
|
55
|
+
This function converts HTML to Mdast, an intermediate representation, which is required when using `mdastSplitter`, etc.
|
|
56
|
+
|
|
57
|
+
**`mdastToMarkdown(mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string`**
|
|
58
|
+
Convert Mdast to Markdown. If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
59
|
+
|
|
60
|
+
### Loader Functions
|
|
61
|
+
|
|
62
|
+
**`loadHtml(url: string, options?: LoadHtmlOptions): Promise<string>`**
|
|
63
|
+
Load HTML from the specified URL. This function uses Playwright internally.
|
|
64
|
+
|
|
65
|
+
## Early Users
|
|
66
|
+
|
|
67
|
+
### Voice Genius - Explanatory video generation by AI
|
|
68
|
+
|
|
69
|
+
<img src="https://github.com/inaridiy/webforai/blob/main/images/voice-genius.png" width="300px">
|
|
70
|
+
|
|
71
|
+
by [@moons14](https://twitter.com/moons_dev)
|
|
72
|
+
|
|
73
|
+
AI can be used to automatically generate highly accurate commentary videos.
|
|
74
|
+
Supports conversations and presentations using synthesized voice, embedding of images in videos, etc.
|
package/dist/extract-hast.js
CHANGED
|
@@ -1,23 +1,19 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const hast_util_select_1 = require("hast-util-select");
|
|
5
|
-
const unist_util_remove_1 = require("unist-util-remove");
|
|
6
|
-
const extractHast = (hast) => {
|
|
1
|
+
import { select, selectAll } from "hast-util-select";
|
|
2
|
+
import { remove } from "unist-util-remove";
|
|
3
|
+
export const extractHast = (hast) => {
|
|
7
4
|
const removeSelectors = ["header", "footer", "nav"];
|
|
8
5
|
for (const selector of removeSelectors) {
|
|
9
|
-
const matchedNodes =
|
|
6
|
+
const matchedNodes = selectAll(selector, hast);
|
|
10
7
|
for (const node of matchedNodes) {
|
|
11
|
-
|
|
8
|
+
remove(hast, { type: node.type, tagName: node.tagName });
|
|
12
9
|
}
|
|
13
10
|
}
|
|
14
11
|
const selectors = ["article", "div#article", "main", "div.container"];
|
|
15
12
|
for (const selector of selectors) {
|
|
16
|
-
const element =
|
|
13
|
+
const element = select(selector, hast);
|
|
17
14
|
if (element)
|
|
18
15
|
return element;
|
|
19
16
|
}
|
|
20
17
|
return hast;
|
|
21
18
|
};
|
|
22
|
-
|
|
23
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZXh0cmFjdC1oYXN0LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2V4dHJhY3QtaGFzdC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFDQSx1REFBcUQ7QUFFckQseURBQTJDO0FBRXBDLE1BQU0sV0FBVyxHQUFHLENBQUMsSUFBVSxFQUFFLEVBQUU7SUFDekMsTUFBTSxlQUFlLEdBQUcsQ0FBQyxRQUFRLEVBQUUsUUFBUSxFQUFFLEtBQUssQ0FBQyxDQUFDO0lBRXBELEtBQUssTUFBTSxRQUFRLElBQUksZUFBZSxFQUFFO1FBQ3ZDLE1BQU0sWUFBWSxHQUFHLElBQUEsNEJBQVMsRUFBQyxRQUFRLEVBQUUsSUFBSSxDQUFDLENBQUM7UUFDL0MsS0FBSyxNQUFNLElBQUksSUFBSSxZQUFZLEVBQUU7WUFDaEMsSUFBQSwwQkFBTSxFQUFDLElBQUksRUFBRSxFQUFFLElBQUksRUFBRSxJQUFJLENBQUMsSUFBSSxFQUFFLE9BQU8sRUFBRSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUMsQ0FBQztTQUN6RDtLQUNEO0lBRUQsTUFBTSxTQUFTLEdBQUcsQ0FBQyxTQUFTLEVBQUUsYUFBYSxFQUFFLE1BQU0sRUFBRSxlQUFlLENBQUMsQ0FBQztJQUN0RSxLQUFLLE1BQU0sUUFBUSxJQUFJLFNBQVMsRUFBRTtRQUNqQyxNQUFNLE9BQU8sR0FBRyxJQUFBLHlCQUFNLEVBQUMsUUFBUSxFQUFFLElBQUksQ0FBQyxDQUFDO1FBQ3ZDLElBQUksT0FBTztZQUFFLE9BQU8sT0FBTyxDQUFDO0tBQzVCO0lBRUQsT0FBTyxJQUFJLENBQUM7QUFDYixDQUFDLENBQUM7QUFqQlcsUUFBQSxXQUFXLGVBaUJ0QiJ9
|
|
19
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZXh0cmFjdC1oYXN0LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2V4dHJhY3QtaGFzdC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxPQUFPLEVBQUUsTUFBTSxFQUFFLFNBQVMsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRXJELE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUUzQyxNQUFNLENBQUMsTUFBTSxXQUFXLEdBQUcsQ0FBQyxJQUFVLEVBQUUsRUFBRTtJQUN6QyxNQUFNLGVBQWUsR0FBRyxDQUFDLFFBQVEsRUFBRSxRQUFRLEVBQUUsS0FBSyxDQUFDLENBQUM7SUFFcEQsS0FBSyxNQUFNLFFBQVEsSUFBSSxlQUFlLEVBQUU7UUFDdkMsTUFBTSxZQUFZLEdBQUcsU0FBUyxDQUFDLFFBQVEsRUFBRSxJQUFJLENBQUMsQ0FBQztRQUMvQyxLQUFLLE1BQU0sSUFBSSxJQUFJLFlBQVksRUFBRTtZQUNoQyxNQUFNLENBQUMsSUFBSSxFQUFFLEVBQUUsSUFBSSxFQUFFLElBQUksQ0FBQyxJQUFJLEVBQUUsT0FBTyxFQUFFLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUFDO1NBQ3pEO0tBQ0Q7SUFFRCxNQUFNLFNBQVMsR0FBRyxDQUFDLFNBQVMsRUFBRSxhQUFhLEVBQUUsTUFBTSxFQUFFLGVBQWUsQ0FBQyxDQUFDO0lBQ3RFLEtBQUssTUFBTSxRQUFRLElBQUksU0FBUyxFQUFFO1FBQ2pDLE1BQU0sT0FBTyxHQUFHLE1BQU0sQ0FBQyxRQUFRLEVBQUUsSUFBSSxDQUFDLENBQUM7UUFDdkMsSUFBSSxPQUFPO1lBQUUsT0FBTyxPQUFPLENBQUM7S0FDNUI7SUFFRCxPQUFPLElBQUksQ0FBQztBQUNiLENBQUMsQ0FBQyJ9
|
package/dist/html-to-markdown.js
CHANGED
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const htmlToMarkdown = (html, options) => {
|
|
7
|
-
const mdast = (0, html_to_mdast_1.htmlToMdast)(html, options);
|
|
8
|
-
const markdown = (0, mdast_to_markdown_1.mdastToMarkdown)(mdast, options);
|
|
1
|
+
import { htmlToMdast } from "./html-to-mdast";
|
|
2
|
+
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
3
|
+
export const htmlToMarkdown = (html, options) => {
|
|
4
|
+
const mdast = htmlToMdast(html, options);
|
|
5
|
+
const markdown = mdastToMarkdown(mdast, options);
|
|
9
6
|
return markdown;
|
|
10
7
|
};
|
|
11
|
-
|
|
12
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHRtbC10by1tYXJrZG93bi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9odG1sLXRvLW1hcmtkb3duLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUNBLG1EQUE4QztBQUM5QywyREFBc0Q7QUFPL0MsTUFBTSxjQUFjLEdBQUcsQ0FBQyxJQUFZLEVBQUUsT0FBK0IsRUFBVSxFQUFFO0lBQ3ZGLE1BQU0sS0FBSyxHQUFHLElBQUEsMkJBQVcsRUFBQyxJQUFJLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFDekMsTUFBTSxRQUFRLEdBQUcsSUFBQSxtQ0FBZSxFQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsQ0FBQztJQUNqRCxPQUFPLFFBQVEsQ0FBQztBQUNqQixDQUFDLENBQUM7QUFKVyxRQUFBLGNBQWMsa0JBSXpCIn0=
|
|
8
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHRtbC10by1tYXJrZG93bi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9odG1sLXRvLW1hcmtkb3duLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUM5QyxPQUFPLEVBQUUsZUFBZSxFQUFFLE1BQU0scUJBQXFCLENBQUM7QUFPdEQsTUFBTSxDQUFDLE1BQU0sY0FBYyxHQUFHLENBQUMsSUFBWSxFQUFFLE9BQStCLEVBQVUsRUFBRTtJQUN2RixNQUFNLEtBQUssR0FBRyxXQUFXLENBQUMsSUFBSSxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQ3pDLE1BQU0sUUFBUSxHQUFHLGVBQWUsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFDakQsT0FBTyxRQUFRLENBQUM7QUFDakIsQ0FBQyxDQUFDIn0=
|
package/dist/html-to-mdast.js
CHANGED
|
@@ -1,18 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
const
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
const extractedHast = options?.extractHast ? options.extractHast(hast) : (0, extract_hast_1.extractHast)(hast);
|
|
12
|
-
const mdast = (0, hast_util_to_mdast_1.toMdast)(extractedHast, {
|
|
13
|
-
handlers: { div: custom_div_handler_1.customDivHandler, math: math_handler_1.mathHandler, br: () => { } },
|
|
1
|
+
import { fromHtml } from "hast-util-from-html";
|
|
2
|
+
import { toMdast } from "hast-util-to-mdast";
|
|
3
|
+
import { extractHast } from "./extract-hast";
|
|
4
|
+
import { customDivHandler } from "./mdast-handlers/custom-div-handler";
|
|
5
|
+
import { mathHandler } from "./mdast-handlers/math-handler";
|
|
6
|
+
export const htmlToMdast = (html, options) => {
|
|
7
|
+
const hast = fromHtml(html, { fragment: true });
|
|
8
|
+
const extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);
|
|
9
|
+
const mdast = toMdast(extractedHast, {
|
|
10
|
+
handlers: { div: customDivHandler, math: mathHandler, br: () => { } },
|
|
14
11
|
});
|
|
15
12
|
return mdast;
|
|
16
13
|
};
|
|
17
|
-
|
|
18
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHRtbC10by1tZGFzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9odG1sLXRvLW1kYXN0LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUdBLDZEQUErQztBQUMvQywyREFBNkM7QUFFN0MsaURBQTZDO0FBQzdDLDRFQUF1RTtBQUN2RSxnRUFBNEQ7QUFNckQsTUFBTSxXQUFXLEdBQUcsQ0FBQyxJQUFZLEVBQUUsT0FBNEIsRUFBUyxFQUFFO0lBQ2hGLE1BQU0sSUFBSSxHQUFHLElBQUEsOEJBQVEsRUFBQyxJQUFJLEVBQUUsRUFBRSxRQUFRLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztJQUNoRCxNQUFNLGFBQWEsR0FBRyxPQUFPLEVBQUUsV0FBVyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFBLDBCQUFXLEVBQUMsSUFBSSxDQUFDLENBQUM7SUFDM0YsTUFBTSxLQUFLLEdBQUcsSUFBQSw0QkFBTyxFQUFDLGFBQWEsRUFBRTtRQUNwQyxRQUFRLEVBQUUsRUFBRSxHQUFHLEVBQUUscUNBQWdCLEVBQUUsSUFBSSxFQUFFLDBCQUFXLEVBQUUsRUFBRSxFQUFFLEdBQUcsRUFBRSxHQUFFLENBQUMsRUFBRTtLQUNwRSxDQUFDLENBQUM7SUFFSCxPQUFPLEtBQUssQ0FBQztBQUNkLENBQUMsQ0FBQztBQVJXLFFBQUEsV0FBVyxlQVF0QiJ9
|
|
14
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHRtbC10by1tZGFzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9odG1sLXRvLW1kYXN0LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUdBLE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUMvQyxPQUFPLEVBQUUsT0FBTyxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFFN0MsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBQzdDLE9BQU8sRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHFDQUFxQyxDQUFDO0FBQ3ZFLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQU01RCxNQUFNLENBQUMsTUFBTSxXQUFXLEdBQUcsQ0FBQyxJQUFZLEVBQUUsT0FBNEIsRUFBUyxFQUFFO0lBQ2hGLE1BQU0sSUFBSSxHQUFHLFFBQVEsQ0FBQyxJQUFJLEVBQUUsRUFBRSxRQUFRLEVBQUUsSUFBSSxFQUFFLENBQUMsQ0FBQztJQUNoRCxNQUFNLGFBQWEsR0FBRyxPQUFPLEVBQUUsV0FBVyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxXQUFXLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDM0YsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLGFBQWEsRUFBRTtRQUNwQyxRQUFRLEVBQUUsRUFBRSxHQUFHLEVBQUUsZ0JBQWdCLEVBQUUsSUFBSSxFQUFFLFdBQVcsRUFBRSxFQUFFLEVBQUUsR0FBRyxFQUFFLEdBQUUsQ0FBQyxFQUFFO0tBQ3BFLENBQUMsQ0FBQztJQUVILE9BQU8sS0FBSyxDQUFDO0FBQ2QsQ0FBQyxDQUFDIn0=
|
package/dist/index.js
CHANGED
|
@@ -1,21 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
-
};
|
|
16
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
__exportStar(require("./html-to-markdown"), exports);
|
|
18
|
-
__exportStar(require("./md-splitter"), exports);
|
|
19
|
-
__exportStar(require("./html-to-mdast"), exports);
|
|
20
|
-
__exportStar(require("./mdast-to-markdown"), exports);
|
|
21
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7Ozs7Ozs7Ozs7OztBQUFBLHFEQUFtQztBQUNuQyxnREFBOEI7QUFDOUIsa0RBQWdDO0FBQ2hDLHNEQUFvQyJ9
|
|
1
|
+
export * from "./html-to-markdown";
|
|
2
|
+
export * from "./md-splitter";
|
|
3
|
+
export * from "./html-to-mdast";
|
|
4
|
+
export * from "./mdast-to-markdown";
|
|
5
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvaW5kZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FBYyxvQkFBb0IsQ0FBQztBQUNuQyxjQUFjLGVBQWUsQ0FBQztBQUM5QixjQUFjLGlCQUFpQixDQUFDO0FBQ2hDLGNBQWMscUJBQXFCLENBQUMifQ==
|
package/dist/link-replacer.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.linkReplacer = void 0;
|
|
4
|
-
const linkReplacer = (markdown, link) => {
|
|
1
|
+
export const linkReplacer = (markdown, link) => {
|
|
5
2
|
const regex = /(!?\[.*?\]\()(.+?)(\))/g;
|
|
6
3
|
return markdown.replace(regex, (match, pre, thisUrl, post) => {
|
|
7
4
|
if (thisUrl.startsWith("http") || thisUrl.startsWith("#"))
|
|
@@ -9,5 +6,4 @@ const linkReplacer = (markdown, link) => {
|
|
|
9
6
|
return `${pre}${new URL(thisUrl, link).href}${post}`;
|
|
10
7
|
});
|
|
11
8
|
};
|
|
12
|
-
|
|
13
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibGluay1yZXBsYWNlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9saW5rLXJlcGxhY2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUFPLE1BQU0sWUFBWSxHQUFHLENBQUMsUUFBZ0IsRUFBRSxJQUFZLEVBQUUsRUFBRTtJQUM5RCxNQUFNLEtBQUssR0FBRyx5QkFBeUIsQ0FBQztJQUN4QyxPQUFPLFFBQVEsQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUMsS0FBSyxFQUFFLEdBQUcsRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLEVBQUU7UUFDNUQsSUFBSSxPQUFPLENBQUMsVUFBVSxDQUFDLE1BQU0sQ0FBQyxJQUFJLE9BQU8sQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDO1lBQUUsT0FBTyxLQUFLLENBQUM7UUFDeEUsT0FBTyxHQUFHLEdBQUcsR0FBRyxJQUFJLEdBQUcsQ0FBQyxPQUFPLEVBQUUsSUFBSSxDQUFDLENBQUMsSUFBSSxHQUFHLElBQUksRUFBRSxDQUFDO0lBQ3RELENBQUMsQ0FBQyxDQUFDO0FBQ0osQ0FBQyxDQUFDO0FBTlcsUUFBQSxZQUFZLGdCQU12QiJ9
|
|
9
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibGluay1yZXBsYWNlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9saW5rLXJlcGxhY2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE1BQU0sQ0FBQyxNQUFNLFlBQVksR0FBRyxDQUFDLFFBQWdCLEVBQUUsSUFBWSxFQUFFLEVBQUU7SUFDOUQsTUFBTSxLQUFLLEdBQUcseUJBQXlCLENBQUM7SUFDeEMsT0FBTyxRQUFRLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDLEtBQUssRUFBRSxHQUFHLEVBQUUsT0FBTyxFQUFFLElBQUksRUFBRSxFQUFFO1FBQzVELElBQUksT0FBTyxDQUFDLFVBQVUsQ0FBQyxNQUFNLENBQUMsSUFBSSxPQUFPLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQztZQUFFLE9BQU8sS0FBSyxDQUFDO1FBQ3hFLE9BQU8sR0FBRyxHQUFHLEdBQUcsSUFBSSxHQUFHLENBQUMsT0FBTyxFQUFFLElBQUksQ0FBQyxDQUFDLElBQUksR0FBRyxJQUFJLEVBQUUsQ0FBQztJQUN0RCxDQUFDLENBQUMsQ0FBQztBQUNKLENBQUMsQ0FBQyJ9
|
|
@@ -1,16 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.loadHtml = void 0;
|
|
7
|
-
const playwright_1 = __importDefault(require("playwright"));
|
|
1
|
+
import playwright from "playwright";
|
|
8
2
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
9
|
-
const loadHtml = async (url, context) => {
|
|
3
|
+
export const loadHtml = async (url, context) => {
|
|
10
4
|
let _browser = null;
|
|
11
5
|
let _context;
|
|
12
6
|
if (!context) {
|
|
13
|
-
_browser = await
|
|
7
|
+
_browser = await playwright.chromium.launch();
|
|
14
8
|
_context = await _browser.newContext();
|
|
15
9
|
}
|
|
16
10
|
else {
|
|
@@ -25,5 +19,4 @@ const loadHtml = async (url, context) => {
|
|
|
25
19
|
await _browser.close();
|
|
26
20
|
return html;
|
|
27
21
|
};
|
|
28
|
-
|
|
29
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGxheXdyaWdodC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9sb2FkZXJzL3BsYXl3cmlnaHQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7O0FBQUEsNERBQW9DO0FBRXBDLE1BQU0sS0FBSyxHQUFHLENBQUMsRUFBVSxFQUFFLEVBQUUsQ0FBQyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsVUFBVSxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDO0FBRXpFLE1BQU0sUUFBUSxHQUFHLEtBQUssRUFBRSxHQUFXLEVBQUUsT0FBbUMsRUFBRSxFQUFFO0lBQ2xGLElBQUksUUFBUSxHQUE4QixJQUFJLENBQUM7SUFDL0MsSUFBSSxRQUFtQyxDQUFDO0lBQ3hDLElBQUksQ0FBQyxPQUFPLEVBQUU7UUFDYixRQUFRLEdBQUcsTUFBTSxvQkFBVSxDQUFDLFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUM5QyxRQUFRLEdBQUcsTUFBTSxRQUFRLENBQUMsVUFBVSxFQUFFLENBQUM7S0FDdkM7U0FBTTtRQUNOLFFBQVEsR0FBRyxPQUFPLENBQUM7S0FDbkI7SUFFRCxNQUFNLElBQUksR0FBRyxNQUFNLFFBQVEsQ0FBQyxPQUFPLEVBQUUsQ0FBQztJQUN0QyxNQUFNLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDckIsTUFBTSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxDQUFDLGdCQUFnQixDQUFDLGFBQWEsQ0FBQyxFQUFFLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDekUsTUFBTSxJQUFJLEdBQUcsTUFBTSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7SUFDbEMsTUFBTSxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDbkIsSUFBSSxRQUFRO1FBQUUsTUFBTSxRQUFRLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDckMsT0FBTyxJQUFJLENBQUM7QUFDYixDQUFDLENBQUM7QUFqQlcsUUFBQSxRQUFRLFlBaUJuQiJ9
|
|
22
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGxheXdyaWdodC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9sb2FkZXJzL3BsYXl3cmlnaHQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxVQUFVLE1BQU0sWUFBWSxDQUFDO0FBRXBDLE1BQU0sS0FBSyxHQUFHLENBQUMsRUFBVSxFQUFFLEVBQUUsQ0FBQyxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsVUFBVSxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDO0FBRWhGLE1BQU0sQ0FBQyxNQUFNLFFBQVEsR0FBRyxLQUFLLEVBQUUsR0FBVyxFQUFFLE9BQW1DLEVBQUUsRUFBRTtJQUNsRixJQUFJLFFBQVEsR0FBOEIsSUFBSSxDQUFDO0lBQy9DLElBQUksUUFBbUMsQ0FBQztJQUN4QyxJQUFJLENBQUMsT0FBTyxFQUFFO1FBQ2IsUUFBUSxHQUFHLE1BQU0sVUFBVSxDQUFDLFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQztRQUM5QyxRQUFRLEdBQUcsTUFBTSxRQUFRLENBQUMsVUFBVSxFQUFFLENBQUM7S0FDdkM7U0FBTTtRQUNOLFFBQVEsR0FBRyxPQUFPLENBQUM7S0FDbkI7SUFFRCxNQUFNLElBQUksR0FBRyxNQUFNLFFBQVEsQ0FBQyxPQUFPLEVBQUUsQ0FBQztJQUN0QyxNQUFNLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDckIsTUFBTSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxDQUFDLGdCQUFnQixDQUFDLGFBQWEsQ0FBQyxFQUFFLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7SUFDekUsTUFBTSxJQUFJLEdBQUcsTUFBTSxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7SUFDbEMsTUFBTSxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDbkIsSUFBSSxRQUFRO1FBQUUsTUFBTSxRQUFRLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDckMsT0FBTyxJQUFJLENBQUM7QUFDYixDQUFDLENBQUMifQ==
|
package/dist/md-splitter.js
CHANGED
|
@@ -1,34 +1,30 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const mdast_to_markdown_1 = require("./mdast-to-markdown");
|
|
5
|
-
const common_1 = require("./utils/common");
|
|
6
|
-
const mdast_utils_1 = require("./utils/mdast-utils");
|
|
1
|
+
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
2
|
+
import { chunk } from "./utils/common";
|
|
3
|
+
import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
|
|
7
4
|
const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
|
|
8
5
|
const getSplitterGenerator = function* () {
|
|
9
6
|
for (const splitter of PRIORITY_SPLITTERS)
|
|
10
7
|
yield splitter;
|
|
11
8
|
};
|
|
12
9
|
const _mdastSplitter = async (contents, checker, splitterGenerator) => {
|
|
13
|
-
const markdown =
|
|
10
|
+
const markdown = mdastToMarkdown(warpRoot(contents));
|
|
14
11
|
if ((await checker(markdown)) || 3 > contents.length)
|
|
15
12
|
return [contents];
|
|
16
13
|
const splitter = splitterGenerator.next().value;
|
|
17
14
|
const chunked = splitter
|
|
18
15
|
? contents.reduce((acc, content) => {
|
|
19
|
-
if (
|
|
16
|
+
if (internalType(content) === splitter || acc.length === 0) {
|
|
20
17
|
acc.push([content]);
|
|
21
18
|
return acc;
|
|
22
19
|
}
|
|
23
20
|
acc[acc.length - 1].push(content);
|
|
24
21
|
return acc;
|
|
25
22
|
}, [])
|
|
26
|
-
:
|
|
23
|
+
: chunk(contents, 3);
|
|
27
24
|
const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
|
|
28
25
|
return Promise.all(splitting).then((chunks) => chunks.flat());
|
|
29
26
|
};
|
|
30
|
-
const mdastSplitter = async (mdast, checker, options) => {
|
|
31
|
-
return _mdastSplitter(
|
|
27
|
+
export const mdastSplitter = async (mdast, checker, options) => {
|
|
28
|
+
return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
|
|
32
29
|
};
|
|
33
|
-
|
|
34
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWQtc3BsaXR0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWQtc3BsaXR0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQ0EsMkRBQXNEO0FBQ3RELDJDQUF1QztBQUN2QyxxREFBeUU7QUFFekUsTUFBTSxrQkFBa0IsR0FBRyxDQUFDLElBQUksRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUM7QUFFekYsTUFBTSxvQkFBb0IsR0FBRyxRQUFRLENBQUM7SUFDckMsS0FBSyxNQUFNLFFBQVEsSUFBSSxrQkFBa0I7UUFBRSxNQUFNLFFBQVEsQ0FBQztBQUMzRCxDQUFDLENBQUM7QUFFRixNQUFNLGNBQWMsR0FBRyxLQUFLLEVBQzNCLFFBQXVCLEVBQ3ZCLE9BQStDLEVBQy9DLGlCQUFvQyxFQUNULEVBQUU7SUFDN0IsTUFBTSxRQUFRLEdBQUcsSUFBQSxtQ0FBZSxFQUFDLElBQUEsc0JBQVEsRUFBQyxRQUFRLENBQUMsQ0FBQyxDQUFDO0lBQ3JELElBQUksQ0FBQyxNQUFNLE9BQU8sQ0FBQyxRQUFRLENBQUMsQ0FBQyxJQUFJLENBQUMsR0FBRyxRQUFRLENBQUMsTUFBTTtRQUFFLE9BQU8sQ0FBQyxRQUFRLENBQUMsQ0FBQztJQUN4RSxNQUFNLFFBQVEsR0FBRyxpQkFBaUIsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxLQUFLLENBQUM7SUFFaEQsTUFBTSxPQUFPLEdBQUcsUUFBUTtRQUN2QixDQUFDLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBa0IsQ0FBQyxHQUFHLEVBQUUsT0FBTyxFQUFFLEVBQUU7WUFDbEQsSUFBSSxJQUFBLDBCQUFZLEVBQUMsT0FBTyxDQUFDLEtBQUssUUFBUSxJQUFJLEdBQUcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFO2dCQUMzRCxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQztnQkFDcEIsT0FBTyxHQUFHLENBQUM7YUFDWDtZQUNELEdBQUcsQ0FBQyxHQUFHLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUNsQyxPQUFPLEdBQUcsQ0FBQztRQUNYLENBQUMsRUFBRSxFQUFFLENBQUM7UUFDUixDQUFDLENBQUMsSUFBQSxjQUFLLEVBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxDQUFDO0lBRXRCLE1BQU0sU0FBUyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLGNBQWMsQ0FBQyxLQUFLLEVBQUUsT0FBTyxFQUFFLGlCQUFpQixDQUFDLENBQUMsQ0FBQztJQUU1RixPQUFPLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBUyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBTSxFQUFFLEVBQUUsQ0FBQyxNQUFNLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztBQUMvRCxDQUFDLENBQUM7QUFFSyxNQUFNLGFBQWEsR0FBRyxLQUFLLEVBQ2pDLEtBQVksRUFDWixPQUErQyxFQUMvQyxPQUFrQyxFQUNQLEVBQUU7SUFDN0IsT0FBTyxjQUFjLENBQUMsSUFBQSx3QkFBVSxFQUFDLEtBQUssQ0FBQyxFQUFFLE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxDQUFDLENBQUM7QUFDM0UsQ0FBQyxDQUFDO0FBTlcsUUFBQSxhQUFhLGlCQU14QiJ9
|
|
30
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWQtc3BsaXR0ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWQtc3BsaXR0ZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUFFLGVBQWUsRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBQ3RELE9BQU8sRUFBRSxLQUFLLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUN2QyxPQUFPLEVBQUUsWUFBWSxFQUFFLFVBQVUsRUFBRSxRQUFRLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUV6RSxNQUFNLGtCQUFrQixHQUFHLENBQUMsSUFBSSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLE9BQU8sRUFBRSxNQUFNLENBQUMsQ0FBQztBQUV6RixNQUFNLG9CQUFvQixHQUFHLFFBQVEsQ0FBQztJQUNyQyxLQUFLLE1BQU0sUUFBUSxJQUFJLGtCQUFrQjtRQUFFLE1BQU0sUUFBUSxDQUFDO0FBQzNELENBQUMsQ0FBQztBQUVGLE1BQU0sY0FBYyxHQUFHLEtBQUssRUFDM0IsUUFBdUIsRUFDdkIsT0FBK0MsRUFDL0MsaUJBQW9DLEVBQ1QsRUFBRTtJQUM3QixNQUFNLFFBQVEsR0FBRyxlQUFlLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxDQUFDLENBQUM7SUFDckQsSUFBSSxDQUFDLE1BQU0sT0FBTyxDQUFDLFFBQVEsQ0FBQyxDQUFDLElBQUksQ0FBQyxHQUFHLFFBQVEsQ0FBQyxNQUFNO1FBQUUsT0FBTyxDQUFDLFFBQVEsQ0FBQyxDQUFDO0lBQ3hFLE1BQU0sUUFBUSxHQUFHLGlCQUFpQixDQUFDLElBQUksRUFBRSxDQUFDLEtBQUssQ0FBQztJQUVoRCxNQUFNLE9BQU8sR0FBRyxRQUFRO1FBQ3ZCLENBQUMsQ0FBQyxRQUFRLENBQUMsTUFBTSxDQUFrQixDQUFDLEdBQUcsRUFBRSxPQUFPLEVBQUUsRUFBRTtZQUNsRCxJQUFJLFlBQVksQ0FBQyxPQUFPLENBQUMsS0FBSyxRQUFRLElBQUksR0FBRyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUU7Z0JBQzNELEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDO2dCQUNwQixPQUFPLEdBQUcsQ0FBQzthQUNYO1lBQ0QsR0FBRyxDQUFDLEdBQUcsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDO1lBQ2xDLE9BQU8sR0FBRyxDQUFDO1FBQ1gsQ0FBQyxFQUFFLEVBQUUsQ0FBQztRQUNSLENBQUMsQ0FBQyxLQUFLLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxDQUFDO0lBRXRCLE1BQU0sU0FBUyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLGNBQWMsQ0FBQyxLQUFLLEVBQUUsT0FBTyxFQUFFLGlCQUFpQixDQUFDLENBQUMsQ0FBQztJQUU1RixPQUFPLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBUyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBTSxFQUFFLEVBQUUsQ0FBQyxNQUFNLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztBQUMvRCxDQUFDLENBQUM7QUFFRixNQUFNLENBQUMsTUFBTSxhQUFhLEdBQUcsS0FBSyxFQUNqQyxLQUFZLEVBQ1osT0FBK0MsRUFDL0MsT0FBa0MsRUFDUCxFQUFFO0lBQzdCLE9BQU8sY0FBYyxDQUFDLFVBQVUsQ0FBQyxLQUFLLENBQUMsRUFBRSxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsQ0FBQyxDQUFDO0FBQzNFLENBQUMsQ0FBQyJ9
|
|
@@ -1,10 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const
|
|
5
|
-
const hast_util_to_text_1 = require("hast-util-to-text");
|
|
6
|
-
const trim_trailing_lines_1 = require("trim-trailing-lines");
|
|
7
|
-
const customDivHandler = (state, node) => {
|
|
1
|
+
import { defaultHandlers } from "hast-util-to-mdast";
|
|
2
|
+
import { toText } from "hast-util-to-text";
|
|
3
|
+
import { trimTrailingLines } from "trim-trailing-lines";
|
|
4
|
+
export const customDivHandler = (state, node) => {
|
|
8
5
|
const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
|
|
9
6
|
let highlightLanguage;
|
|
10
7
|
for (const className of classNames) {
|
|
@@ -21,10 +18,10 @@ const customDivHandler = (state, node) => {
|
|
|
21
18
|
break;
|
|
22
19
|
}
|
|
23
20
|
else if (String(className).includes("code-block_wrapper")) {
|
|
24
|
-
const text =
|
|
21
|
+
const text = toText(node);
|
|
25
22
|
if (!text.includes("."))
|
|
26
23
|
break;
|
|
27
|
-
const lang =
|
|
24
|
+
const lang = toText(node).split("\n")[0].split(".").slice(-1)[0];
|
|
28
25
|
highlightLanguage = lang;
|
|
29
26
|
break;
|
|
30
27
|
}
|
|
@@ -35,12 +32,11 @@ const customDivHandler = (state, node) => {
|
|
|
35
32
|
type: "code",
|
|
36
33
|
lang: highlightLanguage,
|
|
37
34
|
meta: null,
|
|
38
|
-
value:
|
|
35
|
+
value: trimTrailingLines(toText(code)).trim(),
|
|
39
36
|
};
|
|
40
37
|
state.patch(node, result);
|
|
41
38
|
return result;
|
|
42
39
|
}
|
|
43
|
-
return
|
|
40
|
+
return defaultHandlers.div(state, node);
|
|
44
41
|
};
|
|
45
|
-
|
|
46
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3VzdG9tLWRpdi1oYW5kbGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL21kYXN0LWhhbmRsZXJzL2N1c3RvbS1kaXYtaGFuZGxlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFBQSwyREFBNkQ7QUFDN0QseURBQTJDO0FBRTNDLDZEQUF3RDtBQUVqRCxNQUFNLGdCQUFnQixHQUFXLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO0lBQ3RELE1BQU0sVUFBVSxHQUFHLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUU3RixJQUFJLGlCQUFxQyxDQUFDO0lBQzFDLEtBQUssTUFBTSxTQUFTLElBQUksVUFBVSxFQUFFO1FBQ2xDLElBQUksTUFBTSxDQUFDLFNBQVMsQ0FBQyxLQUFLLDJCQUEyQixFQUFFO1lBQ3JELGlCQUFpQixHQUFHLE1BQU0sQ0FBQztZQUMzQixNQUFNO1NBQ1A7YUFBTSxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQyxRQUFRLENBQUMsbUJBQW1CLENBQUMsRUFBRTtZQUMxRCxpQkFBaUIsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUMsS0FBSyxDQUFDLG1CQUFtQixDQUFDLE1BQU0sQ0FBQyxDQUFDO1lBQ3hFLE1BQU07U0FDUDthQUFNLElBQUksTUFBTSxDQUFDLFNBQVMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxXQUFXLENBQUMsRUFBRTtZQUNsRCxpQkFBaUIsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUMsS0FBSyxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUNoRSxNQUFNO1NBQ1A7YUFBTSxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQyxRQUFRLENBQUMsb0JBQW9CLENBQUMsRUFBRTtZQUMzRCxNQUFNLElBQUksR0FBRyxJQUFBLDBCQUFNLEVBQUMsSUFBSSxDQUFDLENBQUM7WUFDMUIsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDO2dCQUFFLE1BQU07WUFDL0IsTUFBTSxJQUFJLEdBQUcsSUFBQSwwQkFBTSxFQUFDLElBQUksQ0FBQyxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDakUsaUJBQWlCLEdBQUcsSUFBSSxDQUFDO1lBQ3pCLE1BQU07U0FDUDtLQUNGO0lBRUQsSUFBSSxpQkFBaUIsS0FBSyxTQUFTLEVBQUU7UUFDbkMsTUFBTSxJQUFJLEdBQ1IsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLEtBQUssQ0FBQyxJQUFJLEtBQUssU0FBUyxJQUFJLEtBQUssQ0FBQyxPQUFPLEtBQUssS0FBSyxDQUFDLElBQUksSUFBSSxDQUFDO1FBRTdGLE1BQU0sTUFBTSxHQUFTO1lBQ25CLElBQUksRUFBRSxNQUFNO1lBQ1osSUFBSSxFQUFFLGlCQUFpQjtZQUN2QixJQUFJLEVBQUUsSUFBSTtZQUNWLEtBQUssRUFBRSxJQUFBLHVDQUFpQixFQUFDLElBQUEsMEJBQU0sRUFBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLElBQUksRUFBRTtTQUM5QyxDQUFDO1FBQ0YsS0FBSyxDQUFDLEtBQUssQ0FBQyxJQUFJLEVBQUUsTUFBTSxDQUFDLENBQUM7UUFDMUIsT0FBTyxNQUFNLENBQUM7S0FDZjtJQUVELE9BQU8sb0NBQWUsQ0FBQyxHQUFHLENBQUMsS0FBSyxFQUFFLElBQUksQ0FBQyxDQUFDO0FBQzFDLENBQUMsQ0FBQztBQXRDVyxRQUFBLGdCQUFnQixvQkFzQzNCIn0=
|
|
42
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3VzdG9tLWRpdi1oYW5kbGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL21kYXN0LWhhbmRsZXJzL2N1c3RvbS1kaXYtaGFuZGxlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQVUsZUFBZSxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDN0QsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBRTNDLE9BQU8sRUFBRSxpQkFBaUIsRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBRXhELE1BQU0sQ0FBQyxNQUFNLGdCQUFnQixHQUFXLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO0lBQ3RELE1BQU0sVUFBVSxHQUFHLEtBQUssQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUU3RixJQUFJLGlCQUFxQyxDQUFDO0lBQzFDLEtBQUssTUFBTSxTQUFTLElBQUksVUFBVSxFQUFFO1FBQ2xDLElBQUksTUFBTSxDQUFDLFNBQVMsQ0FBQyxLQUFLLDJCQUEyQixFQUFFO1lBQ3JELGlCQUFpQixHQUFHLE1BQU0sQ0FBQztZQUMzQixNQUFNO1NBQ1A7YUFBTSxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQyxRQUFRLENBQUMsbUJBQW1CLENBQUMsRUFBRTtZQUMxRCxpQkFBaUIsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUMsS0FBSyxDQUFDLG1CQUFtQixDQUFDLE1BQU0sQ0FBQyxDQUFDO1lBQ3hFLE1BQU07U0FDUDthQUFNLElBQUksTUFBTSxDQUFDLFNBQVMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxXQUFXLENBQUMsRUFBRTtZQUNsRCxpQkFBaUIsR0FBRyxNQUFNLENBQUMsU0FBUyxDQUFDLENBQUMsS0FBSyxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUNoRSxNQUFNO1NBQ1A7YUFBTSxJQUFJLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQyxRQUFRLENBQUMsb0JBQW9CLENBQUMsRUFBRTtZQUMzRCxNQUFNLElBQUksR0FBRyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUM7WUFDMUIsSUFBSSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDO2dCQUFFLE1BQU07WUFDL0IsTUFBTSxJQUFJLEdBQUcsTUFBTSxDQUFDLElBQUksQ0FBQyxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDakUsaUJBQWlCLEdBQUcsSUFBSSxDQUFDO1lBQ3pCLE1BQU07U0FDUDtLQUNGO0lBRUQsSUFBSSxpQkFBaUIsS0FBSyxTQUFTLEVBQUU7UUFDbkMsTUFBTSxJQUFJLEdBQ1IsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDLEtBQUssQ0FBQyxJQUFJLEtBQUssU0FBUyxJQUFJLEtBQUssQ0FBQyxPQUFPLEtBQUssS0FBSyxDQUFDLElBQUksSUFBSSxDQUFDO1FBRTdGLE1BQU0sTUFBTSxHQUFTO1lBQ25CLElBQUksRUFBRSxNQUFNO1lBQ1osSUFBSSxFQUFFLGlCQUFpQjtZQUN2QixJQUFJLEVBQUUsSUFBSTtZQUNWLEtBQUssRUFBRSxpQkFBaUIsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxJQUFJLEVBQUU7U0FDOUMsQ0FBQztRQUNGLEtBQUssQ0FBQyxLQUFLLENBQUMsSUFBSSxFQUFFLE1BQU0sQ0FBQyxDQUFDO1FBQzFCLE9BQU8sTUFBTSxDQUFDO0tBQ2Y7SUFFRCxPQUFPLGVBQWUsQ0FBQyxHQUFHLENBQUMsS0FBSyxFQUFFLElBQUksQ0FBQyxDQUFDO0FBQzFDLENBQUMsQ0FBQyJ9
|
|
@@ -1,14 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const mathHandler = (state, node) => {
|
|
7
|
-
const mathMl = (0, hast_util_to_html_1.toHtml)(node);
|
|
8
|
-
const latex = mathml_to_latex_1.MathMLToLaTeX.convert(mathMl);
|
|
1
|
+
import { toHtml } from "hast-util-to-html";
|
|
2
|
+
import { MathMLToLaTeX } from "mathml-to-latex";
|
|
3
|
+
export const mathHandler = (state, node) => {
|
|
4
|
+
const mathMl = toHtml(node);
|
|
5
|
+
const latex = MathMLToLaTeX.convert(mathMl);
|
|
9
6
|
const result = { type: "inlineMath", value: latex };
|
|
10
7
|
state.patch(node, result);
|
|
11
8
|
return result;
|
|
12
9
|
};
|
|
13
|
-
|
|
14
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWF0aC1oYW5kbGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL21kYXN0LWhhbmRsZXJzL21hdGgtaGFuZGxlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFBQSx5REFBMkM7QUFFM0MscURBQWdEO0FBR3pDLE1BQU0sV0FBVyxHQUFXLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO0lBQ2pELE1BQU0sTUFBTSxHQUFHLElBQUEsMEJBQU0sRUFBQyxJQUFJLENBQUMsQ0FBQztJQUM1QixNQUFNLEtBQUssR0FBRywrQkFBYSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUM1QyxNQUFNLE1BQU0sR0FBc0IsRUFBRSxJQUFJLEVBQUUsWUFBWSxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsQ0FBQztJQUN2RSxLQUFLLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxNQUFNLENBQUMsQ0FBQztJQUMxQixPQUFPLE1BQU0sQ0FBQztBQUNoQixDQUFDLENBQUM7QUFOVyxRQUFBLFdBQVcsZUFNdEIifQ==
|
|
10
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWF0aC1oYW5kbGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL21kYXN0LWhhbmRsZXJzL21hdGgtaGFuZGxlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsTUFBTSxFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFFM0MsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLGlCQUFpQixDQUFDO0FBR2hELE1BQU0sQ0FBQyxNQUFNLFdBQVcsR0FBVyxDQUFDLEtBQUssRUFBRSxJQUFJLEVBQUUsRUFBRTtJQUNqRCxNQUFNLE1BQU0sR0FBRyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDNUIsTUFBTSxLQUFLLEdBQUcsYUFBYSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUM1QyxNQUFNLE1BQU0sR0FBc0IsRUFBRSxJQUFJLEVBQUUsWUFBWSxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsQ0FBQztJQUN2RSxLQUFLLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxNQUFNLENBQUMsQ0FBQztJQUMxQixPQUFPLE1BQU0sQ0FBQztBQUNoQixDQUFDLENBQUMifQ==
|
|
@@ -1,18 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const mdastToMarkdown = (mdast, options) => {
|
|
10
|
-
let markdown = (0, mdast_util_to_markdown_1.toMarkdown)((0, mdast_utils_1.warpRoot)(mdast), {
|
|
11
|
-
extensions: [(0, mdast_util_gfm_1.gfmToMarkdown)(), (0, mdast_util_math_1.mathToMarkdown)()],
|
|
1
|
+
import { gfmToMarkdown } from "mdast-util-gfm";
|
|
2
|
+
import { mathToMarkdown } from "mdast-util-math";
|
|
3
|
+
import { toMarkdown } from "mdast-util-to-markdown";
|
|
4
|
+
import { linkReplacer } from "./link-replacer";
|
|
5
|
+
import { warpRoot } from "./utils/mdast-utils";
|
|
6
|
+
export const mdastToMarkdown = (mdast, options) => {
|
|
7
|
+
let markdown = toMarkdown(warpRoot(mdast), {
|
|
8
|
+
extensions: [gfmToMarkdown(), mathToMarkdown()],
|
|
12
9
|
});
|
|
13
10
|
if (options?.solveLinks)
|
|
14
|
-
markdown =
|
|
11
|
+
markdown = linkReplacer(markdown, options.solveLinks);
|
|
15
12
|
return markdown;
|
|
16
13
|
};
|
|
17
|
-
|
|
18
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdG8tbWFya2Rvd24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWRhc3QtdG8tbWFya2Rvd24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBRUEsbURBQStDO0FBQy9DLHFEQUFpRDtBQUNqRCxtRUFBb0Q7QUFFcEQsbURBQStDO0FBQy9DLHFEQUErQztBQUV4QyxNQUFNLGVBQWUsR0FBRyxDQUFDLEtBQTRCLEVBQUUsT0FBaUMsRUFBVSxFQUFFO0lBQzFHLElBQUksUUFBUSxHQUFHLElBQUEsbUNBQVUsRUFBQyxJQUFBLHNCQUFRLEVBQUMsS0FBSyxDQUFDLEVBQUU7UUFDMUMsVUFBVSxFQUFFLENBQUMsSUFBQSw4QkFBYSxHQUFFLEVBQUUsSUFBQSxnQ0FBYyxHQUFFLENBQUM7S0FDL0MsQ0FBQyxDQUFDO0lBRUgsSUFBSSxPQUFPLEVBQUUsVUFBVTtRQUFFLFFBQVEsR0FBRyxJQUFBLDRCQUFZLEVBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUUvRSxPQUFPLFFBQVEsQ0FBQztBQUNqQixDQUFDLENBQUM7QUFSVyxRQUFBLGVBQWUsbUJBUTFCIn0=
|
|
14
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdG8tbWFya2Rvd24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWRhc3QtdG8tbWFya2Rvd24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLGdCQUFnQixDQUFDO0FBQy9DLE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUNqRCxPQUFPLEVBQUUsVUFBVSxFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFFcEQsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLGlCQUFpQixDQUFDO0FBQy9DLE9BQU8sRUFBRSxRQUFRLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUUvQyxNQUFNLENBQUMsTUFBTSxlQUFlLEdBQUcsQ0FBQyxLQUE0QixFQUFFLE9BQWlDLEVBQVUsRUFBRTtJQUMxRyxJQUFJLFFBQVEsR0FBRyxVQUFVLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxFQUFFO1FBQzFDLFVBQVUsRUFBRSxDQUFDLGFBQWEsRUFBRSxFQUFFLGNBQWMsRUFBRSxDQUFDO0tBQy9DLENBQUMsQ0FBQztJQUVILElBQUksT0FBTyxFQUFFLFVBQVU7UUFBRSxRQUFRLEdBQUcsWUFBWSxDQUFDLFFBQVEsRUFBRSxPQUFPLENBQUMsVUFBVSxDQUFDLENBQUM7SUFFL0UsT0FBTyxRQUFRLENBQUM7QUFDakIsQ0FBQyxDQUFDIn0=
|
package/dist/utils/common.js
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.chunk = void 0;
|
|
4
|
-
const chunk = (array, size) => {
|
|
1
|
+
export const chunk = (array, size) => {
|
|
5
2
|
return array.reduce((acc, _, index) => {
|
|
6
3
|
if (index % size === 0) {
|
|
7
4
|
acc.push(array.slice(index, index + size));
|
|
@@ -10,5 +7,4 @@ const chunk = (array, size) => {
|
|
|
10
7
|
return acc;
|
|
11
8
|
}, []);
|
|
12
9
|
};
|
|
13
|
-
|
|
14
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29tbW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3V0aWxzL2NvbW1vbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7QUFBTyxNQUFNLEtBQUssR0FBRyxDQUFJLEtBQVUsRUFBRSxJQUFZLEVBQVMsRUFBRTtJQUMzRCxPQUFPLEtBQUssQ0FBQyxNQUFNLENBQVEsQ0FBQyxHQUFHLEVBQUUsQ0FBQyxFQUFFLEtBQUssRUFBRSxFQUFFO1FBQzVDLElBQUksS0FBSyxHQUFHLElBQUksS0FBSyxDQUFDLEVBQUU7WUFDdkIsR0FBRyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLEtBQUssRUFBRSxLQUFLLEdBQUcsSUFBSSxDQUFDLENBQUMsQ0FBQztZQUMzQyxPQUFPLEdBQUcsQ0FBQztTQUNYO1FBQ0QsT0FBTyxHQUFHLENBQUM7SUFDWixDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7QUFDUixDQUFDLENBQUM7QUFSVyxRQUFBLEtBQUssU0FRaEIifQ==
|
|
10
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29tbW9uLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3V0aWxzL2NvbW1vbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxNQUFNLENBQUMsTUFBTSxLQUFLLEdBQUcsQ0FBSSxLQUFVLEVBQUUsSUFBWSxFQUFTLEVBQUU7SUFDM0QsT0FBTyxLQUFLLENBQUMsTUFBTSxDQUFRLENBQUMsR0FBRyxFQUFFLENBQUMsRUFBRSxLQUFLLEVBQUUsRUFBRTtRQUM1QyxJQUFJLEtBQUssR0FBRyxJQUFJLEtBQUssQ0FBQyxFQUFFO1lBQ3ZCLEdBQUcsQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxLQUFLLEVBQUUsS0FBSyxHQUFHLElBQUksQ0FBQyxDQUFDLENBQUM7WUFDM0MsT0FBTyxHQUFHLENBQUM7U0FDWDtRQUNELE9BQU8sR0FBRyxDQUFDO0lBQ1osQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO0FBQ1IsQ0FBQyxDQUFDIn0=
|
|
@@ -1,22 +1,16 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.internalType = exports.warpRoot = exports.unwarpRoot = void 0;
|
|
4
|
-
const unwarpRoot = (mdast) => {
|
|
1
|
+
export const unwarpRoot = (mdast) => {
|
|
5
2
|
if (mdast.type === "root")
|
|
6
3
|
return mdast.children;
|
|
7
4
|
return [mdast];
|
|
8
5
|
};
|
|
9
|
-
|
|
10
|
-
const warpRoot = (mdast) => {
|
|
6
|
+
export const warpRoot = (mdast) => {
|
|
11
7
|
if (Array.isArray(mdast))
|
|
12
8
|
return { type: "root", children: mdast };
|
|
13
9
|
return mdast;
|
|
14
10
|
};
|
|
15
|
-
|
|
16
|
-
const internalType = (content) => {
|
|
11
|
+
export const internalType = (content) => {
|
|
17
12
|
if (content.type === "heading")
|
|
18
13
|
return `h${content.depth}`;
|
|
19
14
|
return content.type;
|
|
20
15
|
};
|
|
21
|
-
|
|
22
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdXRpbHMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvdXRpbHMvbWRhc3QtdXRpbHMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBRU8sTUFBTSxVQUFVLEdBQUcsQ0FBQyxLQUFZLEVBQWlCLEVBQUU7SUFDekQsSUFBSSxLQUFLLENBQUMsSUFBSSxLQUFLLE1BQU07UUFBRSxPQUFPLEtBQUssQ0FBQyxRQUFRLENBQUM7SUFDakQsT0FBTyxDQUFDLEtBQUssQ0FBQyxDQUFDO0FBQ2hCLENBQUMsQ0FBQztBQUhXLFFBQUEsVUFBVSxjQUdyQjtBQUVLLE1BQU0sUUFBUSxHQUFHLENBQUMsS0FBNEIsRUFBUyxFQUFFO0lBQy9ELElBQUksS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUM7UUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxRQUFRLEVBQUUsS0FBSyxFQUFFLENBQUM7SUFDbkUsT0FBTyxLQUFLLENBQUM7QUFDZCxDQUFDLENBQUM7QUFIVyxRQUFBLFFBQVEsWUFHbkI7QUFFSyxNQUFNLFlBQVksR0FBRyxDQUFDLE9BQW9CLEVBQVUsRUFBRTtJQUM1RCxJQUFJLE9BQU8sQ0FBQyxJQUFJLEtBQUssU0FBUztRQUFFLE9BQU8sSUFBSSxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDM0QsT0FBTyxPQUFPLENBQUMsSUFBSSxDQUFDO0FBQ3JCLENBQUMsQ0FBQztBQUhXLFFBQUEsWUFBWSxnQkFHdkIifQ==
|
|
16
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdXRpbHMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvdXRpbHMvbWRhc3QtdXRpbHMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHLENBQUMsS0FBWSxFQUFpQixFQUFFO0lBQ3pELElBQUksS0FBSyxDQUFDLElBQUksS0FBSyxNQUFNO1FBQUUsT0FBTyxLQUFLLENBQUMsUUFBUSxDQUFDO0lBQ2pELE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQztBQUNoQixDQUFDLENBQUM7QUFFRixNQUFNLENBQUMsTUFBTSxRQUFRLEdBQUcsQ0FBQyxLQUE0QixFQUFTLEVBQUU7SUFDL0QsSUFBSSxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQztRQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSxFQUFFLFFBQVEsRUFBRSxLQUFLLEVBQUUsQ0FBQztJQUNuRSxPQUFPLEtBQUssQ0FBQztBQUNkLENBQUMsQ0FBQztBQUVGLE1BQU0sQ0FBQyxNQUFNLFlBQVksR0FBRyxDQUFDLE9BQW9CLEVBQVUsRUFBRTtJQUM1RCxJQUFJLE9BQU8sQ0FBQyxJQUFJLEtBQUssU0FBUztRQUFFLE9BQU8sSUFBSSxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDM0QsT0FBTyxPQUFPLENBQUMsSUFBSSxDQUFDO0FBQ3JCLENBQUMsQ0FBQyJ9
|
package/package.json
CHANGED
package/src/extract-hast.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
import { select, selectAll } from "hast-util-select";
|
|
3
|
-
|
|
4
|
-
import { remove } from "unist-util-remove";
|
|
5
|
-
|
|
6
|
-
export const extractHast = (hast: Hast) => {
|
|
7
|
-
const removeSelectors = ["header", "footer", "nav"];
|
|
8
|
-
|
|
9
|
-
for (const selector of removeSelectors) {
|
|
10
|
-
const matchedNodes = selectAll(selector, hast);
|
|
11
|
-
for (const node of matchedNodes) {
|
|
12
|
-
remove(hast, { type: node.type, tagName: node.tagName });
|
|
13
|
-
}
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
const selectors = ["article", "div#article", "main", "div.container"];
|
|
17
|
-
for (const selector of selectors) {
|
|
18
|
-
const element = select(selector, hast);
|
|
19
|
-
if (element) return element;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
return hast;
|
|
23
|
-
};
|
|
1
|
+
import type { Root as Hast } from "hast";
|
|
2
|
+
import { select, selectAll } from "hast-util-select";
|
|
3
|
+
|
|
4
|
+
import { remove } from "unist-util-remove";
|
|
5
|
+
|
|
6
|
+
export const extractHast = (hast: Hast) => {
|
|
7
|
+
const removeSelectors = ["header", "footer", "nav"];
|
|
8
|
+
|
|
9
|
+
for (const selector of removeSelectors) {
|
|
10
|
+
const matchedNodes = selectAll(selector, hast);
|
|
11
|
+
for (const node of matchedNodes) {
|
|
12
|
+
remove(hast, { type: node.type, tagName: node.tagName });
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const selectors = ["article", "div#article", "main", "div.container"];
|
|
17
|
+
for (const selector of selectors) {
|
|
18
|
+
const element = select(selector, hast);
|
|
19
|
+
if (element) return element;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
return hast;
|
|
23
|
+
};
|
package/src/html-to-markdown.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
import { htmlToMdast } from "./html-to-mdast";
|
|
3
|
-
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
4
|
-
|
|
5
|
-
export interface HtmlToMarkdownOptions {
|
|
6
|
-
extractHast?: false | ((hast: Hast) => Hast);
|
|
7
|
-
solveLinks?: string;
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
export const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {
|
|
11
|
-
const mdast = htmlToMdast(html, options);
|
|
12
|
-
const markdown = mdastToMarkdown(mdast, options);
|
|
13
|
-
return markdown;
|
|
14
|
-
};
|
|
1
|
+
import type { Root as Hast } from "hast";
|
|
2
|
+
import { htmlToMdast } from "./html-to-mdast";
|
|
3
|
+
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
4
|
+
|
|
5
|
+
export interface HtmlToMarkdownOptions {
|
|
6
|
+
extractHast?: false | ((hast: Hast) => Hast);
|
|
7
|
+
solveLinks?: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {
|
|
11
|
+
const mdast = htmlToMdast(html, options);
|
|
12
|
+
const markdown = mdastToMarkdown(mdast, options);
|
|
13
|
+
return markdown;
|
|
14
|
+
};
|
package/src/html-to-mdast.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
import type { Root as Hast } from "hast";
|
|
2
|
-
import type { Nodes as Mdast } from "mdast";
|
|
3
|
-
|
|
4
|
-
import { fromHtml } from "hast-util-from-html";
|
|
5
|
-
import { toMdast } from "hast-util-to-mdast";
|
|
6
|
-
|
|
7
|
-
import { extractHast } from "./extract-hast";
|
|
8
|
-
import { customDivHandler } from "./mdast-handlers/custom-div-handler";
|
|
9
|
-
import { mathHandler } from "./mdast-handlers/math-handler";
|
|
10
|
-
|
|
11
|
-
export type HtmlToMdastOptions = {
|
|
12
|
-
extractHast?: false | ((hast: Hast) => Hast);
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
export const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {
|
|
16
|
-
const hast = fromHtml(html, { fragment: true });
|
|
17
|
-
const extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);
|
|
18
|
-
const mdast = toMdast(extractedHast, {
|
|
19
|
-
handlers: { div: customDivHandler, math: mathHandler, br: () => {} },
|
|
20
|
-
});
|
|
21
|
-
|
|
22
|
-
return mdast;
|
|
23
|
-
};
|
|
1
|
+
import type { Root as Hast } from "hast";
|
|
2
|
+
import type { Nodes as Mdast } from "mdast";
|
|
3
|
+
|
|
4
|
+
import { fromHtml } from "hast-util-from-html";
|
|
5
|
+
import { toMdast } from "hast-util-to-mdast";
|
|
6
|
+
|
|
7
|
+
import { extractHast } from "./extract-hast";
|
|
8
|
+
import { customDivHandler } from "./mdast-handlers/custom-div-handler";
|
|
9
|
+
import { mathHandler } from "./mdast-handlers/math-handler";
|
|
10
|
+
|
|
11
|
+
export type HtmlToMdastOptions = {
|
|
12
|
+
extractHast?: false | ((hast: Hast) => Hast);
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {
|
|
16
|
+
const hast = fromHtml(html, { fragment: true });
|
|
17
|
+
const extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);
|
|
18
|
+
const mdast = toMdast(extractedHast, {
|
|
19
|
+
handlers: { div: customDivHandler, math: mathHandler, br: () => {} },
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
return mdast;
|
|
23
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export * from "./html-to-markdown";
|
|
2
|
-
export * from "./md-splitter";
|
|
3
|
-
export * from "./html-to-mdast";
|
|
4
|
-
export * from "./mdast-to-markdown";
|
|
1
|
+
export * from "./html-to-markdown";
|
|
2
|
+
export * from "./md-splitter";
|
|
3
|
+
export * from "./html-to-mdast";
|
|
4
|
+
export * from "./mdast-to-markdown";
|
package/src/link-replacer.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export const linkReplacer = (markdown: string, link: string) => {
|
|
2
|
-
const regex = /(!?\[.*?\]\()(.+?)(\))/g;
|
|
3
|
-
return markdown.replace(regex, (match, pre, thisUrl, post) => {
|
|
4
|
-
if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) return match;
|
|
5
|
-
return `${pre}${new URL(thisUrl, link).href}${post}`;
|
|
6
|
-
});
|
|
7
|
-
};
|
|
1
|
+
export const linkReplacer = (markdown: string, link: string) => {
|
|
2
|
+
const regex = /(!?\[.*?\]\()(.+?)(\))/g;
|
|
3
|
+
return markdown.replace(regex, (match, pre, thisUrl, post) => {
|
|
4
|
+
if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) return match;
|
|
5
|
+
return `${pre}${new URL(thisUrl, link).href}${post}`;
|
|
6
|
+
});
|
|
7
|
+
};
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
import playwright from "playwright";
|
|
2
|
-
|
|
3
|
-
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4
|
-
|
|
5
|
-
export const loadHtml = async (url: string, context?: playwright.BrowserContext) => {
|
|
6
|
-
let _browser: playwright.Browser | null = null;
|
|
7
|
-
let _context: playwright.BrowserContext;
|
|
8
|
-
if (!context) {
|
|
9
|
-
_browser = await playwright.chromium.launch();
|
|
10
|
-
_context = await _browser.newContext();
|
|
11
|
-
} else {
|
|
12
|
-
_context = context;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
const page = await _context.newPage();
|
|
16
|
-
await page.goto(url);
|
|
17
|
-
await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
|
|
18
|
-
const html = await page.content();
|
|
19
|
-
await page.close();
|
|
20
|
-
if (_browser) await _browser.close();
|
|
21
|
-
return html;
|
|
22
|
-
};
|
|
1
|
+
import playwright from "playwright";
|
|
2
|
+
|
|
3
|
+
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4
|
+
|
|
5
|
+
export const loadHtml = async (url: string, context?: playwright.BrowserContext) => {
|
|
6
|
+
let _browser: playwright.Browser | null = null;
|
|
7
|
+
let _context: playwright.BrowserContext;
|
|
8
|
+
if (!context) {
|
|
9
|
+
_browser = await playwright.chromium.launch();
|
|
10
|
+
_context = await _browser.newContext();
|
|
11
|
+
} else {
|
|
12
|
+
_context = context;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const page = await _context.newPage();
|
|
16
|
+
await page.goto(url);
|
|
17
|
+
await Promise.race([page.waitForLoadState("networkidle"), sleep(10000)]);
|
|
18
|
+
const html = await page.content();
|
|
19
|
+
await page.close();
|
|
20
|
+
if (_browser) await _browser.close();
|
|
21
|
+
return html;
|
|
22
|
+
};
|
package/src/md-splitter.ts
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
3
|
-
import { chunk } from "./utils/common";
|
|
4
|
-
import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
|
|
5
|
-
|
|
6
|
-
const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
|
|
7
|
-
type SplitterGenerator = Generator<string, void, unknown>;
|
|
8
|
-
const getSplitterGenerator = function* () {
|
|
9
|
-
for (const splitter of PRIORITY_SPLITTERS) yield splitter;
|
|
10
|
-
};
|
|
11
|
-
|
|
12
|
-
const _mdastSplitter = async (
|
|
13
|
-
contents: RootContent[],
|
|
14
|
-
checker: (markdown: string) => Promise<boolean>,
|
|
15
|
-
splitterGenerator: SplitterGenerator,
|
|
16
|
-
): Promise<RootContent[][]> => {
|
|
17
|
-
const markdown = mdastToMarkdown(warpRoot(contents));
|
|
18
|
-
if ((await checker(markdown)) || 3 > contents.length) return [contents];
|
|
19
|
-
const splitter = splitterGenerator.next().value;
|
|
20
|
-
|
|
21
|
-
const chunked = splitter
|
|
22
|
-
? contents.reduce<RootContent[][]>((acc, content) => {
|
|
23
|
-
if (internalType(content) === splitter || acc.length === 0) {
|
|
24
|
-
acc.push([content]);
|
|
25
|
-
return acc;
|
|
26
|
-
}
|
|
27
|
-
acc[acc.length - 1].push(content);
|
|
28
|
-
return acc;
|
|
29
|
-
}, [])
|
|
30
|
-
: chunk(contents, 3);
|
|
31
|
-
|
|
32
|
-
const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
|
|
33
|
-
|
|
34
|
-
return Promise.all(splitting).then((chunks) => chunks.flat());
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
export const mdastSplitter = async (
|
|
38
|
-
mdast: Mdast,
|
|
39
|
-
checker: (markdown: string) => Promise<boolean>,
|
|
40
|
-
options?: { signal?: AbortSignal }, //TODO
|
|
41
|
-
): Promise<RootContent[][]> => {
|
|
42
|
-
return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
|
|
43
|
-
};
|
|
1
|
+
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
+
import { mdastToMarkdown } from "./mdast-to-markdown";
|
|
3
|
+
import { chunk } from "./utils/common";
|
|
4
|
+
import { internalType, unwarpRoot, warpRoot } from "./utils/mdast-utils";
|
|
5
|
+
|
|
6
|
+
const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
|
|
7
|
+
type SplitterGenerator = Generator<string, void, unknown>;
|
|
8
|
+
const getSplitterGenerator = function* () {
|
|
9
|
+
for (const splitter of PRIORITY_SPLITTERS) yield splitter;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
const _mdastSplitter = async (
|
|
13
|
+
contents: RootContent[],
|
|
14
|
+
checker: (markdown: string) => Promise<boolean>,
|
|
15
|
+
splitterGenerator: SplitterGenerator,
|
|
16
|
+
): Promise<RootContent[][]> => {
|
|
17
|
+
const markdown = mdastToMarkdown(warpRoot(contents));
|
|
18
|
+
if ((await checker(markdown)) || 3 > contents.length) return [contents];
|
|
19
|
+
const splitter = splitterGenerator.next().value;
|
|
20
|
+
|
|
21
|
+
const chunked = splitter
|
|
22
|
+
? contents.reduce<RootContent[][]>((acc, content) => {
|
|
23
|
+
if (internalType(content) === splitter || acc.length === 0) {
|
|
24
|
+
acc.push([content]);
|
|
25
|
+
return acc;
|
|
26
|
+
}
|
|
27
|
+
acc[acc.length - 1].push(content);
|
|
28
|
+
return acc;
|
|
29
|
+
}, [])
|
|
30
|
+
: chunk(contents, 3);
|
|
31
|
+
|
|
32
|
+
const splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));
|
|
33
|
+
|
|
34
|
+
return Promise.all(splitting).then((chunks) => chunks.flat());
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export const mdastSplitter = async (
|
|
38
|
+
mdast: Mdast,
|
|
39
|
+
checker: (markdown: string) => Promise<boolean>,
|
|
40
|
+
options?: { signal?: AbortSignal }, //TODO
|
|
41
|
+
): Promise<RootContent[][]> => {
|
|
42
|
+
return _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());
|
|
43
|
+
};
|
|
@@ -1,44 +1,44 @@
|
|
|
1
|
-
import { Handle, defaultHandlers } from "hast-util-to-mdast";
|
|
2
|
-
import { toText } from "hast-util-to-text";
|
|
3
|
-
import { Code } from "mdast";
|
|
4
|
-
import { trimTrailingLines } from "trim-trailing-lines";
|
|
5
|
-
|
|
6
|
-
export const customDivHandler: Handle = (state, node) => {
|
|
7
|
-
const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
|
|
8
|
-
|
|
9
|
-
let highlightLanguage: string | undefined;
|
|
10
|
-
for (const className of classNames) {
|
|
11
|
-
if (String(className) === "highlight-text-html-basic") {
|
|
12
|
-
highlightLanguage = "html";
|
|
13
|
-
break;
|
|
14
|
-
} else if (String(className).includes("highlight-source-")) {
|
|
15
|
-
highlightLanguage = String(className).slice("highlight-source-".length);
|
|
16
|
-
break;
|
|
17
|
-
} else if (String(className).includes("language-")) {
|
|
18
|
-
highlightLanguage = String(className).slice("language-".length);
|
|
19
|
-
break;
|
|
20
|
-
} else if (String(className).includes("code-block_wrapper")) {
|
|
21
|
-
const text = toText(node);
|
|
22
|
-
if (!text.includes(".")) break;
|
|
23
|
-
const lang = toText(node).split("\n")[0].split(".").slice(-1)[0];
|
|
24
|
-
highlightLanguage = lang;
|
|
25
|
-
break;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
if (highlightLanguage !== undefined) {
|
|
30
|
-
const code =
|
|
31
|
-
node.children.find((child) => child.type === "element" && child.tagName === "pre") || node;
|
|
32
|
-
|
|
33
|
-
const result: Code = {
|
|
34
|
-
type: "code",
|
|
35
|
-
lang: highlightLanguage,
|
|
36
|
-
meta: null,
|
|
37
|
-
value: trimTrailingLines(toText(code)).trim(),
|
|
38
|
-
};
|
|
39
|
-
state.patch(node, result);
|
|
40
|
-
return result;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
return defaultHandlers.div(state, node);
|
|
44
|
-
};
|
|
1
|
+
import { Handle, defaultHandlers } from "hast-util-to-mdast";
|
|
2
|
+
import { toText } from "hast-util-to-text";
|
|
3
|
+
import { Code } from "mdast";
|
|
4
|
+
import { trimTrailingLines } from "trim-trailing-lines";
|
|
5
|
+
|
|
6
|
+
export const customDivHandler: Handle = (state, node) => {
|
|
7
|
+
const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
|
|
8
|
+
|
|
9
|
+
let highlightLanguage: string | undefined;
|
|
10
|
+
for (const className of classNames) {
|
|
11
|
+
if (String(className) === "highlight-text-html-basic") {
|
|
12
|
+
highlightLanguage = "html";
|
|
13
|
+
break;
|
|
14
|
+
} else if (String(className).includes("highlight-source-")) {
|
|
15
|
+
highlightLanguage = String(className).slice("highlight-source-".length);
|
|
16
|
+
break;
|
|
17
|
+
} else if (String(className).includes("language-")) {
|
|
18
|
+
highlightLanguage = String(className).slice("language-".length);
|
|
19
|
+
break;
|
|
20
|
+
} else if (String(className).includes("code-block_wrapper")) {
|
|
21
|
+
const text = toText(node);
|
|
22
|
+
if (!text.includes(".")) break;
|
|
23
|
+
const lang = toText(node).split("\n")[0].split(".").slice(-1)[0];
|
|
24
|
+
highlightLanguage = lang;
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (highlightLanguage !== undefined) {
|
|
30
|
+
const code =
|
|
31
|
+
node.children.find((child) => child.type === "element" && child.tagName === "pre") || node;
|
|
32
|
+
|
|
33
|
+
const result: Code = {
|
|
34
|
+
type: "code",
|
|
35
|
+
lang: highlightLanguage,
|
|
36
|
+
meta: null,
|
|
37
|
+
value: trimTrailingLines(toText(code)).trim(),
|
|
38
|
+
};
|
|
39
|
+
state.patch(node, result);
|
|
40
|
+
return result;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return defaultHandlers.div(state, node);
|
|
44
|
+
};
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { toHtml } from "hast-util-to-html";
|
|
2
|
-
import { Handle } from "hast-util-to-mdast";
|
|
3
|
-
import { MathMLToLaTeX } from "mathml-to-latex";
|
|
4
|
-
import type { InlineMath, Math } from "mdast-util-math";
|
|
5
|
-
|
|
6
|
-
export const mathHandler: Handle = (state, node) => {
|
|
7
|
-
const mathMl = toHtml(node);
|
|
8
|
-
const latex = MathMLToLaTeX.convert(mathMl);
|
|
9
|
-
const result: InlineMath | Math = { type: "inlineMath", value: latex };
|
|
10
|
-
state.patch(node, result);
|
|
11
|
-
return result;
|
|
12
|
-
};
|
|
1
|
+
import { toHtml } from "hast-util-to-html";
|
|
2
|
+
import { Handle } from "hast-util-to-mdast";
|
|
3
|
+
import { MathMLToLaTeX } from "mathml-to-latex";
|
|
4
|
+
import type { InlineMath, Math } from "mdast-util-math";
|
|
5
|
+
|
|
6
|
+
export const mathHandler: Handle = (state, node) => {
|
|
7
|
+
const mathMl = toHtml(node);
|
|
8
|
+
const latex = MathMLToLaTeX.convert(mathMl);
|
|
9
|
+
const result: InlineMath | Math = { type: "inlineMath", value: latex };
|
|
10
|
+
state.patch(node, result);
|
|
11
|
+
return result;
|
|
12
|
+
};
|
package/src/mdast-to-markdown.ts
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
|
|
3
|
-
import { gfmToMarkdown } from "mdast-util-gfm";
|
|
4
|
-
import { mathToMarkdown } from "mdast-util-math";
|
|
5
|
-
import { toMarkdown } from "mdast-util-to-markdown";
|
|
6
|
-
|
|
7
|
-
import { linkReplacer } from "./link-replacer";
|
|
8
|
-
import { warpRoot } from "./utils/mdast-utils";
|
|
9
|
-
|
|
10
|
-
export const mdastToMarkdown = (mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string => {
|
|
11
|
-
let markdown = toMarkdown(warpRoot(mdast), {
|
|
12
|
-
extensions: [gfmToMarkdown(), mathToMarkdown()],
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
if (options?.solveLinks) markdown = linkReplacer(markdown, options.solveLinks);
|
|
16
|
-
|
|
17
|
-
return markdown;
|
|
18
|
-
};
|
|
1
|
+
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
+
|
|
3
|
+
import { gfmToMarkdown } from "mdast-util-gfm";
|
|
4
|
+
import { mathToMarkdown } from "mdast-util-math";
|
|
5
|
+
import { toMarkdown } from "mdast-util-to-markdown";
|
|
6
|
+
|
|
7
|
+
import { linkReplacer } from "./link-replacer";
|
|
8
|
+
import { warpRoot } from "./utils/mdast-utils";
|
|
9
|
+
|
|
10
|
+
export const mdastToMarkdown = (mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string => {
|
|
11
|
+
let markdown = toMarkdown(warpRoot(mdast), {
|
|
12
|
+
extensions: [gfmToMarkdown(), mathToMarkdown()],
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
if (options?.solveLinks) markdown = linkReplacer(markdown, options.solveLinks);
|
|
16
|
+
|
|
17
|
+
return markdown;
|
|
18
|
+
};
|
package/src/utils/common.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export const chunk = <T>(array: T[], size: number): T[][] => {
|
|
2
|
-
return array.reduce<T[][]>((acc, _, index) => {
|
|
3
|
-
if (index % size === 0) {
|
|
4
|
-
acc.push(array.slice(index, index + size));
|
|
5
|
-
return acc;
|
|
6
|
-
}
|
|
7
|
-
return acc;
|
|
8
|
-
}, []);
|
|
9
|
-
};
|
|
1
|
+
export const chunk = <T>(array: T[], size: number): T[][] => {
|
|
2
|
+
return array.reduce<T[][]>((acc, _, index) => {
|
|
3
|
+
if (index % size === 0) {
|
|
4
|
+
acc.push(array.slice(index, index + size));
|
|
5
|
+
return acc;
|
|
6
|
+
}
|
|
7
|
+
return acc;
|
|
8
|
+
}, []);
|
|
9
|
+
};
|
package/src/utils/mdast-utils.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
-
|
|
3
|
-
export const unwarpRoot = (mdast: Mdast): RootContent[] => {
|
|
4
|
-
if (mdast.type === "root") return mdast.children;
|
|
5
|
-
return [mdast];
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
export const warpRoot = (mdast: RootContent[] | Mdast): Mdast => {
|
|
9
|
-
if (Array.isArray(mdast)) return { type: "root", children: mdast };
|
|
10
|
-
return mdast;
|
|
11
|
-
};
|
|
12
|
-
|
|
13
|
-
export const internalType = (content: RootContent): string => {
|
|
14
|
-
if (content.type === "heading") return `h${content.depth}`;
|
|
15
|
-
return content.type;
|
|
16
|
-
};
|
|
1
|
+
import type { Nodes as Mdast, RootContent } from "mdast";
|
|
2
|
+
|
|
3
|
+
export const unwarpRoot = (mdast: Mdast): RootContent[] => {
|
|
4
|
+
if (mdast.type === "root") return mdast.children;
|
|
5
|
+
return [mdast];
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export const warpRoot = (mdast: RootContent[] | Mdast): Mdast => {
|
|
9
|
+
if (Array.isArray(mdast)) return { type: "root", children: mdast };
|
|
10
|
+
return mdast;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export const internalType = (content: RootContent): string => {
|
|
14
|
+
if (content.type === "heading") return `h${content.depth}`;
|
|
15
|
+
return content.type;
|
|
16
|
+
};
|