webforai 1.2.2 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +74 -74
- package/dist/cjs/extract-hast/index.js +35 -3
- package/dist/cjs/extract-hast/readability.js +57 -29
- package/dist/cjs/extract-hast/strongly.js +6 -3
- package/dist/cjs/extract-hast/utils.js +8 -4
- package/dist/cjs/html-to-mdast.js +6 -19
- package/dist/cjs/index.js +18 -9
- package/dist/cjs/link-replacer.js +2 -1
- package/dist/cjs/loaders/playwright.js +8 -16
- package/dist/cjs/md-splitter.js +5 -3
- package/dist/cjs/mdast-handlers/custom-a-handler.js +2 -1
- package/dist/cjs/mdast-handlers/custom-code-handler.js +43 -0
- package/dist/cjs/mdast-handlers/custom-div-handler.js +15 -9
- package/dist/cjs/mdast-handlers/custom-img-handler.js +2 -1
- package/dist/cjs/mdast-handlers/empty-handler.js +29 -0
- package/dist/cjs/mdast-to-markdown.js +2 -1
- package/dist/cjs/utils/mdast-utils.js +6 -3
- package/dist/extract-hast/index.js +28 -1
- package/dist/extract-hast/readability.js +62 -34
- package/dist/extract-hast/strongly.js +6 -3
- package/dist/extract-hast/utils.js +8 -4
- package/dist/html-to-mdast.js +7 -20
- package/dist/index.js +10 -4
- package/dist/link-replacer.js +2 -1
- package/dist/loaders/playwright.js +8 -6
- package/dist/md-splitter.js +5 -3
- package/dist/mdast-handlers/custom-a-handler.js +4 -3
- package/dist/mdast-handlers/custom-code-handler.js +20 -0
- package/dist/mdast-handlers/custom-div-handler.js +17 -11
- package/dist/mdast-handlers/custom-img-handler.js +2 -1
- package/dist/mdast-handlers/empty-handler.js +6 -0
- package/dist/mdast-to-markdown.js +2 -1
- package/dist/types/extract-hast/index.d.ts +10 -1
- package/dist/types/extract-hast/index.js.map +1 -1
- package/dist/types/extract-hast/readability.js.map +1 -1
- package/dist/types/extract-hast/strongly.js.map +1 -1
- package/dist/types/extract-hast/utils.js.map +1 -1
- package/dist/types/html-to-markdown.d.ts +1 -1
- package/dist/types/html-to-markdown.js.map +1 -1
- package/dist/types/html-to-mdast.d.ts +2 -4
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/index.d.ts +4 -4
- package/dist/types/index.js.map +1 -1
- package/dist/types/link-replacer.js.map +1 -1
- package/dist/types/loaders/playwright.d.ts +2 -2
- package/dist/types/loaders/playwright.js.map +1 -1
- package/dist/types/md-splitter.d.ts +1 -1
- package/dist/types/md-splitter.js.map +1 -1
- package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -0
- package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
- package/dist/types/mdast-handlers/empty-handler.d.ts +1 -0
- package/dist/types/mdast-handlers/empty-handler.js.map +1 -0
- package/dist/types/mdast-to-markdown.js.map +1 -1
- package/dist/types/utils/mdast-utils.js.map +1 -1
- package/dist/utils/mdast-utils.js +6 -3
- package/package.json +118 -118
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 inaridiy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,74 +1,74 @@
|
|
|
1
|
-
# Web for AI
|
|
2
|
-
|
|
3
|
-

|
|
4
|
-
|
|
5
|
-
A library that provides a web interface for AI
|
|
6
|
-
|
|
7
|
-
## Features
|
|
8
|
-
|
|
9
|
-
- Ultra-precise HTML2Markdown conversion
|
|
10
|
-
- Ultra-precise Markdown segmentation based on AST
|
|
11
|
-
- Ultra-precise HTML retrieval functions using headless browsers
|
|
12
|
-
- The core functionality is edge-native (runs on Cloudflare Workers!!!)
|
|
13
|
-
|
|
14
|
-
## Demo
|
|
15
|
-
|
|
16
|
-
There is a demo API for Html2Markdown deployed on CloudflareWorker. Please access the following link
|
|
17
|
-
|
|
18
|
-
- [https://webforai.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai](https://webforai.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai)
|
|
19
|
-
- [https://webforai.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E](https://webforai.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E)
|
|
20
|
-
|
|
21
|
-
## Quick Start
|
|
22
|
-
|
|
23
|
-
Just install and execute scripts
|
|
24
|
-
|
|
25
|
-
```bash
|
|
26
|
-
pnpm i webforai playwright
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
```js
|
|
30
|
-
import { promises as fs } from "fs";
|
|
31
|
-
import { htmlToMarkdown, htmlToMdast } from "webforai";
|
|
32
|
-
import { loadHtml } from "webforai/loaders/playwright";
|
|
33
|
-
|
|
34
|
-
const url = "https://www.npmjs.com/package/webforai";
|
|
35
|
-
const html = await loadHtml(url);
|
|
36
|
-
|
|
37
|
-
const markdown = htmlToMarkdown(html, { solveLinks: url });
|
|
38
|
-
|
|
39
|
-
await fs.writeFile("output.md", markdown);
|
|
40
|
-
```
|
|
41
|
-
|
|
42
|
-
other examples are in [examples](./examples/simple/src/index.ts)
|
|
43
|
-
|
|
44
|
-
## Usage
|
|
45
|
-
|
|
46
|
-
### Main Functions
|
|
47
|
-
|
|
48
|
-
**`htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string`**
|
|
49
|
-
Convert HTML to Markdown. By default, unnecessary HTML is excluded and processed.
|
|
50
|
-
If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
51
|
-
This function just calls htmlToMdast and mdastToMarkdown in that order internally.
|
|
52
|
-
|
|
53
|
-
**`htmlToMdast(html: string, options?: HtmlToMdastOptions): Mdast`**
|
|
54
|
-
This project uses Hast and Mdast as defined by syntax-tree internally.
|
|
55
|
-
This function converts HTML to Mdast, an intermediate representation, which is required when using `mdastSplitter`, etc.
|
|
56
|
-
|
|
57
|
-
**`mdastToMarkdown(mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string`**
|
|
58
|
-
Convert Mdast to Markdown. If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
59
|
-
|
|
60
|
-
### Loader Functions
|
|
61
|
-
|
|
62
|
-
**`loadHtml(url: string, options?: LoadHtmlOptions): Promise<string>`**
|
|
63
|
-
Load HTML from the specified URL. This function uses Playwright internally.
|
|
64
|
-
|
|
65
|
-
## Early Users
|
|
66
|
-
|
|
67
|
-
### Voice Genius - Explanatory video generation by AI
|
|
68
|
-
|
|
69
|
-
<img src="https://github.com/inaridiy/webforai/blob/main/images/voice-genius.png" width="300px">
|
|
70
|
-
|
|
71
|
-
by [@moons14](https://twitter.com/moons_dev)
|
|
72
|
-
|
|
73
|
-
AI can be used to automatically generate highly accurate commentary videos.
|
|
74
|
-
Supports conversations and presentations using synthesized voice, embedding of images in videos, etc.
|
|
1
|
+
# Web for AI
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+
|
|
5
|
+
A library that provides a web interface for AI
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- Ultra-precise HTML2Markdown conversion
|
|
10
|
+
- Ultra-precise Markdown segmentation based on AST
|
|
11
|
+
- Ultra-precise HTML retrieval functions using headless browsers
|
|
12
|
+
- The core functionality is edge-native (runs on Cloudflare Workers!!!)
|
|
13
|
+
|
|
14
|
+
## Demo
|
|
15
|
+
|
|
16
|
+
There is a demo API for Html2Markdown deployed on CloudflareWorker. Please access the following link
|
|
17
|
+
|
|
18
|
+
- [https://webforai.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai](https://webforai.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai)
|
|
19
|
+
- [https://webforai.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E](https://webforai.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E)
|
|
20
|
+
|
|
21
|
+
## Quick Start
|
|
22
|
+
|
|
23
|
+
Just install and execute scripts
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pnpm i webforai playwright
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```js
|
|
30
|
+
import { promises as fs } from "fs";
|
|
31
|
+
import { htmlToMarkdown, htmlToMdast } from "webforai";
|
|
32
|
+
import { loadHtml } from "webforai/loaders/playwright";
|
|
33
|
+
|
|
34
|
+
const url = "https://www.npmjs.com/package/webforai";
|
|
35
|
+
const html = await loadHtml(url);
|
|
36
|
+
|
|
37
|
+
const markdown = htmlToMarkdown(html, { solveLinks: url });
|
|
38
|
+
|
|
39
|
+
await fs.writeFile("output.md", markdown);
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
other examples are in [examples](./examples/simple/src/index.ts)
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
### Main Functions
|
|
47
|
+
|
|
48
|
+
**`htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string`**
|
|
49
|
+
Convert HTML to Markdown. By default, unnecessary HTML is excluded and processed.
|
|
50
|
+
If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
51
|
+
This function just calls htmlToMdast and mdastToMarkdown in that order internally.
|
|
52
|
+
|
|
53
|
+
**`htmlToMdast(html: string, options?: HtmlToMdastOptions): Mdast`**
|
|
54
|
+
This project uses Hast and Mdast as defined by syntax-tree internally.
|
|
55
|
+
This function converts HTML to Mdast, an intermediate representation, which is required when using `mdastSplitter`, etc.
|
|
56
|
+
|
|
57
|
+
**`mdastToMarkdown(mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string`**
|
|
58
|
+
Convert Mdast to Markdown. If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
59
|
+
|
|
60
|
+
### Loader Functions
|
|
61
|
+
|
|
62
|
+
**`loadHtml(url: string, options?: LoadHtmlOptions): Promise<string>`**
|
|
63
|
+
Load HTML from the specified URL. This function uses Playwright internally.
|
|
64
|
+
|
|
65
|
+
## Early Users
|
|
66
|
+
|
|
67
|
+
### Voice Genius - Explanatory video generation by AI
|
|
68
|
+
|
|
69
|
+
<img src="https://github.com/inaridiy/webforai/blob/main/images/voice-genius.png" width="300px">
|
|
70
|
+
|
|
71
|
+
by [@moons14](https://twitter.com/moons_dev)
|
|
72
|
+
|
|
73
|
+
AI can be used to automatically generate highly accurate commentary videos.
|
|
74
|
+
Supports conversations and presentations using synthesized voice, embedding of images in videos, etc.
|
|
@@ -3,6 +3,10 @@ var __defProp = Object.defineProperty;
|
|
|
3
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
6
10
|
var __copyProps = (to, from, except, desc) => {
|
|
7
11
|
if (from && typeof from === "object" || typeof from === "function") {
|
|
8
12
|
for (let key of __getOwnPropNames(from))
|
|
@@ -11,12 +15,40 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
11
15
|
}
|
|
12
16
|
return to;
|
|
13
17
|
};
|
|
14
|
-
var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
|
|
15
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
16
19
|
var extract_hast_exports = {};
|
|
20
|
+
__export(extract_hast_exports, {
|
|
21
|
+
DEFAULT_EXTRACT_HAST: () => DEFAULT_EXTRACT_HAST,
|
|
22
|
+
PRESET_EXTRACT_HAST: () => PRESET_EXTRACT_HAST,
|
|
23
|
+
extractHast: () => extractHast
|
|
24
|
+
});
|
|
17
25
|
module.exports = __toCommonJS(extract_hast_exports);
|
|
18
|
-
|
|
26
|
+
var import_readability = require("./readability");
|
|
27
|
+
var import_strongly = require("./strongly");
|
|
28
|
+
const PRESET_EXTRACT_HAST = {
|
|
29
|
+
strongly: import_strongly.stronglyExtractHast,
|
|
30
|
+
readability: import_readability.readabilityExtractHast
|
|
31
|
+
};
|
|
32
|
+
const DEFAULT_EXTRACT_HAST = ["readability"];
|
|
33
|
+
const extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
|
|
34
|
+
const _extractors = Array.isArray(extractors) ? extractors : [extractors];
|
|
35
|
+
const extracted = _extractors.reduce((acc, extractor) => {
|
|
36
|
+
if (extractor === false) {
|
|
37
|
+
return acc;
|
|
38
|
+
}
|
|
39
|
+
if (typeof extractor === "string" && extractor in DEFAULT_EXTRACT_HAST) {
|
|
40
|
+
return PRESET_EXTRACT_HAST[extractor](acc);
|
|
41
|
+
}
|
|
42
|
+
if (typeof extractor === "function") {
|
|
43
|
+
return extractor(acc);
|
|
44
|
+
}
|
|
45
|
+
throw new Error(`Invalid extractor: ${extractor}`);
|
|
46
|
+
}, hast) || hast;
|
|
47
|
+
return extracted;
|
|
48
|
+
};
|
|
19
49
|
// Annotate the CommonJS export names for ESM import in node:
|
|
20
50
|
0 && (module.exports = {
|
|
21
|
-
|
|
51
|
+
DEFAULT_EXTRACT_HAST,
|
|
52
|
+
PRESET_EXTRACT_HAST,
|
|
53
|
+
extractHast
|
|
22
54
|
});
|
|
@@ -31,7 +31,7 @@ const REGEXPS = {
|
|
|
31
31
|
hidden: /hidden|invisible|fallback-image/i,
|
|
32
32
|
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
33
33
|
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
|
|
34
|
-
|
|
34
|
+
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
35
35
|
};
|
|
36
36
|
const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
|
|
37
37
|
const PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
|
|
@@ -40,60 +40,79 @@ const BASE_MINIMAL_LENGTH = {
|
|
|
40
40
|
en: 500
|
|
41
41
|
};
|
|
42
42
|
const tryGetLang = (node) => {
|
|
43
|
-
if (node.type !== "element")
|
|
43
|
+
if (node.type !== "element") {
|
|
44
44
|
return;
|
|
45
|
+
}
|
|
45
46
|
const element = node;
|
|
46
|
-
if (element.tagName !== "html")
|
|
47
|
+
if (element.tagName !== "html") {
|
|
47
48
|
return;
|
|
49
|
+
}
|
|
48
50
|
const langAttr = element.properties.lang || element.properties["xml:lang"];
|
|
49
|
-
if (langAttr)
|
|
51
|
+
if (langAttr) {
|
|
50
52
|
return langAttr;
|
|
53
|
+
}
|
|
51
54
|
};
|
|
52
55
|
const metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
|
|
53
56
|
const universalElementFilter = (node) => {
|
|
54
|
-
if (node.type !== "element")
|
|
57
|
+
if (node.type !== "element") {
|
|
55
58
|
return true;
|
|
59
|
+
}
|
|
56
60
|
const element = node;
|
|
57
|
-
if (["script", "aside"].includes(element.tagName))
|
|
61
|
+
if (["script", "aside"].includes(element.tagName)) {
|
|
58
62
|
return false;
|
|
59
|
-
|
|
63
|
+
}
|
|
64
|
+
if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
|
|
60
65
|
return false;
|
|
61
|
-
|
|
66
|
+
}
|
|
67
|
+
if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname))) {
|
|
62
68
|
return false;
|
|
63
|
-
|
|
69
|
+
}
|
|
70
|
+
if (element.tagName === "dialog") {
|
|
64
71
|
return false;
|
|
65
|
-
|
|
72
|
+
}
|
|
73
|
+
if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
|
|
66
74
|
return false;
|
|
67
|
-
|
|
75
|
+
}
|
|
76
|
+
if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author")) {
|
|
68
77
|
return false;
|
|
69
|
-
|
|
78
|
+
}
|
|
79
|
+
if (REGEXPS.byline.test((0, import_utils.matchString)(element))) {
|
|
70
80
|
return false;
|
|
71
|
-
|
|
81
|
+
}
|
|
82
|
+
if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
|
|
72
83
|
return false;
|
|
84
|
+
}
|
|
73
85
|
return true;
|
|
74
86
|
};
|
|
75
87
|
const unlikelyElementFilter = (node) => {
|
|
76
|
-
if (node.type !== "element")
|
|
88
|
+
if (node.type !== "element") {
|
|
77
89
|
return true;
|
|
90
|
+
}
|
|
78
91
|
const element = node;
|
|
79
92
|
const match = (0, import_utils.matchString)(element);
|
|
80
|
-
if (["body", "article", "main", "section", "a"].includes(element.tagName))
|
|
93
|
+
if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
|
|
81
94
|
return true;
|
|
82
|
-
|
|
95
|
+
}
|
|
96
|
+
if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3)) {
|
|
83
97
|
return true;
|
|
84
|
-
|
|
98
|
+
}
|
|
99
|
+
if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
|
|
85
100
|
return false;
|
|
101
|
+
}
|
|
86
102
|
return true;
|
|
87
103
|
};
|
|
88
104
|
const removeEmptyFilter = (node) => {
|
|
89
|
-
if (node.type !== "element")
|
|
105
|
+
if (node.type !== "element") {
|
|
90
106
|
return true;
|
|
107
|
+
}
|
|
91
108
|
const element = node;
|
|
92
|
-
if (!PARAGRAPH_TAGS.includes(element.tagName))
|
|
109
|
+
if (!PARAGRAPH_TAGS.includes(element.tagName)) {
|
|
93
110
|
return true;
|
|
111
|
+
}
|
|
94
112
|
const text = (0, import_hast_util_to_string.toString)(element);
|
|
95
|
-
if (text.length < 10)
|
|
113
|
+
if (text.length < 10) {
|
|
96
114
|
return false;
|
|
115
|
+
}
|
|
97
116
|
return true;
|
|
98
117
|
};
|
|
99
118
|
const readabilityExtractHast = (hast) => {
|
|
@@ -101,45 +120,54 @@ const readabilityExtractHast = (hast) => {
|
|
|
101
120
|
const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
|
|
102
121
|
const proxiedHast = (0, import_unist_util_parents.parents)(body);
|
|
103
122
|
let baseFilterd = (0, import_unist_util_filter.filter)(proxiedHast, (node) => {
|
|
104
|
-
if (!metadataFilter(node))
|
|
123
|
+
if (!metadataFilter(node)) {
|
|
105
124
|
return false;
|
|
106
|
-
|
|
125
|
+
}
|
|
126
|
+
if (!universalElementFilter(node)) {
|
|
107
127
|
return false;
|
|
128
|
+
}
|
|
108
129
|
return true;
|
|
109
130
|
});
|
|
110
|
-
if (!baseFilterd)
|
|
131
|
+
if (!baseFilterd) {
|
|
111
132
|
return { type: "root", children: [] };
|
|
133
|
+
}
|
|
112
134
|
const baseText = (0, import_hast_util_to_string.toString)(baseFilterd);
|
|
113
135
|
let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
|
|
114
136
|
if (baseText.length > minimalLength) {
|
|
115
137
|
const filterd = (0, import_unist_util_filter.filter)(baseFilterd, (node) => {
|
|
116
|
-
if (!unlikelyElementFilter(node))
|
|
138
|
+
if (!unlikelyElementFilter(node)) {
|
|
117
139
|
return false;
|
|
140
|
+
}
|
|
118
141
|
return true;
|
|
119
142
|
});
|
|
120
|
-
if (filterd)
|
|
143
|
+
if (filterd) {
|
|
121
144
|
baseFilterd = filterd;
|
|
122
|
-
|
|
145
|
+
}
|
|
146
|
+
} else {
|
|
123
147
|
minimalLength = Math.max(0, baseText.length - 200);
|
|
148
|
+
}
|
|
124
149
|
let bodyTree = baseFilterd;
|
|
125
150
|
for (const selector of BODY_SELECTORS) {
|
|
126
151
|
const body2 = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
|
|
127
152
|
const bodyText = (0, import_hast_util_to_string.toString)(body2);
|
|
128
|
-
if (bodyText.length < 25)
|
|
153
|
+
if (bodyText.length < 25) {
|
|
129
154
|
continue;
|
|
155
|
+
}
|
|
130
156
|
const links = (0, import_hast_util_select.selectAll)("a", body2);
|
|
131
157
|
const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
|
|
132
158
|
const linkDensity = linkText.length / bodyText.length;
|
|
133
|
-
if (linkDensity > 0.4)
|
|
159
|
+
if (linkDensity > 0.4) {
|
|
134
160
|
continue;
|
|
161
|
+
}
|
|
135
162
|
if (bodyText.length > minimalLength) {
|
|
136
163
|
bodyTree = body2;
|
|
137
164
|
break;
|
|
138
165
|
}
|
|
139
166
|
}
|
|
140
167
|
const finalTree = (0, import_unist_util_filter.filter)(bodyTree, (node) => {
|
|
141
|
-
if (!removeEmptyFilter(node))
|
|
168
|
+
if (!removeEmptyFilter(node)) {
|
|
142
169
|
return false;
|
|
170
|
+
}
|
|
143
171
|
return true;
|
|
144
172
|
});
|
|
145
173
|
return finalTree;
|
|
@@ -35,14 +35,17 @@ const stronglyExtractHast = (hast) => {
|
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
const removedElements = (0, import_unist_util_filter.filter)(selectedElement, (node) => {
|
|
38
|
-
if (node.type === "comment")
|
|
38
|
+
if (node.type === "comment") {
|
|
39
39
|
return false;
|
|
40
|
-
|
|
40
|
+
}
|
|
41
|
+
if (node.type !== "element") {
|
|
41
42
|
return true;
|
|
43
|
+
}
|
|
42
44
|
const element = node;
|
|
43
45
|
const elementName = `${element.tagName} ${element.properties.id} ${element.properties.className}`;
|
|
44
|
-
if (REMOVE_REGEX.test(elementName))
|
|
46
|
+
if (REMOVE_REGEX.test(elementName)) {
|
|
45
47
|
return false;
|
|
48
|
+
}
|
|
46
49
|
return true;
|
|
47
50
|
});
|
|
48
51
|
return removedElements;
|
|
@@ -26,22 +26,26 @@ __export(utils_exports, {
|
|
|
26
26
|
module.exports = __toCommonJS(utils_exports);
|
|
27
27
|
const matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")} }`;
|
|
28
28
|
const classnames = (element) => {
|
|
29
|
-
if (Array.isArray(element.properties.className))
|
|
29
|
+
if (Array.isArray(element.properties.className)) {
|
|
30
30
|
return element.properties.className;
|
|
31
|
+
}
|
|
31
32
|
return [];
|
|
32
33
|
};
|
|
33
34
|
const isStrInclude = (value, match) => {
|
|
34
|
-
if (typeof value === "string")
|
|
35
|
+
if (typeof value === "string") {
|
|
35
36
|
return value.includes(match);
|
|
37
|
+
}
|
|
36
38
|
return false;
|
|
37
39
|
};
|
|
38
40
|
const hasAncestors = (element, tagNames, depth = 3) => {
|
|
39
41
|
let parent = element.parent;
|
|
40
42
|
for (let i = 0; i < depth; i++) {
|
|
41
|
-
if (!parent || parent.type !== "element")
|
|
43
|
+
if (!parent || parent.type !== "element") {
|
|
42
44
|
return false;
|
|
43
|
-
|
|
45
|
+
}
|
|
46
|
+
if (tagNames.includes(parent.tagName)) {
|
|
44
47
|
return true;
|
|
48
|
+
}
|
|
45
49
|
parent = parent.parent;
|
|
46
50
|
}
|
|
47
51
|
return false;
|
|
@@ -24,39 +24,26 @@ module.exports = __toCommonJS(html_to_mdast_exports);
|
|
|
24
24
|
var import_hast_util_from_html = require("hast-util-from-html");
|
|
25
25
|
var import_hast_util_to_mdast = require("hast-util-to-mdast");
|
|
26
26
|
var import_extract_hast = require("./extract-hast");
|
|
27
|
-
var import_readability = require("./extract-hast/readability");
|
|
28
27
|
var import_custom_a_handler = require("./mdast-handlers/custom-a-handler");
|
|
28
|
+
var import_custom_code_handler = require("./mdast-handlers/custom-code-handler");
|
|
29
29
|
var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
|
|
30
30
|
var import_custom_img_handler = require("./mdast-handlers/custom-img-handler");
|
|
31
31
|
var import_custom_table_handler = require("./mdast-handlers/custom-table-handler");
|
|
32
|
+
var import_empty_handler = require("./mdast-handlers/empty-handler");
|
|
32
33
|
var import_math_handler = require("./mdast-handlers/math-handler");
|
|
33
|
-
const PRESET_EXTRACT_HAST = {
|
|
34
|
-
strongly: import_extract_hast.stronglyExtractHast,
|
|
35
|
-
readability: import_readability.readabilityExtractHast
|
|
36
|
-
};
|
|
37
|
-
const DEFAULT_EXTRACT_HAST = ["readability"];
|
|
38
34
|
const htmlToMdast = (html, options) => {
|
|
39
|
-
const { extractHast
|
|
40
|
-
const _extractHast = Array.isArray(extractHast) ? extractHast : [extractHast];
|
|
35
|
+
const { extractHast: _extractHast } = options || {};
|
|
41
36
|
const hast = (0, import_hast_util_from_html.fromHtml)(html, { fragment: true });
|
|
42
|
-
const extractedHast =
|
|
43
|
-
if (extractHast2 === false)
|
|
44
|
-
return acc;
|
|
45
|
-
if (typeof extractHast2 === "string" && extractHast2 in PRESET_EXTRACT_HAST)
|
|
46
|
-
return PRESET_EXTRACT_HAST[extractHast2](acc);
|
|
47
|
-
if (typeof extractHast2 === "function")
|
|
48
|
-
return extractHast2(acc);
|
|
49
|
-
throw new Error(`Invalid extractHast: ${extractHast2}`);
|
|
50
|
-
}, hast) || hast;
|
|
37
|
+
const extractedHast = (0, import_extract_hast.extractHast)(hast, _extractHast);
|
|
51
38
|
const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
|
|
52
39
|
handlers: {
|
|
53
40
|
math: import_math_handler.mathHandler,
|
|
54
41
|
div: import_custom_div_handler.customDivHandler,
|
|
42
|
+
pre: import_custom_code_handler.customCodeHandler,
|
|
55
43
|
a: (0, import_custom_a_handler.customAHandler)({ asText: options?.linkAsText }),
|
|
56
44
|
img: (0, import_custom_img_handler.customImgHandler)({ hideImage: options?.hideImage }),
|
|
57
45
|
table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText }),
|
|
58
|
-
br:
|
|
59
|
-
}
|
|
46
|
+
br: import_empty_handler.emptyHandler
|
|
60
47
|
}
|
|
61
48
|
});
|
|
62
49
|
return mdast;
|
package/dist/cjs/index.js
CHANGED
|
@@ -3,6 +3,10 @@ var __defProp = Object.defineProperty;
|
|
|
3
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
6
10
|
var __copyProps = (to, from, except, desc) => {
|
|
7
11
|
if (from && typeof from === "object" || typeof from === "function") {
|
|
8
12
|
for (let key of __getOwnPropNames(from))
|
|
@@ -11,18 +15,23 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
11
15
|
}
|
|
12
16
|
return to;
|
|
13
17
|
};
|
|
14
|
-
var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
|
|
15
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
16
19
|
var src_exports = {};
|
|
20
|
+
__export(src_exports, {
|
|
21
|
+
htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
|
|
22
|
+
htmlToMdast: () => import_html_to_mdast.htmlToMdast,
|
|
23
|
+
mdastSplitter: () => import_md_splitter.mdastSplitter,
|
|
24
|
+
mdastToMarkdown: () => import_mdast_to_markdown.mdastToMarkdown
|
|
25
|
+
});
|
|
17
26
|
module.exports = __toCommonJS(src_exports);
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
27
|
+
var import_html_to_markdown = require("./html-to-markdown");
|
|
28
|
+
var import_md_splitter = require("./md-splitter");
|
|
29
|
+
var import_html_to_mdast = require("./html-to-mdast");
|
|
30
|
+
var import_mdast_to_markdown = require("./mdast-to-markdown");
|
|
22
31
|
// Annotate the CommonJS export names for ESM import in node:
|
|
23
32
|
0 && (module.exports = {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
33
|
+
htmlToMarkdown,
|
|
34
|
+
htmlToMdast,
|
|
35
|
+
mdastSplitter,
|
|
36
|
+
mdastToMarkdown
|
|
28
37
|
});
|
|
@@ -24,8 +24,9 @@ module.exports = __toCommonJS(link_replacer_exports);
|
|
|
24
24
|
const linkReplacer = (markdown, link) => {
|
|
25
25
|
const regex = /(!?\[.*?\]\()(.+?)(\))/g;
|
|
26
26
|
return markdown.replace(regex, (match, pre, thisUrl, post) => {
|
|
27
|
-
if (thisUrl.startsWith("http") || thisUrl.startsWith("#"))
|
|
27
|
+
if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
|
|
28
28
|
return match;
|
|
29
|
+
}
|
|
29
30
|
return `${pre}${new URL(thisUrl, link).href}${post}`;
|
|
30
31
|
});
|
|
31
32
|
};
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
2
|
var __defProp = Object.defineProperty;
|
|
4
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
6
|
var __export = (target, all) => {
|
|
9
7
|
for (var name in all)
|
|
@@ -17,37 +15,31 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
17
15
|
}
|
|
18
16
|
return to;
|
|
19
17
|
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
19
|
var playwright_exports = {};
|
|
30
20
|
__export(playwright_exports, {
|
|
31
21
|
loadHtml: () => loadHtml
|
|
32
22
|
});
|
|
33
23
|
module.exports = __toCommonJS(playwright_exports);
|
|
34
|
-
var
|
|
24
|
+
var import_playwright = require("playwright");
|
|
35
25
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
36
26
|
const loadHtml = async (url, context) => {
|
|
37
27
|
let _browser = null;
|
|
38
28
|
let _context;
|
|
39
|
-
if (
|
|
40
|
-
_browser = await playwright.chromium.launch({ headless: true });
|
|
41
|
-
_context = await _browser.newContext();
|
|
42
|
-
} else
|
|
29
|
+
if (context) {
|
|
43
30
|
_context = context;
|
|
31
|
+
} else {
|
|
32
|
+
_browser = await import_playwright.chromium.launch({ headless: true });
|
|
33
|
+
_context = await _browser.newContext();
|
|
34
|
+
}
|
|
44
35
|
const page = await _context.newPage();
|
|
45
36
|
await page.goto(url);
|
|
46
37
|
await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
|
|
47
38
|
const html = await page.content();
|
|
48
39
|
await page.close();
|
|
49
|
-
if (_browser)
|
|
40
|
+
if (_browser) {
|
|
50
41
|
await _browser.close();
|
|
42
|
+
}
|
|
51
43
|
return html;
|
|
52
44
|
};
|
|
53
45
|
// Annotate the CommonJS export names for ESM import in node:
|
package/dist/cjs/md-splitter.js
CHANGED
|
@@ -26,14 +26,16 @@ var import_common = require("./utils/common");
|
|
|
26
26
|
var import_mdast_utils = require("./utils/mdast-utils");
|
|
27
27
|
const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
|
|
28
28
|
const getSplitterGenerator = function* () {
|
|
29
|
-
for (const splitter of PRIORITY_SPLITTERS)
|
|
29
|
+
for (const splitter of PRIORITY_SPLITTERS) {
|
|
30
30
|
yield splitter;
|
|
31
|
+
}
|
|
31
32
|
};
|
|
32
33
|
const _mdastSplitter = async (contents, checker, splitterGenerator) => {
|
|
33
34
|
const splitter = splitterGenerator.next().value;
|
|
34
35
|
const markdown = (0, import_mdast_to_markdown.mdastToMarkdown)((0, import_mdast_utils.warpRoot)(contents));
|
|
35
|
-
if (await checker(markdown) || contents.length === 1)
|
|
36
|
+
if (await checker(markdown) || contents.length === 1) {
|
|
36
37
|
return [contents];
|
|
38
|
+
}
|
|
37
39
|
const chunked = splitter ? contents.reduce((acc, content) => {
|
|
38
40
|
if ((0, import_mdast_utils.internalType)(content) === splitter || acc.length === 0) {
|
|
39
41
|
acc.push([content]);
|
|
@@ -45,7 +47,7 @@ const _mdastSplitter = async (contents, checker, splitterGenerator) => {
|
|
|
45
47
|
const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
|
|
46
48
|
return Promise.all(splitting).then((chunks) => chunks.flat());
|
|
47
49
|
};
|
|
48
|
-
const mdastSplitter =
|
|
50
|
+
const mdastSplitter = (mdast, checker, _options) => {
|
|
49
51
|
return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, getSplitterGenerator());
|
|
50
52
|
};
|
|
51
53
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -30,8 +30,9 @@ const customAHandler = (options) => (state, node) => {
|
|
|
30
30
|
return link2;
|
|
31
31
|
}
|
|
32
32
|
const link = import_hast_util_to_mdast.defaultHandlers.a(state, node);
|
|
33
|
-
if (link.children.length > 0)
|
|
33
|
+
if (link.children.length > 0) {
|
|
34
34
|
return link;
|
|
35
|
+
}
|
|
35
36
|
return void 0;
|
|
36
37
|
};
|
|
37
38
|
// Annotate the CommonJS export names for ESM import in node:
|