webforai 1.2.0 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/extract-hast/index.js +35 -3
- package/dist/cjs/extract-hast/readability.js +57 -29
- package/dist/cjs/extract-hast/strongly.js +6 -3
- package/dist/cjs/extract-hast/utils.js +8 -4
- package/dist/cjs/html-to-mdast.js +6 -19
- package/dist/cjs/index.js +18 -9
- package/dist/cjs/link-replacer.js +2 -1
- package/dist/cjs/loaders/playwright.js +8 -16
- package/dist/cjs/md-splitter.js +5 -3
- package/dist/cjs/mdast-handlers/custom-a-handler.js +2 -1
- package/dist/cjs/mdast-handlers/custom-code-handler.js +43 -0
- package/dist/cjs/mdast-handlers/custom-div-handler.js +15 -9
- package/dist/cjs/mdast-handlers/custom-img-handler.js +2 -1
- package/dist/cjs/mdast-handlers/empty-handler.js +29 -0
- package/dist/cjs/mdast-to-markdown.js +2 -1
- package/dist/cjs/utils/mdast-utils.js +6 -3
- package/dist/extract-hast/index.js +28 -1
- package/dist/extract-hast/readability.js +62 -34
- package/dist/extract-hast/strongly.js +6 -3
- package/dist/extract-hast/utils.js +8 -4
- package/dist/html-to-mdast.js +7 -20
- package/dist/index.js +10 -4
- package/dist/link-replacer.js +2 -1
- package/dist/loaders/playwright.js +8 -6
- package/dist/md-splitter.js +5 -3
- package/dist/mdast-handlers/custom-a-handler.js +4 -3
- package/dist/mdast-handlers/custom-code-handler.js +20 -0
- package/dist/mdast-handlers/custom-div-handler.js +17 -11
- package/dist/mdast-handlers/custom-img-handler.js +2 -1
- package/dist/mdast-handlers/empty-handler.js +6 -0
- package/dist/mdast-to-markdown.js +2 -1
- package/dist/types/extract-hast/index.d.ts +10 -1
- package/dist/types/extract-hast/index.js.map +1 -1
- package/dist/types/extract-hast/readability.js.map +1 -1
- package/dist/types/extract-hast/strongly.js.map +1 -1
- package/dist/types/extract-hast/utils.js.map +1 -1
- package/dist/types/html-to-markdown.d.ts +1 -1
- package/dist/types/html-to-markdown.js.map +1 -1
- package/dist/types/html-to-mdast.d.ts +2 -4
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/index.d.ts +4 -4
- package/dist/types/index.js.map +1 -1
- package/dist/types/link-replacer.js.map +1 -1
- package/dist/types/loaders/playwright.d.ts +2 -2
- package/dist/types/loaders/playwright.js.map +1 -1
- package/dist/types/md-splitter.d.ts +1 -1
- package/dist/types/md-splitter.js.map +1 -1
- package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -0
- package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
- package/dist/types/mdast-handlers/empty-handler.d.ts +1 -0
- package/dist/types/mdast-handlers/empty-handler.js.map +1 -0
- package/dist/types/mdast-to-markdown.js.map +1 -1
- package/dist/types/utils/mdast-utils.js.map +1 -1
- package/dist/utils/mdast-utils.js +6 -3
- package/package.json +2 -2
|
@@ -3,6 +3,10 @@ var __defProp = Object.defineProperty;
|
|
|
3
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
6
10
|
var __copyProps = (to, from, except, desc) => {
|
|
7
11
|
if (from && typeof from === "object" || typeof from === "function") {
|
|
8
12
|
for (let key of __getOwnPropNames(from))
|
|
@@ -11,12 +15,40 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
11
15
|
}
|
|
12
16
|
return to;
|
|
13
17
|
};
|
|
14
|
-
var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
|
|
15
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
16
19
|
var extract_hast_exports = {};
|
|
20
|
+
__export(extract_hast_exports, {
|
|
21
|
+
DEFAULT_EXTRACT_HAST: () => DEFAULT_EXTRACT_HAST,
|
|
22
|
+
PRESET_EXTRACT_HAST: () => PRESET_EXTRACT_HAST,
|
|
23
|
+
extractHast: () => extractHast
|
|
24
|
+
});
|
|
17
25
|
module.exports = __toCommonJS(extract_hast_exports);
|
|
18
|
-
|
|
26
|
+
var import_readability = require("./readability");
|
|
27
|
+
var import_strongly = require("./strongly");
|
|
28
|
+
const PRESET_EXTRACT_HAST = {
|
|
29
|
+
strongly: import_strongly.stronglyExtractHast,
|
|
30
|
+
readability: import_readability.readabilityExtractHast
|
|
31
|
+
};
|
|
32
|
+
const DEFAULT_EXTRACT_HAST = ["readability"];
|
|
33
|
+
const extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
|
|
34
|
+
const _extractors = Array.isArray(extractors) ? extractors : [extractors];
|
|
35
|
+
const extracted = _extractors.reduce((acc, extractor) => {
|
|
36
|
+
if (extractor === false) {
|
|
37
|
+
return acc;
|
|
38
|
+
}
|
|
39
|
+
if (typeof extractor === "string" && extractor in DEFAULT_EXTRACT_HAST) {
|
|
40
|
+
return PRESET_EXTRACT_HAST[extractor](acc);
|
|
41
|
+
}
|
|
42
|
+
if (typeof extractor === "function") {
|
|
43
|
+
return extractor(acc);
|
|
44
|
+
}
|
|
45
|
+
throw new Error(`Invalid extractor: ${extractor}`);
|
|
46
|
+
}, hast) || hast;
|
|
47
|
+
return extracted;
|
|
48
|
+
};
|
|
19
49
|
// Annotate the CommonJS export names for ESM import in node:
|
|
20
50
|
0 && (module.exports = {
|
|
21
|
-
|
|
51
|
+
DEFAULT_EXTRACT_HAST,
|
|
52
|
+
PRESET_EXTRACT_HAST,
|
|
53
|
+
extractHast
|
|
22
54
|
});
|
|
@@ -31,7 +31,7 @@ const REGEXPS = {
|
|
|
31
31
|
hidden: /hidden|invisible|fallback-image/i,
|
|
32
32
|
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
33
33
|
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
|
|
34
|
-
|
|
34
|
+
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
35
35
|
};
|
|
36
36
|
const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
|
|
37
37
|
const PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
|
|
@@ -40,60 +40,79 @@ const BASE_MINIMAL_LENGTH = {
|
|
|
40
40
|
en: 500
|
|
41
41
|
};
|
|
42
42
|
const tryGetLang = (node) => {
|
|
43
|
-
if (node.type !== "element")
|
|
43
|
+
if (node.type !== "element") {
|
|
44
44
|
return;
|
|
45
|
+
}
|
|
45
46
|
const element = node;
|
|
46
|
-
if (element.tagName !== "html")
|
|
47
|
+
if (element.tagName !== "html") {
|
|
47
48
|
return;
|
|
49
|
+
}
|
|
48
50
|
const langAttr = element.properties.lang || element.properties["xml:lang"];
|
|
49
|
-
if (langAttr)
|
|
51
|
+
if (langAttr) {
|
|
50
52
|
return langAttr;
|
|
53
|
+
}
|
|
51
54
|
};
|
|
52
55
|
const metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
|
|
53
56
|
const universalElementFilter = (node) => {
|
|
54
|
-
if (node.type !== "element")
|
|
57
|
+
if (node.type !== "element") {
|
|
55
58
|
return true;
|
|
59
|
+
}
|
|
56
60
|
const element = node;
|
|
57
|
-
if (["script", "aside"].includes(element.tagName))
|
|
61
|
+
if (["script", "aside"].includes(element.tagName)) {
|
|
58
62
|
return false;
|
|
59
|
-
|
|
63
|
+
}
|
|
64
|
+
if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
|
|
60
65
|
return false;
|
|
61
|
-
|
|
66
|
+
}
|
|
67
|
+
if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname))) {
|
|
62
68
|
return false;
|
|
63
|
-
|
|
69
|
+
}
|
|
70
|
+
if (element.tagName === "dialog") {
|
|
64
71
|
return false;
|
|
65
|
-
|
|
72
|
+
}
|
|
73
|
+
if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
|
|
66
74
|
return false;
|
|
67
|
-
|
|
75
|
+
}
|
|
76
|
+
if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author")) {
|
|
68
77
|
return false;
|
|
69
|
-
|
|
78
|
+
}
|
|
79
|
+
if (REGEXPS.byline.test((0, import_utils.matchString)(element))) {
|
|
70
80
|
return false;
|
|
71
|
-
|
|
81
|
+
}
|
|
82
|
+
if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
|
|
72
83
|
return false;
|
|
84
|
+
}
|
|
73
85
|
return true;
|
|
74
86
|
};
|
|
75
87
|
const unlikelyElementFilter = (node) => {
|
|
76
|
-
if (node.type !== "element")
|
|
88
|
+
if (node.type !== "element") {
|
|
77
89
|
return true;
|
|
90
|
+
}
|
|
78
91
|
const element = node;
|
|
79
92
|
const match = (0, import_utils.matchString)(element);
|
|
80
|
-
if (["body", "article", "main", "section", "a"].includes(element.tagName))
|
|
93
|
+
if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
|
|
81
94
|
return true;
|
|
82
|
-
|
|
95
|
+
}
|
|
96
|
+
if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3)) {
|
|
83
97
|
return true;
|
|
84
|
-
|
|
98
|
+
}
|
|
99
|
+
if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
|
|
85
100
|
return false;
|
|
101
|
+
}
|
|
86
102
|
return true;
|
|
87
103
|
};
|
|
88
104
|
const removeEmptyFilter = (node) => {
|
|
89
|
-
if (node.type !== "element")
|
|
105
|
+
if (node.type !== "element") {
|
|
90
106
|
return true;
|
|
107
|
+
}
|
|
91
108
|
const element = node;
|
|
92
|
-
if (!PARAGRAPH_TAGS.includes(element.tagName))
|
|
109
|
+
if (!PARAGRAPH_TAGS.includes(element.tagName)) {
|
|
93
110
|
return true;
|
|
111
|
+
}
|
|
94
112
|
const text = (0, import_hast_util_to_string.toString)(element);
|
|
95
|
-
if (text.length < 10)
|
|
113
|
+
if (text.length < 10) {
|
|
96
114
|
return false;
|
|
115
|
+
}
|
|
97
116
|
return true;
|
|
98
117
|
};
|
|
99
118
|
const readabilityExtractHast = (hast) => {
|
|
@@ -101,45 +120,54 @@ const readabilityExtractHast = (hast) => {
|
|
|
101
120
|
const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
|
|
102
121
|
const proxiedHast = (0, import_unist_util_parents.parents)(body);
|
|
103
122
|
let baseFilterd = (0, import_unist_util_filter.filter)(proxiedHast, (node) => {
|
|
104
|
-
if (!metadataFilter(node))
|
|
123
|
+
if (!metadataFilter(node)) {
|
|
105
124
|
return false;
|
|
106
|
-
|
|
125
|
+
}
|
|
126
|
+
if (!universalElementFilter(node)) {
|
|
107
127
|
return false;
|
|
128
|
+
}
|
|
108
129
|
return true;
|
|
109
130
|
});
|
|
110
|
-
if (!baseFilterd)
|
|
131
|
+
if (!baseFilterd) {
|
|
111
132
|
return { type: "root", children: [] };
|
|
133
|
+
}
|
|
112
134
|
const baseText = (0, import_hast_util_to_string.toString)(baseFilterd);
|
|
113
135
|
let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
|
|
114
136
|
if (baseText.length > minimalLength) {
|
|
115
137
|
const filterd = (0, import_unist_util_filter.filter)(baseFilterd, (node) => {
|
|
116
|
-
if (!unlikelyElementFilter(node))
|
|
138
|
+
if (!unlikelyElementFilter(node)) {
|
|
117
139
|
return false;
|
|
140
|
+
}
|
|
118
141
|
return true;
|
|
119
142
|
});
|
|
120
|
-
if (filterd)
|
|
143
|
+
if (filterd) {
|
|
121
144
|
baseFilterd = filterd;
|
|
122
|
-
|
|
145
|
+
}
|
|
146
|
+
} else {
|
|
123
147
|
minimalLength = Math.max(0, baseText.length - 200);
|
|
148
|
+
}
|
|
124
149
|
let bodyTree = baseFilterd;
|
|
125
150
|
for (const selector of BODY_SELECTORS) {
|
|
126
151
|
const body2 = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
|
|
127
152
|
const bodyText = (0, import_hast_util_to_string.toString)(body2);
|
|
128
|
-
if (bodyText.length < 25)
|
|
153
|
+
if (bodyText.length < 25) {
|
|
129
154
|
continue;
|
|
155
|
+
}
|
|
130
156
|
const links = (0, import_hast_util_select.selectAll)("a", body2);
|
|
131
157
|
const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
|
|
132
158
|
const linkDensity = linkText.length / bodyText.length;
|
|
133
|
-
if (linkDensity > 0.4)
|
|
159
|
+
if (linkDensity > 0.4) {
|
|
134
160
|
continue;
|
|
161
|
+
}
|
|
135
162
|
if (bodyText.length > minimalLength) {
|
|
136
163
|
bodyTree = body2;
|
|
137
164
|
break;
|
|
138
165
|
}
|
|
139
166
|
}
|
|
140
167
|
const finalTree = (0, import_unist_util_filter.filter)(bodyTree, (node) => {
|
|
141
|
-
if (!removeEmptyFilter(node))
|
|
168
|
+
if (!removeEmptyFilter(node)) {
|
|
142
169
|
return false;
|
|
170
|
+
}
|
|
143
171
|
return true;
|
|
144
172
|
});
|
|
145
173
|
return finalTree;
|
|
@@ -35,14 +35,17 @@ const stronglyExtractHast = (hast) => {
|
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
37
|
const removedElements = (0, import_unist_util_filter.filter)(selectedElement, (node) => {
|
|
38
|
-
if (node.type === "comment")
|
|
38
|
+
if (node.type === "comment") {
|
|
39
39
|
return false;
|
|
40
|
-
|
|
40
|
+
}
|
|
41
|
+
if (node.type !== "element") {
|
|
41
42
|
return true;
|
|
43
|
+
}
|
|
42
44
|
const element = node;
|
|
43
45
|
const elementName = `${element.tagName} ${element.properties.id} ${element.properties.className}`;
|
|
44
|
-
if (REMOVE_REGEX.test(elementName))
|
|
46
|
+
if (REMOVE_REGEX.test(elementName)) {
|
|
45
47
|
return false;
|
|
48
|
+
}
|
|
46
49
|
return true;
|
|
47
50
|
});
|
|
48
51
|
return removedElements;
|
|
@@ -26,22 +26,26 @@ __export(utils_exports, {
|
|
|
26
26
|
module.exports = __toCommonJS(utils_exports);
|
|
27
27
|
const matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")} }`;
|
|
28
28
|
const classnames = (element) => {
|
|
29
|
-
if (Array.isArray(element.properties.className))
|
|
29
|
+
if (Array.isArray(element.properties.className)) {
|
|
30
30
|
return element.properties.className;
|
|
31
|
+
}
|
|
31
32
|
return [];
|
|
32
33
|
};
|
|
33
34
|
const isStrInclude = (value, match) => {
|
|
34
|
-
if (typeof value === "string")
|
|
35
|
+
if (typeof value === "string") {
|
|
35
36
|
return value.includes(match);
|
|
37
|
+
}
|
|
36
38
|
return false;
|
|
37
39
|
};
|
|
38
40
|
const hasAncestors = (element, tagNames, depth = 3) => {
|
|
39
41
|
let parent = element.parent;
|
|
40
42
|
for (let i = 0; i < depth; i++) {
|
|
41
|
-
if (!parent || parent.type !== "element")
|
|
43
|
+
if (!parent || parent.type !== "element") {
|
|
42
44
|
return false;
|
|
43
|
-
|
|
45
|
+
}
|
|
46
|
+
if (tagNames.includes(parent.tagName)) {
|
|
44
47
|
return true;
|
|
48
|
+
}
|
|
45
49
|
parent = parent.parent;
|
|
46
50
|
}
|
|
47
51
|
return false;
|
|
@@ -24,39 +24,26 @@ module.exports = __toCommonJS(html_to_mdast_exports);
|
|
|
24
24
|
var import_hast_util_from_html = require("hast-util-from-html");
|
|
25
25
|
var import_hast_util_to_mdast = require("hast-util-to-mdast");
|
|
26
26
|
var import_extract_hast = require("./extract-hast");
|
|
27
|
-
var import_readability = require("./extract-hast/readability");
|
|
28
27
|
var import_custom_a_handler = require("./mdast-handlers/custom-a-handler");
|
|
28
|
+
var import_custom_code_handler = require("./mdast-handlers/custom-code-handler");
|
|
29
29
|
var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
|
|
30
30
|
var import_custom_img_handler = require("./mdast-handlers/custom-img-handler");
|
|
31
31
|
var import_custom_table_handler = require("./mdast-handlers/custom-table-handler");
|
|
32
|
+
var import_empty_handler = require("./mdast-handlers/empty-handler");
|
|
32
33
|
var import_math_handler = require("./mdast-handlers/math-handler");
|
|
33
|
-
const PRESET_EXTRACT_HAST = {
|
|
34
|
-
strongly: import_extract_hast.stronglyExtractHast,
|
|
35
|
-
readability: import_readability.readabilityExtractHast
|
|
36
|
-
};
|
|
37
|
-
const DEFAULT_EXTRACT_HAST = ["readability"];
|
|
38
34
|
const htmlToMdast = (html, options) => {
|
|
39
|
-
const { extractHast
|
|
40
|
-
const _extractHast = Array.isArray(extractHast) ? extractHast : [extractHast];
|
|
35
|
+
const { extractHast: _extractHast } = options || {};
|
|
41
36
|
const hast = (0, import_hast_util_from_html.fromHtml)(html, { fragment: true });
|
|
42
|
-
const extractedHast =
|
|
43
|
-
if (extractHast2 === false)
|
|
44
|
-
return acc;
|
|
45
|
-
if (typeof extractHast2 === "string" && extractHast2 in PRESET_EXTRACT_HAST)
|
|
46
|
-
return PRESET_EXTRACT_HAST[extractHast2](acc);
|
|
47
|
-
if (typeof extractHast2 === "function")
|
|
48
|
-
return extractHast2(acc);
|
|
49
|
-
throw new Error(`Invalid extractHast: ${extractHast2}`);
|
|
50
|
-
}, hast) || hast;
|
|
37
|
+
const extractedHast = (0, import_extract_hast.extractHast)(hast, _extractHast);
|
|
51
38
|
const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
|
|
52
39
|
handlers: {
|
|
53
40
|
math: import_math_handler.mathHandler,
|
|
54
41
|
div: import_custom_div_handler.customDivHandler,
|
|
42
|
+
pre: import_custom_code_handler.customCodeHandler,
|
|
55
43
|
a: (0, import_custom_a_handler.customAHandler)({ asText: options?.linkAsText }),
|
|
56
44
|
img: (0, import_custom_img_handler.customImgHandler)({ hideImage: options?.hideImage }),
|
|
57
45
|
table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText }),
|
|
58
|
-
br:
|
|
59
|
-
}
|
|
46
|
+
br: import_empty_handler.emptyHandler
|
|
60
47
|
}
|
|
61
48
|
});
|
|
62
49
|
return mdast;
|
package/dist/cjs/index.js
CHANGED
|
@@ -3,6 +3,10 @@ var __defProp = Object.defineProperty;
|
|
|
3
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
6
10
|
var __copyProps = (to, from, except, desc) => {
|
|
7
11
|
if (from && typeof from === "object" || typeof from === "function") {
|
|
8
12
|
for (let key of __getOwnPropNames(from))
|
|
@@ -11,18 +15,23 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
11
15
|
}
|
|
12
16
|
return to;
|
|
13
17
|
};
|
|
14
|
-
var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
|
|
15
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
16
19
|
var src_exports = {};
|
|
20
|
+
__export(src_exports, {
|
|
21
|
+
htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
|
|
22
|
+
htmlToMdast: () => import_html_to_mdast.htmlToMdast,
|
|
23
|
+
mdastSplitter: () => import_md_splitter.mdastSplitter,
|
|
24
|
+
mdastToMarkdown: () => import_mdast_to_markdown.mdastToMarkdown
|
|
25
|
+
});
|
|
17
26
|
module.exports = __toCommonJS(src_exports);
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
27
|
+
var import_html_to_markdown = require("./html-to-markdown");
|
|
28
|
+
var import_md_splitter = require("./md-splitter");
|
|
29
|
+
var import_html_to_mdast = require("./html-to-mdast");
|
|
30
|
+
var import_mdast_to_markdown = require("./mdast-to-markdown");
|
|
22
31
|
// Annotate the CommonJS export names for ESM import in node:
|
|
23
32
|
0 && (module.exports = {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
33
|
+
htmlToMarkdown,
|
|
34
|
+
htmlToMdast,
|
|
35
|
+
mdastSplitter,
|
|
36
|
+
mdastToMarkdown
|
|
28
37
|
});
|
|
@@ -24,8 +24,9 @@ module.exports = __toCommonJS(link_replacer_exports);
|
|
|
24
24
|
const linkReplacer = (markdown, link) => {
|
|
25
25
|
const regex = /(!?\[.*?\]\()(.+?)(\))/g;
|
|
26
26
|
return markdown.replace(regex, (match, pre, thisUrl, post) => {
|
|
27
|
-
if (thisUrl.startsWith("http") || thisUrl.startsWith("#"))
|
|
27
|
+
if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
|
|
28
28
|
return match;
|
|
29
|
+
}
|
|
29
30
|
return `${pre}${new URL(thisUrl, link).href}${post}`;
|
|
30
31
|
});
|
|
31
32
|
};
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
2
|
var __defProp = Object.defineProperty;
|
|
4
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
6
|
var __export = (target, all) => {
|
|
9
7
|
for (var name in all)
|
|
@@ -17,37 +15,31 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
17
15
|
}
|
|
18
16
|
return to;
|
|
19
17
|
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
19
|
var playwright_exports = {};
|
|
30
20
|
__export(playwright_exports, {
|
|
31
21
|
loadHtml: () => loadHtml
|
|
32
22
|
});
|
|
33
23
|
module.exports = __toCommonJS(playwright_exports);
|
|
34
|
-
var
|
|
24
|
+
var import_playwright = require("playwright");
|
|
35
25
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
36
26
|
const loadHtml = async (url, context) => {
|
|
37
27
|
let _browser = null;
|
|
38
28
|
let _context;
|
|
39
|
-
if (
|
|
40
|
-
_browser = await playwright.chromium.launch({ headless: true });
|
|
41
|
-
_context = await _browser.newContext();
|
|
42
|
-
} else
|
|
29
|
+
if (context) {
|
|
43
30
|
_context = context;
|
|
31
|
+
} else {
|
|
32
|
+
_browser = await import_playwright.chromium.launch({ headless: true });
|
|
33
|
+
_context = await _browser.newContext();
|
|
34
|
+
}
|
|
44
35
|
const page = await _context.newPage();
|
|
45
36
|
await page.goto(url);
|
|
46
37
|
await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
|
|
47
38
|
const html = await page.content();
|
|
48
39
|
await page.close();
|
|
49
|
-
if (_browser)
|
|
40
|
+
if (_browser) {
|
|
50
41
|
await _browser.close();
|
|
42
|
+
}
|
|
51
43
|
return html;
|
|
52
44
|
};
|
|
53
45
|
// Annotate the CommonJS export names for ESM import in node:
|
package/dist/cjs/md-splitter.js
CHANGED
|
@@ -26,14 +26,16 @@ var import_common = require("./utils/common");
|
|
|
26
26
|
var import_mdast_utils = require("./utils/mdast-utils");
|
|
27
27
|
const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
|
|
28
28
|
const getSplitterGenerator = function* () {
|
|
29
|
-
for (const splitter of PRIORITY_SPLITTERS)
|
|
29
|
+
for (const splitter of PRIORITY_SPLITTERS) {
|
|
30
30
|
yield splitter;
|
|
31
|
+
}
|
|
31
32
|
};
|
|
32
33
|
const _mdastSplitter = async (contents, checker, splitterGenerator) => {
|
|
33
34
|
const splitter = splitterGenerator.next().value;
|
|
34
35
|
const markdown = (0, import_mdast_to_markdown.mdastToMarkdown)((0, import_mdast_utils.warpRoot)(contents));
|
|
35
|
-
if (await checker(markdown) || contents.length === 1)
|
|
36
|
+
if (await checker(markdown) || contents.length === 1) {
|
|
36
37
|
return [contents];
|
|
38
|
+
}
|
|
37
39
|
const chunked = splitter ? contents.reduce((acc, content) => {
|
|
38
40
|
if ((0, import_mdast_utils.internalType)(content) === splitter || acc.length === 0) {
|
|
39
41
|
acc.push([content]);
|
|
@@ -45,7 +47,7 @@ const _mdastSplitter = async (contents, checker, splitterGenerator) => {
|
|
|
45
47
|
const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
|
|
46
48
|
return Promise.all(splitting).then((chunks) => chunks.flat());
|
|
47
49
|
};
|
|
48
|
-
const mdastSplitter =
|
|
50
|
+
const mdastSplitter = (mdast, checker, _options) => {
|
|
49
51
|
return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, getSplitterGenerator());
|
|
50
52
|
};
|
|
51
53
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -30,8 +30,9 @@ const customAHandler = (options) => (state, node) => {
|
|
|
30
30
|
return link2;
|
|
31
31
|
}
|
|
32
32
|
const link = import_hast_util_to_mdast.defaultHandlers.a(state, node);
|
|
33
|
-
if (link.children.length > 0)
|
|
33
|
+
if (link.children.length > 0) {
|
|
34
34
|
return link;
|
|
35
|
+
}
|
|
35
36
|
return void 0;
|
|
36
37
|
};
|
|
37
38
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var custom_code_handler_exports = {};
|
|
20
|
+
__export(custom_code_handler_exports, {
|
|
21
|
+
customCodeHandler: () => customCodeHandler
|
|
22
|
+
});
|
|
23
|
+
module.exports = __toCommonJS(custom_code_handler_exports);
|
|
24
|
+
var import_hast_util_to_text = require("hast-util-to-text");
|
|
25
|
+
var import_trim_trailing_lines = require("trim-trailing-lines");
|
|
26
|
+
var import_detect_code_lang = require("../utils/detect-code-lang");
|
|
27
|
+
const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
|
|
28
|
+
const customCodeHandler = (state, node) => {
|
|
29
|
+
const classNames = node.properties?.className || [];
|
|
30
|
+
const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(node)).trim();
|
|
31
|
+
const classLang = classNames.map((className) => {
|
|
32
|
+
const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
|
|
33
|
+
return match?.[1];
|
|
34
|
+
}).find((className) => className);
|
|
35
|
+
const lang = classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
|
|
36
|
+
const result = { type: "code", lang, meta: null, value: codeValue };
|
|
37
|
+
state.patch(node, result);
|
|
38
|
+
return result;
|
|
39
|
+
};
|
|
40
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
41
|
+
0 && (module.exports = {
|
|
42
|
+
customCodeHandler
|
|
43
|
+
});
|
|
@@ -29,35 +29,41 @@ var import_trim_trailing_lines = require("trim-trailing-lines");
|
|
|
29
29
|
var import_detect_code_lang = require("../utils/detect-code-lang");
|
|
30
30
|
const CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
|
|
31
31
|
const CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
|
|
32
|
-
const LANGUAGE_MATCH_REGEX = /language-(\w+)
|
|
32
|
+
const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
|
|
33
33
|
const findRecursive = (array, condition, maxDepth = 3) => {
|
|
34
|
-
if (maxDepth <= 0)
|
|
34
|
+
if (maxDepth <= 0) {
|
|
35
35
|
return null;
|
|
36
|
+
}
|
|
36
37
|
for (const value of array) {
|
|
37
38
|
const result = condition(value);
|
|
38
|
-
if (Array.isArray(result))
|
|
39
|
+
if (Array.isArray(result)) {
|
|
39
40
|
return findRecursive(result, condition, maxDepth - 1);
|
|
40
|
-
|
|
41
|
+
}
|
|
42
|
+
if (result) {
|
|
41
43
|
return value;
|
|
44
|
+
}
|
|
42
45
|
}
|
|
43
46
|
return null;
|
|
44
47
|
};
|
|
45
48
|
const customDivHandler = (state, node) => {
|
|
46
49
|
const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
|
|
47
50
|
const codeBlock = findRecursive(node.children, (child) => {
|
|
48
|
-
if (child.type !== "element")
|
|
51
|
+
if (child.type !== "element") {
|
|
49
52
|
return false;
|
|
50
|
-
|
|
53
|
+
}
|
|
54
|
+
if (child.tagName === "pre") {
|
|
51
55
|
return true;
|
|
56
|
+
}
|
|
52
57
|
return child.children.filter((child2) => child2.type === "element");
|
|
53
58
|
});
|
|
54
59
|
if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
|
|
60
|
+
const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
|
|
55
61
|
const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(codeBlock)).trim();
|
|
56
62
|
const filenameElement = (0, import_hast_util_select.select)(CODE_FILENAME_SELECTORS, node);
|
|
57
63
|
const fileLang = filenameElement ? (0, import_hast_util_to_string.toString)(filenameElement).match(/\.(\w+)$/)?.[1] : null;
|
|
58
|
-
const classLang = classNames.map((className) => {
|
|
59
|
-
const match = className.match(
|
|
60
|
-
return match?.[1]
|
|
64
|
+
const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
|
|
65
|
+
const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
|
|
66
|
+
return match?.[1];
|
|
61
67
|
}).find((className) => className);
|
|
62
68
|
const lang = fileLang || classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
|
|
63
69
|
const result = { type: "code", lang, meta: null, value: codeValue };
|
|
@@ -23,8 +23,9 @@ __export(custom_img_handler_exports, {
|
|
|
23
23
|
module.exports = __toCommonJS(custom_img_handler_exports);
|
|
24
24
|
var import_hast_util_to_mdast = require("hast-util-to-mdast");
|
|
25
25
|
const customImgHandler = (options) => (state, node) => {
|
|
26
|
-
if (options?.hideImage)
|
|
26
|
+
if (options?.hideImage) {
|
|
27
27
|
return void 0;
|
|
28
|
+
}
|
|
28
29
|
return import_hast_util_to_mdast.defaultHandlers.image(state, node);
|
|
29
30
|
};
|
|
30
31
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var empty_handler_exports = {};
|
|
20
|
+
__export(empty_handler_exports, {
|
|
21
|
+
emptyHandler: () => emptyHandler
|
|
22
|
+
});
|
|
23
|
+
module.exports = __toCommonJS(empty_handler_exports);
|
|
24
|
+
const emptyHandler = () => {
|
|
25
|
+
};
|
|
26
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
27
|
+
0 && (module.exports = {
|
|
28
|
+
emptyHandler
|
|
29
|
+
});
|
|
@@ -30,8 +30,9 @@ const mdastToMarkdown = (mdast, options) => {
|
|
|
30
30
|
let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), {
|
|
31
31
|
extensions: [(0, import_mdast_util_gfm.gfmToMarkdown)(), (0, import_mdast_util_math.mathToMarkdown)()]
|
|
32
32
|
});
|
|
33
|
-
if (options?.url)
|
|
33
|
+
if (options?.url) {
|
|
34
34
|
markdown = (0, import_link_replacer.linkReplacer)(markdown, options.url);
|
|
35
|
+
}
|
|
35
36
|
return markdown;
|
|
36
37
|
};
|
|
37
38
|
// Annotate the CommonJS export names for ESM import in node:
|
|
@@ -24,18 +24,21 @@ __export(mdast_utils_exports, {
|
|
|
24
24
|
});
|
|
25
25
|
module.exports = __toCommonJS(mdast_utils_exports);
|
|
26
26
|
const unwarpRoot = (mdast) => {
|
|
27
|
-
if (mdast.type === "root")
|
|
27
|
+
if (mdast.type === "root") {
|
|
28
28
|
return mdast.children;
|
|
29
|
+
}
|
|
29
30
|
return [mdast];
|
|
30
31
|
};
|
|
31
32
|
const warpRoot = (mdast) => {
|
|
32
|
-
if (Array.isArray(mdast))
|
|
33
|
+
if (Array.isArray(mdast)) {
|
|
33
34
|
return { type: "root", children: mdast };
|
|
35
|
+
}
|
|
34
36
|
return mdast;
|
|
35
37
|
};
|
|
36
38
|
const internalType = (content) => {
|
|
37
|
-
if (content.type === "heading")
|
|
39
|
+
if (content.type === "heading") {
|
|
38
40
|
return `h${content.depth}`;
|
|
41
|
+
}
|
|
39
42
|
return content.type;
|
|
40
43
|
};
|
|
41
44
|
// Annotate the CommonJS export names for ESM import in node:
|