webforai 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +18 -14
- package/dist/cjs/constants.js +3 -8
- package/dist/cjs/extractors/pipeExtractors.js +0 -3
- package/dist/cjs/extractors/presets/minimal-filter.js +87 -0
- package/dist/cjs/extractors/presets/takumi.js +8 -6
- package/dist/cjs/loaders/playwright.js +8 -0
- package/dist/constants.js +2 -6
- package/dist/extractors/pipeExtractors.js +1 -4
- package/dist/extractors/presets/minimal-filter.js +64 -0
- package/dist/extractors/presets/takumi.js +8 -6
- package/dist/loaders/playwright.js +8 -0
- package/dist/types/constants.d.ts +1 -4
- package/dist/types/constants.js.map +1 -1
- package/dist/types/extractors/index.d.ts +1 -1
- package/dist/types/extractors/index.js.map +1 -1
- package/dist/types/extractors/pipeExtractors.d.ts +1 -3
- package/dist/types/extractors/pipeExtractors.js.map +1 -1
- package/dist/types/extractors/presets/minimal-filter.d.ts +9 -0
- package/dist/types/extractors/presets/minimal-filter.js.map +1 -0
- package/dist/types/extractors/presets/takumi.js.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/loaders/playwright.js.map +1 -1
- package/package.json +1 -1
package/dist/bin.js
CHANGED
|
@@ -6,7 +6,7 @@ import { program } from "commander";
|
|
|
6
6
|
// package.json
|
|
7
7
|
var package_default = {
|
|
8
8
|
name: "webforai",
|
|
9
|
-
version: "2.0
|
|
9
|
+
version: "2.1.0",
|
|
10
10
|
description: "A library that provides a web interface for AI",
|
|
11
11
|
author: "inaridiy",
|
|
12
12
|
license: "Apache-2.0",
|
|
@@ -205,15 +205,14 @@ var REGEXPS = {
|
|
|
205
205
|
hidden: /hidden|invisible|fallback-image/i,
|
|
206
206
|
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
207
207
|
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
|
|
208
|
-
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|
|
|
208
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
|
|
209
209
|
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
210
210
|
};
|
|
211
211
|
var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
|
|
212
212
|
var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
|
|
213
|
-
var
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
};
|
|
213
|
+
var CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
|
|
214
|
+
var TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
|
|
215
|
+
var BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
|
|
217
216
|
var metadataFilter = (node) => {
|
|
218
217
|
return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
|
|
219
218
|
};
|
|
@@ -253,7 +252,7 @@ var unlikelyElementFilter = (node) => {
|
|
|
253
252
|
return true;
|
|
254
253
|
}
|
|
255
254
|
const element = node;
|
|
256
|
-
if (
|
|
255
|
+
if (CONTENTABLE_TAGS.includes(element.tagName)) {
|
|
257
256
|
return true;
|
|
258
257
|
}
|
|
259
258
|
const match = matchString(element);
|
|
@@ -276,6 +275,9 @@ var removeEmptyFilter = (node, _lang) => {
|
|
|
276
275
|
if (element.tagName === "img" && !element.properties.src) {
|
|
277
276
|
return false;
|
|
278
277
|
}
|
|
278
|
+
if (TEXTABLE_TAGS.includes(element.tagName) && hastToString(element).length === 0) {
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
279
281
|
return true;
|
|
280
282
|
};
|
|
281
283
|
var takumiExtractor = (params) => {
|
|
@@ -327,10 +329,7 @@ var takumiExtractor = (params) => {
|
|
|
327
329
|
};
|
|
328
330
|
|
|
329
331
|
// src/constants.ts
|
|
330
|
-
var
|
|
331
|
-
takumi: takumiExtractor
|
|
332
|
-
};
|
|
333
|
-
var DEFAULT_EXTRACTORS = ["takumi"];
|
|
332
|
+
var DEFAULT_EXTRACTORS = [takumiExtractor];
|
|
334
333
|
|
|
335
334
|
// src/extractors/pipeExtractors.ts
|
|
336
335
|
var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
|
|
@@ -340,9 +339,6 @@ var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
|
|
|
340
339
|
if (extractor === false) {
|
|
341
340
|
return acc;
|
|
342
341
|
}
|
|
343
|
-
if (typeof extractor === "string" && extractor in PRESET_EXTRACTORS) {
|
|
344
|
-
return PRESET_EXTRACTORS[extractor]({ hast: acc, lang });
|
|
345
|
-
}
|
|
346
342
|
if (typeof extractor === "function") {
|
|
347
343
|
return extractor({ hast: acc, lang });
|
|
348
344
|
}
|
|
@@ -887,6 +883,14 @@ var loadHtml2 = async (url, options) => {
|
|
|
887
883
|
});
|
|
888
884
|
}
|
|
889
885
|
await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
|
|
886
|
+
await page.evaluate(() => {
|
|
887
|
+
const elements = document.querySelectorAll("*");
|
|
888
|
+
for (const element of elements) {
|
|
889
|
+
const rect = element.getBoundingClientRect();
|
|
890
|
+
element.setAttribute("data-rwidth", rect.width.toString());
|
|
891
|
+
element.setAttribute("data-rheight", rect.height.toString());
|
|
892
|
+
}
|
|
893
|
+
});
|
|
890
894
|
const html = await page.content();
|
|
891
895
|
await page.close();
|
|
892
896
|
if (browser) {
|
package/dist/cjs/constants.js
CHANGED
|
@@ -18,17 +18,12 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
18
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
19
|
var constants_exports = {};
|
|
20
20
|
__export(constants_exports, {
|
|
21
|
-
DEFAULT_EXTRACTORS: () => DEFAULT_EXTRACTORS
|
|
22
|
-
PRESET_EXTRACTORS: () => PRESET_EXTRACTORS
|
|
21
|
+
DEFAULT_EXTRACTORS: () => DEFAULT_EXTRACTORS
|
|
23
22
|
});
|
|
24
23
|
module.exports = __toCommonJS(constants_exports);
|
|
25
24
|
var import_takumi = require("./extractors/presets/takumi");
|
|
26
|
-
const
|
|
27
|
-
takumi: import_takumi.takumiExtractor
|
|
28
|
-
};
|
|
29
|
-
const DEFAULT_EXTRACTORS = ["takumi"];
|
|
25
|
+
const DEFAULT_EXTRACTORS = [import_takumi.takumiExtractor];
|
|
30
26
|
// Annotate the CommonJS export names for ESM import in node:
|
|
31
27
|
0 && (module.exports = {
|
|
32
|
-
DEFAULT_EXTRACTORS
|
|
33
|
-
PRESET_EXTRACTORS
|
|
28
|
+
DEFAULT_EXTRACTORS
|
|
34
29
|
});
|
|
@@ -29,9 +29,6 @@ const pipeExtractors = (params, extractors = import_constants.DEFAULT_EXTRACTORS
|
|
|
29
29
|
if (extractor === false) {
|
|
30
30
|
return acc;
|
|
31
31
|
}
|
|
32
|
-
if (typeof extractor === "string" && extractor in import_constants.PRESET_EXTRACTORS) {
|
|
33
|
-
return import_constants.PRESET_EXTRACTORS[extractor]({ hast: acc, lang });
|
|
34
|
-
}
|
|
35
32
|
if (typeof extractor === "function") {
|
|
36
33
|
return extractor({ hast: acc, lang });
|
|
37
34
|
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var minimal_filter_exports = {};
|
|
20
|
+
__export(minimal_filter_exports, {
|
|
21
|
+
minimalFilter: () => minimalFilter
|
|
22
|
+
});
|
|
23
|
+
module.exports = __toCommonJS(minimal_filter_exports);
|
|
24
|
+
var import_hast_util_select = require("hast-util-select");
|
|
25
|
+
var import_hast_util_to_string = require("hast-util-to-string");
|
|
26
|
+
var import_unist_util_filter = require("unist-util-filter");
|
|
27
|
+
var import_utils = require("./utils");
|
|
28
|
+
const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
|
|
29
|
+
const REGEXPS = {
|
|
30
|
+
hidden: /hidden|invisible|fallback-image/i,
|
|
31
|
+
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
32
|
+
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
|
|
33
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
|
|
34
|
+
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
35
|
+
};
|
|
36
|
+
const metadataFilter = (node) => {
|
|
37
|
+
return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
|
|
38
|
+
};
|
|
39
|
+
const universalElementFilter = (node) => {
|
|
40
|
+
if (node.type !== "element") {
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
const element = node;
|
|
44
|
+
if (["aside", "nav"].includes(element.tagName)) {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname))) {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
if (element.tagName === "dialog") {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author")) {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
if (REGEXPS.byline.test((0, import_utils.matchString)(element))) {
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
return true;
|
|
69
|
+
};
|
|
70
|
+
const minimalFilter = (params) => {
|
|
71
|
+
const { hast } = params;
|
|
72
|
+
const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
|
|
73
|
+
const metadataFilteredHast = (0, import_unist_util_filter.filter)(body, (node) => metadataFilter(node));
|
|
74
|
+
const metadataFilteredHastText = metadataFilteredHast && (0, import_hast_util_to_string.toString)(metadataFilteredHast);
|
|
75
|
+
if (!(metadataFilteredHast && metadataFilteredHastText)) {
|
|
76
|
+
return body;
|
|
77
|
+
}
|
|
78
|
+
const baseFilterd = (0, import_unist_util_filter.filter)(metadataFilteredHast, (node) => universalElementFilter(node));
|
|
79
|
+
const baseFilterdText = baseFilterd ? (0, import_hast_util_to_string.toString)(baseFilterd) : "";
|
|
80
|
+
const isOverExtracted = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3;
|
|
81
|
+
const baseTree = isOverExtracted && baseFilterd ? baseFilterd : metadataFilteredHast;
|
|
82
|
+
return baseTree;
|
|
83
|
+
};
|
|
84
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
85
|
+
0 && (module.exports = {
|
|
86
|
+
minimalFilter
|
|
87
|
+
});
|
|
@@ -30,15 +30,14 @@ const REGEXPS = {
|
|
|
30
30
|
hidden: /hidden|invisible|fallback-image/i,
|
|
31
31
|
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
32
32
|
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
|
|
33
|
-
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|
|
|
33
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
|
|
34
34
|
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
35
35
|
};
|
|
36
36
|
const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
|
|
37
37
|
const PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
};
|
|
38
|
+
const CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
|
|
39
|
+
const TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
|
|
40
|
+
const BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
|
|
42
41
|
const metadataFilter = (node) => {
|
|
43
42
|
return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
|
|
44
43
|
};
|
|
@@ -78,7 +77,7 @@ const unlikelyElementFilter = (node) => {
|
|
|
78
77
|
return true;
|
|
79
78
|
}
|
|
80
79
|
const element = node;
|
|
81
|
-
if (
|
|
80
|
+
if (CONTENTABLE_TAGS.includes(element.tagName)) {
|
|
82
81
|
return true;
|
|
83
82
|
}
|
|
84
83
|
const match = (0, import_utils.matchString)(element);
|
|
@@ -101,6 +100,9 @@ const removeEmptyFilter = (node, _lang) => {
|
|
|
101
100
|
if (element.tagName === "img" && !element.properties.src) {
|
|
102
101
|
return false;
|
|
103
102
|
}
|
|
103
|
+
if (TEXTABLE_TAGS.includes(element.tagName) && (0, import_hast_util_to_string.toString)(element).length === 0) {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
104
106
|
return true;
|
|
105
107
|
};
|
|
106
108
|
const takumiExtractor = (params) => {
|
|
@@ -53,6 +53,14 @@ const loadHtml = async (url, options) => {
|
|
|
53
53
|
});
|
|
54
54
|
}
|
|
55
55
|
await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
|
|
56
|
+
await page.evaluate(() => {
|
|
57
|
+
const elements = document.querySelectorAll("*");
|
|
58
|
+
for (const element of elements) {
|
|
59
|
+
const rect = element.getBoundingClientRect();
|
|
60
|
+
element.setAttribute("data-rwidth", rect.width.toString());
|
|
61
|
+
element.setAttribute("data-rheight", rect.height.toString());
|
|
62
|
+
}
|
|
63
|
+
});
|
|
56
64
|
const html = await page.content();
|
|
57
65
|
await page.close();
|
|
58
66
|
if (browser) {
|
package/dist/constants.js
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
// src/constants.ts
|
|
2
2
|
import { takumiExtractor } from "./extractors/presets/takumi.js";
|
|
3
|
-
var
|
|
4
|
-
takumi: takumiExtractor
|
|
5
|
-
};
|
|
6
|
-
var DEFAULT_EXTRACTORS = ["takumi"];
|
|
3
|
+
var DEFAULT_EXTRACTORS = [takumiExtractor];
|
|
7
4
|
export {
|
|
8
|
-
DEFAULT_EXTRACTORS
|
|
9
|
-
PRESET_EXTRACTORS
|
|
5
|
+
DEFAULT_EXTRACTORS
|
|
10
6
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/extractors/pipeExtractors.ts
|
|
2
|
-
import { DEFAULT_EXTRACTORS
|
|
2
|
+
import { DEFAULT_EXTRACTORS } from "../constants.js";
|
|
3
3
|
var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
|
|
4
4
|
const { hast, lang } = params;
|
|
5
5
|
const _extractors = Array.isArray(extractors) ? extractors : [extractors];
|
|
@@ -7,9 +7,6 @@ var pipeExtractors = (params, extractors = DEFAULT_EXTRACTORS) => {
|
|
|
7
7
|
if (extractor === false) {
|
|
8
8
|
return acc;
|
|
9
9
|
}
|
|
10
|
-
if (typeof extractor === "string" && extractor in PRESET_EXTRACTORS) {
|
|
11
|
-
return PRESET_EXTRACTORS[extractor]({ hast: acc, lang });
|
|
12
|
-
}
|
|
13
10
|
if (typeof extractor === "function") {
|
|
14
11
|
return extractor({ hast: acc, lang });
|
|
15
12
|
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// src/extractors/presets/minimal-filter.ts
|
|
2
|
+
import { select } from "hast-util-select";
|
|
3
|
+
import { toString as hastToString } from "hast-util-to-string";
|
|
4
|
+
import { filter } from "unist-util-filter";
|
|
5
|
+
import { classnames, isStrInclude, matchString } from "./utils.js";
|
|
6
|
+
var UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
|
|
7
|
+
var REGEXPS = {
|
|
8
|
+
hidden: /hidden|invisible|fallback-image/i,
|
|
9
|
+
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
10
|
+
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
|
|
11
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
|
|
12
|
+
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
13
|
+
};
|
|
14
|
+
var metadataFilter = (node) => {
|
|
15
|
+
return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
|
|
16
|
+
};
|
|
17
|
+
var universalElementFilter = (node) => {
|
|
18
|
+
if (node.type !== "element") {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
const element = node;
|
|
22
|
+
if (["aside", "nav"].includes(element.tagName)) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
if (element.tagName === "dialog") {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
if (REGEXPS.byline.test(matchString(element))) {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
return true;
|
|
47
|
+
};
|
|
48
|
+
var minimalFilter = (params) => {
|
|
49
|
+
const { hast } = params;
|
|
50
|
+
const body = select("body", hast) ?? hast;
|
|
51
|
+
const metadataFilteredHast = filter(body, (node) => metadataFilter(node));
|
|
52
|
+
const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
|
|
53
|
+
if (!(metadataFilteredHast && metadataFilteredHastText)) {
|
|
54
|
+
return body;
|
|
55
|
+
}
|
|
56
|
+
const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node));
|
|
57
|
+
const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
|
|
58
|
+
const isOverExtracted = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5e3;
|
|
59
|
+
const baseTree = isOverExtracted && baseFilterd ? baseFilterd : metadataFilteredHast;
|
|
60
|
+
return baseTree;
|
|
61
|
+
};
|
|
62
|
+
export {
|
|
63
|
+
minimalFilter
|
|
64
|
+
};
|
|
@@ -8,15 +8,14 @@ var REGEXPS = {
|
|
|
8
8
|
hidden: /hidden|invisible|fallback-image/i,
|
|
9
9
|
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
10
10
|
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
|
|
11
|
-
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|
|
|
11
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
|
|
12
12
|
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
|
|
13
13
|
};
|
|
14
14
|
var BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
|
|
15
15
|
var PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
|
|
16
|
-
var
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
};
|
|
16
|
+
var CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
|
|
17
|
+
var TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
|
|
18
|
+
var BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
|
|
20
19
|
var metadataFilter = (node) => {
|
|
21
20
|
return !(["comment", "doctype"].includes(node.type) || node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName));
|
|
22
21
|
};
|
|
@@ -56,7 +55,7 @@ var unlikelyElementFilter = (node) => {
|
|
|
56
55
|
return true;
|
|
57
56
|
}
|
|
58
57
|
const element = node;
|
|
59
|
-
if (
|
|
58
|
+
if (CONTENTABLE_TAGS.includes(element.tagName)) {
|
|
60
59
|
return true;
|
|
61
60
|
}
|
|
62
61
|
const match = matchString(element);
|
|
@@ -79,6 +78,9 @@ var removeEmptyFilter = (node, _lang) => {
|
|
|
79
78
|
if (element.tagName === "img" && !element.properties.src) {
|
|
80
79
|
return false;
|
|
81
80
|
}
|
|
81
|
+
if (TEXTABLE_TAGS.includes(element.tagName) && hastToString(element).length === 0) {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
82
84
|
return true;
|
|
83
85
|
};
|
|
84
86
|
var takumiExtractor = (params) => {
|
|
@@ -31,6 +31,14 @@ var loadHtml = async (url, options) => {
|
|
|
31
31
|
});
|
|
32
32
|
}
|
|
33
33
|
await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
|
|
34
|
+
await page.evaluate(() => {
|
|
35
|
+
const elements = document.querySelectorAll("*");
|
|
36
|
+
for (const element of elements) {
|
|
37
|
+
const rect = element.getBoundingClientRect();
|
|
38
|
+
element.setAttribute("data-rwidth", rect.width.toString());
|
|
39
|
+
element.setAttribute("data-rheight", rect.height.toString());
|
|
40
|
+
}
|
|
41
|
+
});
|
|
34
42
|
const html = await page.content();
|
|
35
43
|
await page.close();
|
|
36
44
|
if (browser) {
|
|
@@ -1,4 +1 @@
|
|
|
1
|
-
export declare const
|
|
2
|
-
takumi: (params: import("./extractors").ExtractParams) => import("hast").Nodes;
|
|
3
|
-
};
|
|
4
|
-
export declare const DEFAULT_EXTRACTORS: "takumi"[];
|
|
1
|
+
export declare const DEFAULT_EXTRACTORS: ((params: import("./extractors").ExtractParams) => import("hast").Nodes)[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAE9D,MAAM,CAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAE9D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,eAAe,CAAC,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { pipeExtractors, type ExtractorSelectors, type ExtractorSelector,
|
|
1
|
+
export { pipeExtractors, type ExtractorSelectors, type ExtractorSelector, } from "./pipeExtractors";
|
|
2
2
|
export { takumiExtractor } from "./presets/takumi";
|
|
3
3
|
export { type ExtractParams, type Extractor } from "./types";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,cAAc,GAGd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import type { Nodes as Hast } from "hast";
|
|
2
|
-
import { PRESET_EXTRACTORS } from "../constants";
|
|
3
2
|
import type { ExtractParams, Extractor } from "./types";
|
|
4
|
-
export type
|
|
5
|
-
export type ExtractorSelector = Extractor | false | PresetExtractors;
|
|
3
|
+
export type ExtractorSelector = Extractor | false;
|
|
6
4
|
export type ExtractorSelectors = ExtractorSelector | ExtractorSelector[];
|
|
7
5
|
export declare const pipeExtractors: (params: ExtractParams, extractors?: ExtractorSelectors) => Hast;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeExtractors.js","sourceRoot":"","sources":["../../../src/extractors/pipeExtractors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,
|
|
1
|
+
{"version":3,"file":"pipeExtractors.js","sourceRoot":"","sources":["../../../src/extractors/pipeExtractors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAMlD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAqB,EAAE,aAAiC,kBAAkB,EAAQ,EAAE;IAClH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAC9B,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAE1E,MAAM,SAAS,GACd,WAAW,CAAC,MAAM,CAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;QAC3C,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACzB,OAAO,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACrC,OAAO,SAAS,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;IACpD,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAElB,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Nodes as Hast } from "hast";
|
|
2
|
+
import type { ExtractParams } from "../types";
|
|
3
|
+
/**
|
|
4
|
+
* Simple filter to remove unwanted elements from the HAST tree.
|
|
5
|
+
*
|
|
6
|
+
* @param params - {@link ExtractParams}
|
|
7
|
+
* @returns The HAST tree.
|
|
8
|
+
*/
|
|
9
|
+
export declare const minimalFilter: (params: ExtractParams) => Hast;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minimal-filter.js","sourceRoot":"","sources":["../../../../src/extractors/presets/minimal-filter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEhE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G;;;;;;GAMG;AACH,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,iRAAiR;IAClR,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE;IACrC,OAAO,CAAC,CACP,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAC1C,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CACnH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,MAAqB,EAAQ,EAAE;IAC5D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IACxB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAY,CAAC,CAAC,CAAC;IAClF,MAAM,wBAAwB,GAAG,oBAAoB,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC5F,IAAI,CAAC,CAAC,oBAAoB,IAAI,wBAAwB,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,oBAAoB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAY,CAAC,CAAC,CAAC;IACjG,MAAM,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAErE,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,GAAG,wBAAwB,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC;IACtH,MAAM,QAAQ,GAAG,eAAe,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAErF,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"takumi.js","sourceRoot":"","sources":["../../../../src/extractors/presets/takumi.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEhE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G;;;;;;GAMG;AACH,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,
|
|
1
|
+
{"version":3,"file":"takumi.js","sourceRoot":"","sources":["../../../../src/extractors/presets/takumi.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEhE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G;;;;;;GAMG;AACH,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,iRAAiR;IAClR,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEvG,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEzF,MAAM,gBAAgB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;AAEjG,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAE5E,MAAM,mBAAmB,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;AAEjD,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE;IACrC,OAAO,CAAC,CACP,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAC1C,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CACnH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,KAAa,EAAE,EAAE;IACvD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACd,CAAC;IAED,IAAI,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAqB,EAAQ,EAAE;IAC9D,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAY,CAAC,CAAC,CAAC;IAClF,MAAM,wBAAwB,GAAG,oBAAoB,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC5F,IAAI,CAAC,CAAC,oBAAoB,IAAI,wBAAwB,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,oBAAoB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAY,CAAC,CAAC,CAAC;IACjG,MAAM,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GACzB,eAAe,CAAC,MAAM,GAAG,wBAAwB,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI;QAC5F,CAAC,CAAE,CAAC,WAAmB,EAAE,eAAe,CAAW;QACnD,CAAC,CAAE,CAAC,oBAA4B,EAAE,wBAAwB,CAAW,CAAC;IAExE,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,aAAa,GAAS,QAAQ,CAAC;IACnC,IAAI,aAAa,GAAG,QAAQ,CAAC;IAE7B,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACtF,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC7B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;QACzD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACxC,aAAa,GAAG,OAAO,CAAC;YACxB,aAAa,GAAG,WAAW,CAAC;YAC5B,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE;QACxD,IAAI,CAAC,iBAAiB,CAAC,IAAY,EAAE,IAAI,CAAC,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,MAAM,SAAS,GACd,YAAY,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC;IACvG,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -2,4 +2,4 @@ export { htmlToMarkdown, type HtmlToMarkdownOptions } from "./html-to-markdown";
|
|
|
2
2
|
export { mdastSplitter } from "./md-splitter";
|
|
3
3
|
export { htmlToMdast, type HtmlToMdastOptions } from "./html-to-mdast";
|
|
4
4
|
export { mdastToMarkdown } from "./mdast-to-markdown";
|
|
5
|
-
export { pipeExtractors, takumiExtractor, type ExtractorSelectors, type ExtractorSelector, type
|
|
5
|
+
export { pipeExtractors, takumiExtractor, type ExtractorSelectors, type ExtractorSelector, type ExtractParams, type Extractor, } from "./extractors";
|
package/dist/types/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EAAE,cAAc,EAA8B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EACN,cAAc,EACd,eAAe,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EAAE,cAAc,EAA8B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EACN,cAAc,EACd,eAAe,GAKf,MAAM,cAAc,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,QAAQ,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AASlE,MAAM,mBAAmB,GAAG;IAC3B,SAAS,EACR,qHAAqH;IACtH,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACvC,iBAAiB,EAAE,CAAC;IACpB,QAAQ,EAAE,KAAK;IACf,QAAQ,EAAE,KAAK;IACf,iBAAiB,EAAE,IAAI;IACvB,MAAM,EAAE,OAAO;IACf,UAAU,EAAE,kBAAkB;CAC9B,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAyB,EAAE,EAAE;IACxE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IACvE,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAE7G,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE;YAChC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9E,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBACxD,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;YACD,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;SAAM,CAAC;QACP,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,QAAQ,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AASlE,MAAM,mBAAmB,GAAG;IAC3B,SAAS,EACR,qHAAqH;IACtH,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACvC,iBAAiB,EAAE,CAAC;IACpB,QAAQ,EAAE,KAAK;IACf,QAAQ,EAAE,KAAK;IACf,iBAAiB,EAAE,IAAI;IACvB,MAAM,EAAE,OAAO;IACf,UAAU,EAAE,kBAAkB;CAC9B,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAyB,EAAE,EAAE;IACxE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IACvE,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAE7G,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE;YAChC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9E,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBACxD,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;YACD,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAClE,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACxB,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAC7C,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC3D,OAAO,CAAC,YAAY,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9D,CAAC;IACF,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;SAAM,CAAC;QACP,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
|