webforai 4.0.0 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -69
- package/dist/bin.js +1895 -74
- package/dist/cjs/extract-mdast.js +58 -0
- package/dist/cjs/extractors/index.js +5 -0
- package/dist/cjs/extractors/lib/block-features.js +465 -0
- package/dist/cjs/extractors/lib/block-model.generated.js +45 -0
- package/dist/cjs/extractors/lib/block-model.js +72 -0
- package/dist/cjs/extractors/lib/blocks.js +212 -0
- package/dist/cjs/extractors/lib/code-tabs.js +191 -0
- package/dist/cjs/extractors/lib/constants.js +1 -1
- package/dist/cjs/extractors/lib/layout-tables.js +123 -0
- package/dist/cjs/extractors/lib/sanitize.js +144 -28
- package/dist/cjs/extractors/lib/takumi-signal.js +76 -0
- package/dist/cjs/extractors/lib/ui-chrome.js +7 -0
- package/dist/cjs/extractors/presets/auto.js +3 -3
- package/dist/cjs/extractors/presets/kiwame.js +231 -0
- package/dist/cjs/extractors/presets/takumi.js +34 -3
- package/dist/cjs/html-to-markdown.js +58 -4
- package/dist/cjs/html-to-mdast.js +2 -2
- package/dist/cjs/index.js +4 -0
- package/dist/cjs/mdast-handlers/custom-code-handler.js +16 -4
- package/dist/cjs/mdast-handlers/custom-div-handler.js +3 -3
- package/dist/cjs/mdast-handlers/custom-img-handler.js +10 -1
- package/dist/cjs/mdast-handlers/custom-table-handler.js +18 -1
- package/dist/cjs/mdast-handlers/table-cell-flatten.js +195 -0
- package/dist/cjs/mdast-handlers/twoslash.js +71 -0
- package/dist/cjs/metadata/index.js +4 -3
- package/dist/cjs/utils/hast-fast.js +1 -1
- package/dist/cjs/utils/parse-html.js +113 -0
- package/dist/cjs/utils/pre-text.js +104 -0
- package/dist/extract-mdast.js +58 -0
- package/dist/extractors/index.js +3 -0
- package/dist/extractors/lib/block-features.js +437 -0
- package/dist/extractors/lib/block-model.generated.js +21 -0
- package/dist/extractors/lib/block-model.js +48 -0
- package/dist/extractors/lib/blocks.js +185 -0
- package/dist/extractors/lib/code-tabs.js +168 -0
- package/dist/extractors/lib/constants.js +1 -1
- package/dist/extractors/lib/layout-tables.js +99 -0
- package/dist/extractors/lib/sanitize.js +142 -28
- package/dist/extractors/lib/takumi-signal.js +53 -0
- package/dist/extractors/lib/ui-chrome.js +5 -0
- package/dist/extractors/presets/auto.js +3 -3
- package/dist/extractors/presets/kiwame.js +206 -0
- package/dist/extractors/presets/takumi.js +34 -3
- package/dist/html-to-markdown.js +57 -4
- package/dist/html-to-mdast.js +2 -2
- package/dist/index.js +4 -0
- package/dist/mdast-handlers/custom-code-handler.js +15 -4
- package/dist/mdast-handlers/custom-div-handler.js +4 -4
- package/dist/mdast-handlers/custom-img-handler.js +10 -1
- package/dist/mdast-handlers/custom-table-handler.js +19 -2
- package/dist/mdast-handlers/table-cell-flatten.js +170 -0
- package/dist/mdast-handlers/twoslash.js +48 -0
- package/dist/metadata/index.js +4 -3
- package/dist/types/cli/commands/jobs/options.d.ts +2 -2
- package/dist/types/cli/commands/jobs/options.js.map +1 -1
- package/dist/types/cli/commands/webforai/options.js.map +1 -1
- package/dist/types/cli/commands/webforai/run.js.map +1 -1
- package/dist/types/cli/constants.d.ts +3 -1
- package/dist/types/cli/constants.js.map +1 -1
- package/dist/types/extract-mdast.js.map +1 -1
- package/dist/types/extractors/index.d.ts +1 -0
- package/dist/types/extractors/index.js.map +1 -1
- package/dist/types/extractors/lib/block-features.d.ts +43 -0
- package/dist/types/extractors/lib/block-features.js.map +1 -0
- package/dist/types/extractors/lib/block-model.d.ts +28 -0
- package/dist/types/extractors/lib/block-model.generated.d.ts +5 -0
- package/dist/types/extractors/lib/block-model.generated.js.map +1 -0
- package/dist/types/extractors/lib/block-model.js.map +1 -0
- package/dist/types/extractors/lib/blocks.d.ts +53 -0
- package/dist/types/extractors/lib/blocks.js.map +1 -0
- package/dist/types/extractors/lib/code-tabs.d.ts +30 -0
- package/dist/types/extractors/lib/code-tabs.js.map +1 -0
- package/dist/types/extractors/lib/constants.js.map +1 -1
- package/dist/types/extractors/lib/layout-tables.d.ts +17 -0
- package/dist/types/extractors/lib/layout-tables.js.map +1 -0
- package/dist/types/extractors/lib/sanitize.d.ts +33 -3
- package/dist/types/extractors/lib/sanitize.js.map +1 -1
- package/dist/types/extractors/lib/takumi-signal.d.ts +12 -0
- package/dist/types/extractors/lib/takumi-signal.js.map +1 -0
- package/dist/types/extractors/lib/ui-chrome.d.ts +8 -0
- package/dist/types/extractors/lib/ui-chrome.js.map +1 -1
- package/dist/types/extractors/presets/auto.d.ts +6 -1
- package/dist/types/extractors/presets/auto.js.map +1 -1
- package/dist/types/extractors/presets/kiwame.d.ts +31 -0
- package/dist/types/extractors/presets/kiwame.js.map +1 -0
- package/dist/types/extractors/presets/takumi.js.map +1 -1
- package/dist/types/html-to-markdown.d.ts +9 -0
- package/dist/types/html-to-markdown.js.map +1 -1
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/mdast-handlers/custom-code-handler.d.ts +7 -0
- package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
- package/dist/types/mdast-handlers/table-cell-flatten.d.ts +18 -0
- package/dist/types/mdast-handlers/table-cell-flatten.js.map +1 -0
- package/dist/types/mdast-handlers/twoslash.d.ts +14 -0
- package/dist/types/mdast-handlers/twoslash.js.map +1 -0
- package/dist/types/metadata/index.js.map +1 -1
- package/dist/types/utils/hast-fast.d.ts +1 -1
- package/dist/types/utils/hast-fast.js.map +1 -1
- package/dist/types/utils/parse-html.d.ts +33 -0
- package/dist/types/utils/parse-html.js.map +1 -0
- package/dist/types/utils/pre-text.d.ts +12 -0
- package/dist/types/utils/pre-text.js.map +1 -0
- package/dist/utils/hast-fast.js +1 -1
- package/dist/utils/parse-html.js +87 -0
- package/dist/utils/pre-text.js +81 -0
- package/package.json +3 -1
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// src/extractors/lib/code-tabs.ts
|
|
2
|
+
import { isElement, stringProperty } from "../../utils/hast-fast.js";
|
|
3
|
+
var MAX_LABEL_LENGTH = 80;
|
|
4
|
+
var CodeTabCollector = class {
|
|
5
|
+
#groups = [];
|
|
6
|
+
/** Groups whose tab list's parent is still being traversed, innermost last. */
|
|
7
|
+
#open = [];
|
|
8
|
+
#isHidden;
|
|
9
|
+
#radioStrips = [];
|
|
10
|
+
/** @param isHidden - Elements whose text a reader never sees, excluded from labels. */
|
|
11
|
+
constructor(isHidden) {
|
|
12
|
+
this.#isHidden = isHidden;
|
|
13
|
+
}
|
|
14
|
+
#applyRadioStrip(strip, parent) {
|
|
15
|
+
const labels = strip.children.filter((child) => isElement(child) && child.tagName === "label");
|
|
16
|
+
const panels = radioStripPanels(strip, parent);
|
|
17
|
+
if (!panels || panels.length !== labels.length) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
panels.forEach((panel, index) => labelCode(panel, tabLabel(labels[index], this.#isHidden)));
|
|
21
|
+
parent.children.splice(parent.children.indexOf(strip), 1);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Call for every element, in document order, with its depth.
|
|
25
|
+
*
|
|
26
|
+
* A group only collects panels inside the tab list's parent: a page-level tab strip must not
|
|
27
|
+
* lend its labels to an unrelated code switcher further down, and an inner group closes before
|
|
28
|
+
* the outer group's next panel.
|
|
29
|
+
*/
|
|
30
|
+
visit(element, depth, parent) {
|
|
31
|
+
if (isRadioTabStrip(element)) {
|
|
32
|
+
this.#radioStrips.push({ strip: element, parent });
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
while (this.#open.length > 0 && depth < (this.#open.at(-1)?.depth ?? 0)) {
|
|
36
|
+
this.#open.pop();
|
|
37
|
+
}
|
|
38
|
+
const role = stringProperty(element, "role");
|
|
39
|
+
if (role === "tablist") {
|
|
40
|
+
const group2 = { tabs: [], panels: [] };
|
|
41
|
+
this.#groups.push(group2);
|
|
42
|
+
this.#open.push({ group: group2, depth });
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
const group = this.#open.at(-1)?.group;
|
|
46
|
+
if (!group) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
if (role === "tab") {
|
|
50
|
+
group.tabs.push(element);
|
|
51
|
+
} else if (role === "tabpanel") {
|
|
52
|
+
group.panels.push(element);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** Labels the code block of every panel whose tab could be identified. */
|
|
56
|
+
apply() {
|
|
57
|
+
for (const { strip, parent } of this.#radioStrips) {
|
|
58
|
+
this.#applyRadioStrip(strip, parent);
|
|
59
|
+
}
|
|
60
|
+
for (const group of this.#groups) {
|
|
61
|
+
if (group.tabs.length < 2 || group.panels.length === 0) {
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
for (const [panel, tab] of matchTabs(group)) {
|
|
65
|
+
labelCode(panel, tabLabel(tab, this.#isHidden));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
var radioStripPanels = (strip, parent) => {
|
|
71
|
+
const index = parent.children.indexOf(strip);
|
|
72
|
+
for (let cursor = index + 1; cursor < parent.children.length; cursor++) {
|
|
73
|
+
const sibling = parent.children[cursor];
|
|
74
|
+
if (isElement(sibling)) {
|
|
75
|
+
return sibling.children.filter(isElement);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return void 0;
|
|
79
|
+
};
|
|
80
|
+
var isRadioTabStrip = (element) => {
|
|
81
|
+
const first = element.children.find(isElement);
|
|
82
|
+
if (!(first?.tagName === "input" && stringProperty(first, "type") === "radio")) {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
let labels = 0;
|
|
86
|
+
for (const child of element.children) {
|
|
87
|
+
if (!isElement(child)) {
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (child.tagName === "label") {
|
|
91
|
+
labels += 1;
|
|
92
|
+
} else if (!(child.tagName === "input" && stringProperty(child, "type") === "radio")) {
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return labels >= 2;
|
|
97
|
+
};
|
|
98
|
+
var indexById = (elements) => {
|
|
99
|
+
const byId = /* @__PURE__ */ new Map();
|
|
100
|
+
for (const element of elements) {
|
|
101
|
+
const id = stringProperty(element, "id");
|
|
102
|
+
if (id) {
|
|
103
|
+
byId.set(id, element);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return byId;
|
|
107
|
+
};
|
|
108
|
+
var matchTabs = ({ tabs, panels }) => {
|
|
109
|
+
const tabById = indexById(tabs);
|
|
110
|
+
const panelById = indexById(panels);
|
|
111
|
+
const tabOf = /* @__PURE__ */ new Map();
|
|
112
|
+
for (const panel of panels) {
|
|
113
|
+
const tab = tabById.get(stringProperty(panel, "ariaLabelledBy") ?? "");
|
|
114
|
+
if (tab) {
|
|
115
|
+
tabOf.set(panel, tab);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
for (const tab of tabs) {
|
|
119
|
+
const panel = panelById.get(stringProperty(tab, "ariaControls") ?? "");
|
|
120
|
+
if (panel && !tabOf.has(panel)) {
|
|
121
|
+
tabOf.set(panel, tab);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (tabOf.size === 0 && tabs.length === panels.length) {
|
|
125
|
+
panels.forEach((panel, index) => tabOf.set(panel, tabs[index]));
|
|
126
|
+
}
|
|
127
|
+
return tabOf;
|
|
128
|
+
};
|
|
129
|
+
var tabLabel = (tab, isHidden) => {
|
|
130
|
+
const label = (textOf(tab, isHidden) || stringProperty(tab, "dataTitle") || "").replace(/\s+/g, " ").trim();
|
|
131
|
+
return label.length > 0 && label.length <= MAX_LABEL_LENGTH ? label : void 0;
|
|
132
|
+
};
|
|
133
|
+
var labelCode = (panel, label) => {
|
|
134
|
+
if (!label) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const blocks = [];
|
|
138
|
+
collectPre(panel, blocks);
|
|
139
|
+
if (blocks.length === 1 && stringProperty(blocks[0], "dataTitle") === void 0) {
|
|
140
|
+
blocks[0].properties.dataTitle = label;
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
var collectPre = (element, into) => {
|
|
144
|
+
for (const child of element.children) {
|
|
145
|
+
if (!isElement(child) || into.length > 1) {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (child.tagName === "pre") {
|
|
149
|
+
into.push(child);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
collectPre(child, into);
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
var textOf = (element, isHidden) => {
|
|
156
|
+
let text = "";
|
|
157
|
+
for (const child of element.children) {
|
|
158
|
+
if (child.type === "text") {
|
|
159
|
+
text += child.value;
|
|
160
|
+
} else if (isElement(child) && !isHidden(child)) {
|
|
161
|
+
text += textOf(child, isHidden);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return text;
|
|
165
|
+
};
|
|
166
|
+
export {
|
|
167
|
+
CodeTabCollector
|
|
168
|
+
};
|
|
@@ -8,7 +8,7 @@ var REGEXPS = {
|
|
|
8
8
|
* `class="post-header"`), so matching them here deletes the page's headings. The landmark
|
|
9
9
|
* versions of those elements are handled structurally instead.
|
|
10
10
|
*/
|
|
11
|
-
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumb|combx|comment(?!ary)|community|cover-wrap|disqus|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|site-index|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar|subscribe|newsletter|paywall|consent|cookie|toolbar|masthead|promo|share|sharing|widget|tooltip|dropdown|modal|overlay|backdrop|carousel|announcement|notification|toast|skip-link|screen-reader|visually-hidden|sr-only/i,
|
|
11
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumb|combx|comment(?!ary)|community|cover-wrap|disqus|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|site-index|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar|subscribe|newsletter|paywall|consent|cookie|toolbar|masthead|promo|share|sharing|widget|tooltip|dropdown|modal|overlay|backdrop|carousel|announcement|notification|toast|skip-link|screen-reader|(?<!not-)visually-hidden|(?<!not-)sr-only/i,
|
|
12
12
|
/** Rescues containers whose class merely happens to contain an unlikely substring. */
|
|
13
13
|
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code|post|entry|story|markdown|prose|doc/i,
|
|
14
14
|
/** Very high-confidence boilerplate, not rescued by the clause above. */
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// src/extractors/lib/layout-tables.ts
|
|
2
|
+
import { isElement, stringProperty } from "../../utils/hast-fast.js";
|
|
3
|
+
var DATA_MARKERS = /* @__PURE__ */ new Set(["caption", "thead", "tfoot", "th", "col", "colgroup"]);
|
|
4
|
+
var BLOCK_IN_CELL = /* @__PURE__ */ new Set(["h1", "h2", "h3", "h4", "h5", "h6", "table", "form"]);
|
|
5
|
+
var TABLE_PARTS = /* @__PURE__ */ new Set(["tbody", "thead", "tfoot", "tr", "td", "th"]);
|
|
6
|
+
var TABLE_CONTEXT = /* @__PURE__ */ new Set(["table", "tbody", "thead", "tfoot", "tr"]);
|
|
7
|
+
var shapeOf = (table) => {
|
|
8
|
+
const shape = { rows: 0, columns: 0, dataMarker: false, blockInCell: false };
|
|
9
|
+
const visit = (node, inCell) => {
|
|
10
|
+
for (const child of node.children) {
|
|
11
|
+
if (!isElement(child)) {
|
|
12
|
+
continue;
|
|
13
|
+
}
|
|
14
|
+
const tag = child.tagName;
|
|
15
|
+
if (tag === "table") {
|
|
16
|
+
shape.blockInCell = shape.blockInCell || inCell;
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
if (DATA_MARKERS.has(tag)) {
|
|
20
|
+
shape.dataMarker = true;
|
|
21
|
+
}
|
|
22
|
+
if (tag === "tr") {
|
|
23
|
+
shape.rows += 1;
|
|
24
|
+
const cells = child.children.filter(
|
|
25
|
+
(cell) => isElement(cell) && (cell.tagName === "td" || cell.tagName === "th")
|
|
26
|
+
).length;
|
|
27
|
+
shape.columns = Math.max(shape.columns, cells);
|
|
28
|
+
}
|
|
29
|
+
if (inCell && BLOCK_IN_CELL.has(tag)) {
|
|
30
|
+
shape.blockInCell = true;
|
|
31
|
+
}
|
|
32
|
+
visit(child, inCell || tag === "td" || tag === "th");
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
visit(table, false);
|
|
36
|
+
return shape;
|
|
37
|
+
};
|
|
38
|
+
var isLayoutTable = (table) => {
|
|
39
|
+
const role = stringProperty(table, "role");
|
|
40
|
+
if (role === "presentation" || role === "none" || stringProperty(table, "datatable") === "0") {
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
const shape = shapeOf(table);
|
|
44
|
+
if (shape.blockInCell) {
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
if (shape.dataMarker) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
if (shape.rows <= 1 || shape.columns <= 1) {
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
if (shape.rows >= 10 || shape.columns > 4) {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
return shape.rows * shape.columns <= 10;
|
|
57
|
+
};
|
|
58
|
+
var unwrap = (table) => {
|
|
59
|
+
const cells = [];
|
|
60
|
+
const collect = (node) => {
|
|
61
|
+
for (const child of node.children) {
|
|
62
|
+
if (!isElement(child)) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (child.tagName === "td" || child.tagName === "th") {
|
|
66
|
+
cells.push({ type: "element", tagName: "div", properties: {}, children: child.children });
|
|
67
|
+
} else if (child.tagName !== "table") {
|
|
68
|
+
collect(child);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
collect(table);
|
|
73
|
+
return { type: "element", tagName: "div", properties: {}, children: cells };
|
|
74
|
+
};
|
|
75
|
+
var unwrapLayoutTables = (root) => {
|
|
76
|
+
const visit = (parent) => {
|
|
77
|
+
const parentTag = isElement(parent) ? parent.tagName : "";
|
|
78
|
+
for (let index = 0; index < parent.children.length; index++) {
|
|
79
|
+
const child = parent.children[index];
|
|
80
|
+
if (!isElement(child)) {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (TABLE_PARTS.has(child.tagName) && !TABLE_CONTEXT.has(parentTag)) {
|
|
84
|
+
child.tagName = "div";
|
|
85
|
+
}
|
|
86
|
+
visit(child);
|
|
87
|
+
if (child.tagName === "table" && isLayoutTable(child)) {
|
|
88
|
+
parent.children[index] = unwrap(child);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
if ("children" in root) {
|
|
93
|
+
visit(root);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
export {
|
|
97
|
+
isLayoutTable,
|
|
98
|
+
unwrapLayoutTables
|
|
99
|
+
};
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
stringProperty,
|
|
11
11
|
walk
|
|
12
12
|
} from "../../utils/hast-fast.js";
|
|
13
|
+
import { CodeTabCollector } from "./code-tabs.js";
|
|
13
14
|
import {
|
|
14
15
|
CHROME_TAGS,
|
|
15
16
|
CONTEXTUAL_LANDMARK_TAGS,
|
|
@@ -21,25 +22,39 @@ import {
|
|
|
21
22
|
RESPONSIVE_DISPLAY_OVERRIDE,
|
|
22
23
|
UNLIKELY_ROLES
|
|
23
24
|
} from "./constants.js";
|
|
24
|
-
import { isUiChromeText } from "./ui-chrome.js";
|
|
25
|
+
import { MAX_CONSENT_PLACEHOLDER_LENGTH, isConsentPlaceholderText, isUiChromeText } from "./ui-chrome.js";
|
|
25
26
|
var INVISIBLE_STYLE = /(^|;)\s*(display\s*:\s*none|visibility\s*:\s*hidden)\s*(;|$)/i;
|
|
26
27
|
var hasHiddenClass = (element) => {
|
|
27
28
|
const classes = classList(element);
|
|
28
29
|
if (!classes.some((name) => HIDDEN_CLASS_NAMES.has(name))) {
|
|
29
|
-
return classes.some((name) => REGEXPS.hidden.test(name));
|
|
30
|
+
return classes.some((name) => REGEXPS.hidden.test(name) || isScreenReaderOnlyClass(name));
|
|
30
31
|
}
|
|
31
32
|
return !classes.some((name) => RESPONSIVE_DISPLAY_OVERRIDE.test(name));
|
|
32
33
|
};
|
|
34
|
+
var SCREEN_READER_ONLY = /^(sronly|visuallyhidden|screenreader(only|text))(focusable|styles\w*)?$/;
|
|
35
|
+
var isScreenReaderOnlyClass = (name) => {
|
|
36
|
+
const first = name.charCodeAt(0) | 32;
|
|
37
|
+
if (first !== 115 && first !== 118) {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
return SCREEN_READER_ONLY.test(name.toLowerCase().replace(/[-_]/g, ""));
|
|
41
|
+
};
|
|
33
42
|
var isNonContent = (node) => {
|
|
34
43
|
if (node.type === "comment" || node.type === "doctype") {
|
|
35
44
|
return true;
|
|
36
45
|
}
|
|
37
46
|
return isElement(node) && NON_CONTENT_TAGS.has(node.tagName);
|
|
38
47
|
};
|
|
39
|
-
var isHidden = (element) => {
|
|
48
|
+
var isHidden = (element, parent, siblingCode) => {
|
|
49
|
+
if (!looksHidden(element)) {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
40
52
|
if (containsMath(element)) {
|
|
41
53
|
return false;
|
|
42
54
|
}
|
|
55
|
+
return !isHiddenCodePanel(element, parent, siblingCode);
|
|
56
|
+
};
|
|
57
|
+
var looksHidden = (element) => {
|
|
43
58
|
if (isTruthyAttribute(element, "hidden")) {
|
|
44
59
|
return true;
|
|
45
60
|
}
|
|
@@ -52,10 +67,57 @@ var isHidden = (element) => {
|
|
|
52
67
|
}
|
|
53
68
|
const width = numericProperty(element, "data-rwidth");
|
|
54
69
|
const height = numericProperty(element, "data-rheight");
|
|
55
|
-
|
|
70
|
+
return width === 0 && height === 0;
|
|
71
|
+
};
|
|
72
|
+
var MAX_CODE_PANEL_PROSE = 200;
|
|
73
|
+
var panelCode = (element) => {
|
|
74
|
+
let code = "";
|
|
75
|
+
let prose = 0;
|
|
76
|
+
const visit = (node, insideCode) => {
|
|
77
|
+
for (const child of node.children) {
|
|
78
|
+
if (child.type === "text") {
|
|
79
|
+
if (insideCode) {
|
|
80
|
+
code += child.value;
|
|
81
|
+
} else {
|
|
82
|
+
prose += child.value.trim().length;
|
|
83
|
+
if (prose > MAX_CODE_PANEL_PROSE) {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
} else if (isElement(child) && !visit(child, insideCode || child.tagName === "pre")) {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
56
91
|
return true;
|
|
92
|
+
};
|
|
93
|
+
return visit(element, element.tagName === "pre") && code.trim().length > 0 ? code : void 0;
|
|
94
|
+
};
|
|
95
|
+
var visibleSiblingCode = (parent, cache) => {
|
|
96
|
+
let codes = cache.get(parent);
|
|
97
|
+
if (!codes) {
|
|
98
|
+
codes = /* @__PURE__ */ new Set();
|
|
99
|
+
for (const child of parent.children) {
|
|
100
|
+
if (isElement(child) && !looksHidden(child)) {
|
|
101
|
+
const code = panelCode(child);
|
|
102
|
+
if (code !== void 0) {
|
|
103
|
+
codes.add(code);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
cache.set(parent, codes);
|
|
57
108
|
}
|
|
58
|
-
return
|
|
109
|
+
return codes;
|
|
110
|
+
};
|
|
111
|
+
var isHiddenCodePanel = (element, parent, cache) => {
|
|
112
|
+
const code = panelCode(element);
|
|
113
|
+
if (code === void 0) {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
const siblings = visibleSiblingCode(parent, cache);
|
|
117
|
+
if (siblings.has(code)) {
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
return stringProperty(element, "role") === "tabpanel" || siblings.size > 0;
|
|
59
121
|
};
|
|
60
122
|
var PLACEHOLDER_IMAGE_SRC = /^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
|
|
61
123
|
var isPlaceholderImage = (element) => {
|
|
@@ -73,15 +135,16 @@ var isPlaceholderImage = (element) => {
|
|
|
73
135
|
}
|
|
74
136
|
return false;
|
|
75
137
|
};
|
|
76
|
-
var hoistNoscriptImages = (tree) => {
|
|
77
|
-
const visit = (parent) => {
|
|
138
|
+
var hoistNoscriptImages = (tree, onElement) => {
|
|
139
|
+
const visit = (parent, depth) => {
|
|
78
140
|
for (let index = 0; index < parent.children.length; index++) {
|
|
79
141
|
const child = parent.children[index];
|
|
80
142
|
if (!isElement(child)) {
|
|
81
143
|
continue;
|
|
82
144
|
}
|
|
83
145
|
if (child.tagName !== "noscript") {
|
|
84
|
-
|
|
146
|
+
onElement?.(child, depth, parent);
|
|
147
|
+
visit(child, depth + 1);
|
|
85
148
|
continue;
|
|
86
149
|
}
|
|
87
150
|
const hasImage = child.children.some(
|
|
@@ -101,7 +164,7 @@ var hoistNoscriptImages = (tree) => {
|
|
|
101
164
|
}
|
|
102
165
|
};
|
|
103
166
|
if ("children" in tree) {
|
|
104
|
-
visit(tree);
|
|
167
|
+
visit(tree, 0);
|
|
105
168
|
}
|
|
106
169
|
};
|
|
107
170
|
var containsMath = (element) => {
|
|
@@ -116,12 +179,15 @@ var containsMath = (element) => {
|
|
|
116
179
|
return false;
|
|
117
180
|
};
|
|
118
181
|
var stripNonContent = (tree) => {
|
|
119
|
-
|
|
120
|
-
|
|
182
|
+
const tabs = new CodeTabCollector((element) => looksHidden(element) || NON_CONTENT_TAGS.has(element.tagName));
|
|
183
|
+
hoistNoscriptImages(tree, (element, depth, parent) => tabs.visit(element, depth, parent));
|
|
184
|
+
tabs.apply();
|
|
185
|
+
const siblingCode = /* @__PURE__ */ new WeakMap();
|
|
186
|
+
return pruneInPlace(tree, (node, parent) => {
|
|
121
187
|
if (isNonContent(node)) {
|
|
122
188
|
return false;
|
|
123
189
|
}
|
|
124
|
-
if (isElement(node) && isHidden(node)) {
|
|
190
|
+
if (isElement(node) && isHidden(node, parent, siblingCode)) {
|
|
125
191
|
return false;
|
|
126
192
|
}
|
|
127
193
|
return true;
|
|
@@ -148,32 +214,32 @@ var findLandmarkChrome = (tree) => {
|
|
|
148
214
|
visit(tree, 0, false);
|
|
149
215
|
return chrome;
|
|
150
216
|
};
|
|
151
|
-
var
|
|
217
|
+
var unlikelyTier = (element) => {
|
|
152
218
|
if (element.tagName === "article" || element.tagName === "main" || element.tagName === "body") {
|
|
153
|
-
return
|
|
219
|
+
return void 0;
|
|
154
220
|
}
|
|
155
221
|
const role = stringProperty(element, "role");
|
|
156
222
|
if (role && UNLIKELY_ROLES.has(role)) {
|
|
157
|
-
return
|
|
223
|
+
return "strong";
|
|
158
224
|
}
|
|
159
225
|
if (CHROME_TAGS.has(element.tagName)) {
|
|
160
|
-
return
|
|
226
|
+
return "strong";
|
|
161
227
|
}
|
|
162
228
|
const match = matchString(element);
|
|
163
229
|
if (REGEXPS.specialUnlikelyCandidates.test(match)) {
|
|
164
|
-
return
|
|
230
|
+
return "strong";
|
|
165
231
|
}
|
|
166
232
|
for (const name of [...classList(element), stringProperty(element, "id") ?? ""]) {
|
|
167
233
|
if (REGEXPS.stronglyUnlikely.test(name)) {
|
|
168
|
-
return
|
|
234
|
+
return "strong";
|
|
169
235
|
}
|
|
170
236
|
}
|
|
171
237
|
if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
|
|
172
|
-
return
|
|
238
|
+
return "weak";
|
|
173
239
|
}
|
|
174
|
-
return
|
|
240
|
+
return void 0;
|
|
175
241
|
};
|
|
176
|
-
var findUnlikelyElements = (tree) => {
|
|
242
|
+
var findUnlikelyElements = (tree, spareWeak) => {
|
|
177
243
|
const landmarks = findLandmarkChrome(tree);
|
|
178
244
|
const doomed = [];
|
|
179
245
|
const visit = (node) => {
|
|
@@ -184,7 +250,15 @@ var findUnlikelyElements = (tree) => {
|
|
|
184
250
|
if (!isElement(child)) {
|
|
185
251
|
continue;
|
|
186
252
|
}
|
|
187
|
-
if (landmarks.has(child)
|
|
253
|
+
if (landmarks.has(child)) {
|
|
254
|
+
doomed.push(child);
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
if (PREFORMATTED_TAGS.has(child.tagName)) {
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
const tier = unlikelyTier(child);
|
|
261
|
+
if (tier === "strong" || tier === "weak" && !spareWeak?.(child)) {
|
|
188
262
|
doomed.push(child);
|
|
189
263
|
continue;
|
|
190
264
|
}
|
|
@@ -329,11 +403,15 @@ var findCleanupTargets = (tree, collector) => {
|
|
|
329
403
|
visit(child);
|
|
330
404
|
continue;
|
|
331
405
|
}
|
|
406
|
+
if (LITERAL_TEXT_TAGS.has(child.tagName)) {
|
|
407
|
+
collectButtons(child, doomed);
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
332
410
|
if (isLinkOnlyBlock(child, collector)) {
|
|
333
411
|
condemnLinkBlock(child, siblings, index, collector, doomed);
|
|
334
412
|
continue;
|
|
335
413
|
}
|
|
336
|
-
if (isMidArticleWidget(child)) {
|
|
414
|
+
if (isMidArticleWidget(child, collector)) {
|
|
337
415
|
doomed.push(child);
|
|
338
416
|
continue;
|
|
339
417
|
}
|
|
@@ -375,18 +453,52 @@ var countAnchors = (element) => {
|
|
|
375
453
|
return count;
|
|
376
454
|
};
|
|
377
455
|
var LITERAL_TEXT_TAGS = /* @__PURE__ */ new Set(["code", "kbd", "samp", "var", "pre"]);
|
|
378
|
-
var
|
|
379
|
-
|
|
380
|
-
|
|
456
|
+
var PREFORMATTED_TAGS = /* @__PURE__ */ new Set(["pre", "code"]);
|
|
457
|
+
var collectButtons = (element, into) => {
|
|
458
|
+
for (const child of element.children) {
|
|
459
|
+
if (!isElement(child)) {
|
|
460
|
+
continue;
|
|
461
|
+
}
|
|
462
|
+
if (child.tagName === "button") {
|
|
463
|
+
into.push(child);
|
|
464
|
+
continue;
|
|
465
|
+
}
|
|
466
|
+
collectButtons(child, into);
|
|
381
467
|
}
|
|
468
|
+
};
|
|
469
|
+
var isMidArticleWidget = (element, collector) => {
|
|
382
470
|
if (element.tagName === "form" || element.tagName === "fieldset") {
|
|
383
471
|
return true;
|
|
384
472
|
}
|
|
385
473
|
if (element.tagName === "button") {
|
|
386
474
|
return true;
|
|
387
475
|
}
|
|
388
|
-
|
|
476
|
+
if (collector.metrics(element).text > MAX_CONSENT_PLACEHOLDER_LENGTH) {
|
|
477
|
+
return false;
|
|
478
|
+
}
|
|
479
|
+
const text = elementText(element);
|
|
480
|
+
return isUiChromeText(text) || isConsentPlaceholderText(text) && !hasBlockDescendant(element);
|
|
389
481
|
};
|
|
482
|
+
var BLOCK_TAGS = /* @__PURE__ */ new Set([
|
|
483
|
+
"p",
|
|
484
|
+
"div",
|
|
485
|
+
"section",
|
|
486
|
+
"article",
|
|
487
|
+
"figure",
|
|
488
|
+
"figcaption",
|
|
489
|
+
"blockquote",
|
|
490
|
+
"ul",
|
|
491
|
+
"ol",
|
|
492
|
+
"table",
|
|
493
|
+
"pre",
|
|
494
|
+
"h1",
|
|
495
|
+
"h2",
|
|
496
|
+
"h3",
|
|
497
|
+
"h4",
|
|
498
|
+
"h5",
|
|
499
|
+
"h6"
|
|
500
|
+
]);
|
|
501
|
+
var hasBlockDescendant = (element) => element.children.some((child) => isElement(child) && (BLOCK_TAGS.has(child.tagName) || hasBlockDescendant(child)));
|
|
390
502
|
var cleanContent = (tree, collector) => {
|
|
391
503
|
const doomed = new Set(findCleanupTargets(tree, collector));
|
|
392
504
|
if (doomed.size > 0) {
|
|
@@ -408,6 +520,7 @@ var elementText = (node) => {
|
|
|
408
520
|
return text;
|
|
409
521
|
};
|
|
410
522
|
export {
|
|
523
|
+
REFERENCE_HEADING,
|
|
411
524
|
cleanContent,
|
|
412
525
|
dropEmptyElements,
|
|
413
526
|
findCleanupTargets,
|
|
@@ -415,5 +528,6 @@ export {
|
|
|
415
528
|
findUnlikelyElements,
|
|
416
529
|
hoistNoscriptImages,
|
|
417
530
|
stripNonContent,
|
|
418
|
-
stripUnlikely
|
|
531
|
+
stripUnlikely,
|
|
532
|
+
unlikelyTier
|
|
419
533
|
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// src/extractors/lib/takumi-signal.ts
|
|
2
|
+
import { isElement, walk } from "../../utils/hast-fast.js";
|
|
3
|
+
import { takumiExtractor } from "../presets/takumi.js";
|
|
4
|
+
var MARK = "dataWebforaiIndex";
|
|
5
|
+
var cloneElements = (root) => {
|
|
6
|
+
const copyOf = (node) => "children" in node ? {
|
|
7
|
+
...node,
|
|
8
|
+
...isElement(node) ? { properties: { ...node.properties } } : {},
|
|
9
|
+
children: [...node.children]
|
|
10
|
+
} : node;
|
|
11
|
+
const copy = copyOf(root);
|
|
12
|
+
const stack = [copy];
|
|
13
|
+
while (stack.length > 0) {
|
|
14
|
+
const parent = stack.pop();
|
|
15
|
+
if (!("children" in parent)) {
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
const children = parent.children;
|
|
19
|
+
for (let index = 0; index < children.length; index++) {
|
|
20
|
+
if ("children" in children[index]) {
|
|
21
|
+
children[index] = copyOf(children[index]);
|
|
22
|
+
stack.push(children[index]);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return copy;
|
|
27
|
+
};
|
|
28
|
+
var takumiSelection = (root, lang, url) => {
|
|
29
|
+
const elements = [];
|
|
30
|
+
walk(root, (node) => {
|
|
31
|
+
if (isElement(node)) {
|
|
32
|
+
node.properties[MARK] = elements.length;
|
|
33
|
+
elements.push(node);
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
const copy = cloneElements(root);
|
|
37
|
+
for (const element of elements) {
|
|
38
|
+
delete element.properties[MARK];
|
|
39
|
+
}
|
|
40
|
+
const kept = /* @__PURE__ */ new Set();
|
|
41
|
+
walk(takumiExtractor({ hast: copy, lang, url, owned: true }), (node) => {
|
|
42
|
+
if (isElement(node)) {
|
|
43
|
+
const index = node.properties[MARK];
|
|
44
|
+
if (typeof index === "number" && elements[index]) {
|
|
45
|
+
kept.add(elements[index]);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
return kept;
|
|
50
|
+
};
|
|
51
|
+
export {
|
|
52
|
+
takumiSelection
|
|
53
|
+
};
|
|
@@ -91,6 +91,11 @@ var isUiChromeText = (text) => {
|
|
|
91
91
|
}
|
|
92
92
|
return UI_PHRASES.has(normalized);
|
|
93
93
|
};
|
|
94
|
+
var CONSENT_PLACEHOLDER = /^(this content (isn['’]?t|is not) (visible|available) (due to|because of) your (cookie|privacy) (preferences|settings)|we need your (consent|permission) to load|to (view|see|load) this (content|embed|video|post)[, ]+(please )?(enable|accept|allow|update)|(please )?(accept|allow|enable) (all |marketing |functional )?cookies to (view|see|load|watch)|このコンテンツを表示するには.{0,30}(cookie|クッキー|同意|許可))/i;
|
|
95
|
+
var MAX_CONSENT_PLACEHOLDER_LENGTH = 400;
|
|
96
|
+
var isConsentPlaceholderText = (text) => text.length <= MAX_CONSENT_PLACEHOLDER_LENGTH && CONSENT_PLACEHOLDER.test(text.trimStart());
|
|
94
97
|
export {
|
|
98
|
+
MAX_CONSENT_PLACEHOLDER_LENGTH,
|
|
99
|
+
isConsentPlaceholderText,
|
|
95
100
|
isUiChromeText
|
|
96
101
|
};
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// src/extractors/presets/auto.ts
|
|
2
2
|
import { runAdapters } from "../../adapters/index.js";
|
|
3
|
-
import {
|
|
3
|
+
import { kiwameExtractor } from "./kiwame.js";
|
|
4
4
|
var createAutoExtractor = (options = {}) => {
|
|
5
|
-
const { adapters } = options;
|
|
5
|
+
const { adapters, fallback = kiwameExtractor } = options;
|
|
6
6
|
return (params) => {
|
|
7
7
|
if (adapters !== false) {
|
|
8
8
|
const extracted = runAdapters({ hast: params.hast, url: params.url, lang: params.lang }, adapters ?? void 0);
|
|
@@ -10,7 +10,7 @@ var createAutoExtractor = (options = {}) => {
|
|
|
10
10
|
return extracted;
|
|
11
11
|
}
|
|
12
12
|
}
|
|
13
|
-
return
|
|
13
|
+
return fallback(params);
|
|
14
14
|
};
|
|
15
15
|
};
|
|
16
16
|
var autoExtractor = createAutoExtractor();
|