webforai 2.1.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/README.md +63 -1
  2. package/dist/adapters/index.js +58 -0
  3. package/dist/adapters/registry.js +149 -0
  4. package/dist/adapters/sites/declarative.js +178 -0
  5. package/dist/adapters/sites/hackernews.js +94 -0
  6. package/dist/adapters/sites/youtube.js +165 -0
  7. package/dist/adapters/types.js +0 -0
  8. package/dist/bin.js +3197 -402
  9. package/dist/cjs/adapters/index.js +86 -0
  10. package/dist/cjs/adapters/registry.js +178 -0
  11. package/dist/cjs/adapters/sites/declarative.js +202 -0
  12. package/dist/cjs/adapters/sites/hackernews.js +117 -0
  13. package/dist/cjs/adapters/sites/youtube.js +191 -0
  14. package/dist/cjs/adapters/types.js +16 -0
  15. package/dist/cjs/constants.js +2 -2
  16. package/dist/cjs/extractors/index.js +5 -0
  17. package/dist/cjs/extractors/lib/constants.js +186 -0
  18. package/dist/cjs/extractors/lib/sanitize.js +425 -0
  19. package/dist/cjs/extractors/lib/score.js +264 -0
  20. package/dist/cjs/extractors/lib/terminators.js +131 -0
  21. package/dist/cjs/extractors/lib/ui-chrome.js +119 -0
  22. package/dist/cjs/extractors/pipeExtractors.js +12 -9
  23. package/dist/cjs/extractors/presets/auto.js +44 -0
  24. package/dist/cjs/extractors/presets/minimal-filter.js +27 -48
  25. package/dist/cjs/extractors/presets/takumi.js +84 -122
  26. package/dist/cjs/html-to-markdown.js +44 -7
  27. package/dist/cjs/html-to-mdast.js +9 -1
  28. package/dist/cjs/index.js +31 -2
  29. package/dist/cjs/mdast-handlers/custom-a-handler.js +25 -9
  30. package/dist/cjs/mdast-handlers/custom-img-handler.js +23 -1
  31. package/dist/cjs/mdast-handlers/custom-table-handler.js +16 -0
  32. package/dist/cjs/mdast-handlers/definition-list-handler.js +99 -0
  33. package/dist/cjs/mdast-handlers/math-handler.js +12 -3
  34. package/dist/cjs/mdast-handlers/sup-sub-handler.js +39 -0
  35. package/dist/cjs/metadata/index.js +223 -0
  36. package/dist/cjs/normalize/index.js +168 -0
  37. package/dist/cjs/platform/client.js +147 -0
  38. package/dist/cjs/platform/error.js +36 -0
  39. package/dist/cjs/platform/fetch.js +37 -0
  40. package/dist/cjs/platform/index.js +40 -0
  41. package/dist/cjs/platform/types.js +34 -0
  42. package/dist/cjs/utils/hast-fast.js +284 -0
  43. package/dist/constants.js +2 -2
  44. package/dist/extractors/index.js +3 -0
  45. package/dist/extractors/lib/constants.js +150 -0
  46. package/dist/extractors/lib/sanitize.js +415 -0
  47. package/dist/extractors/lib/score.js +242 -0
  48. package/dist/extractors/lib/terminators.js +107 -0
  49. package/dist/extractors/lib/ui-chrome.js +96 -0
  50. package/dist/extractors/pipeExtractors.js +12 -9
  51. package/dist/extractors/presets/auto.js +20 -0
  52. package/dist/extractors/presets/minimal-filter.js +35 -47
  53. package/dist/extractors/presets/takumi.js +91 -122
  54. package/dist/html-to-markdown.js +42 -6
  55. package/dist/html-to-mdast.js +9 -1
  56. package/dist/index.js +31 -2
  57. package/dist/mdast-handlers/custom-a-handler.js +25 -9
  58. package/dist/mdast-handlers/custom-img-handler.js +23 -1
  59. package/dist/mdast-handlers/custom-table-handler.js +16 -0
  60. package/dist/mdast-handlers/definition-list-handler.js +76 -0
  61. package/dist/mdast-handlers/math-handler.js +12 -3
  62. package/dist/mdast-handlers/sup-sub-handler.js +16 -0
  63. package/dist/metadata/index.js +198 -0
  64. package/dist/normalize/index.js +149 -0
  65. package/dist/platform/client.js +125 -0
  66. package/dist/platform/error.js +13 -0
  67. package/dist/platform/fetch.js +14 -0
  68. package/dist/platform/index.js +19 -0
  69. package/dist/platform/types.js +9 -0
  70. package/dist/types/adapters/index.d.ts +24 -0
  71. package/dist/types/adapters/index.js.map +1 -0
  72. package/dist/types/adapters/registry.d.ts +35 -0
  73. package/dist/types/adapters/registry.js.map +1 -0
  74. package/dist/types/adapters/sites/declarative.d.ts +13 -0
  75. package/dist/types/adapters/sites/declarative.js.map +1 -0
  76. package/dist/types/adapters/sites/hackernews.d.ts +13 -0
  77. package/dist/types/adapters/sites/hackernews.js.map +1 -0
  78. package/dist/types/adapters/sites/youtube.d.ts +31 -0
  79. package/dist/types/adapters/sites/youtube.js.map +1 -0
  80. package/dist/types/adapters/types.d.ts +99 -0
  81. package/dist/types/adapters/types.js.map +1 -0
  82. package/dist/types/cli/bin.js.map +1 -1
  83. package/dist/types/cli/commands/skill/content.d.ts +9 -0
  84. package/dist/types/cli/commands/skill/content.js.map +1 -0
  85. package/dist/types/cli/commands/skill/index.d.ts +19 -0
  86. package/dist/types/cli/commands/skill/index.js.map +1 -0
  87. package/dist/types/cli/commands/webforai/interactive.d.ts +7 -0
  88. package/dist/types/cli/commands/webforai/interactive.js.map +1 -0
  89. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -1
  90. package/dist/types/cli/commands/webforai/options.d.ts +55 -0
  91. package/dist/types/cli/commands/webforai/options.js.map +1 -0
  92. package/dist/types/cli/commands/webforai/run.d.ts +10 -0
  93. package/dist/types/cli/commands/webforai/run.js.map +1 -0
  94. package/dist/types/cli/constants.d.ts +14 -4
  95. package/dist/types/cli/constants.js.map +1 -1
  96. package/dist/types/cli/helpers/selectExtractMode.js.map +1 -1
  97. package/dist/types/cli/helpers/selectLoader.js.map +1 -1
  98. package/dist/types/constants.d.ts +7 -1
  99. package/dist/types/constants.js.map +1 -1
  100. package/dist/types/extractors/index.d.ts +1 -0
  101. package/dist/types/extractors/index.js.map +1 -1
  102. package/dist/types/extractors/lib/constants.d.ts +81 -0
  103. package/dist/types/extractors/lib/constants.js.map +1 -0
  104. package/dist/types/extractors/lib/sanitize.d.ts +66 -0
  105. package/dist/types/extractors/lib/sanitize.js.map +1 -0
  106. package/dist/types/extractors/lib/score.d.ts +85 -0
  107. package/dist/types/extractors/lib/score.js.map +1 -0
  108. package/dist/types/extractors/lib/terminators.d.ts +26 -0
  109. package/dist/types/extractors/lib/terminators.js.map +1 -0
  110. package/dist/types/extractors/lib/ui-chrome.d.ts +18 -0
  111. package/dist/types/extractors/lib/ui-chrome.js.map +1 -0
  112. package/dist/types/extractors/pipeExtractors.d.ts +5 -0
  113. package/dist/types/extractors/pipeExtractors.js.map +1 -1
  114. package/dist/types/extractors/presets/auto.d.ts +30 -0
  115. package/dist/types/extractors/presets/auto.js.map +1 -0
  116. package/dist/types/extractors/presets/minimal-filter.d.ts +10 -2
  117. package/dist/types/extractors/presets/minimal-filter.js.map +1 -1
  118. package/dist/types/extractors/presets/takumi.d.ts +19 -3
  119. package/dist/types/extractors/presets/takumi.js.map +1 -1
  120. package/dist/types/extractors/types.d.ts +8 -0
  121. package/dist/types/html-to-markdown.d.ts +38 -0
  122. package/dist/types/html-to-markdown.js.map +1 -1
  123. package/dist/types/html-to-mdast.d.ts +11 -0
  124. package/dist/types/html-to-mdast.js.map +1 -1
  125. package/dist/types/index.d.ts +6 -3
  126. package/dist/types/index.js.map +1 -1
  127. package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
  128. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  129. package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
  130. package/dist/types/mdast-handlers/definition-list-handler.d.ts +13 -0
  131. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -0
  132. package/dist/types/mdast-handlers/math-handler.js.map +1 -1
  133. package/dist/types/mdast-handlers/sup-sub-handler.d.ts +10 -0
  134. package/dist/types/mdast-handlers/sup-sub-handler.js.map +1 -0
  135. package/dist/types/metadata/index.d.ts +47 -0
  136. package/dist/types/metadata/index.js.map +1 -0
  137. package/dist/types/normalize/index.d.ts +53 -0
  138. package/dist/types/normalize/index.js.map +1 -0
  139. package/dist/types/platform/client.d.ts +54 -0
  140. package/dist/types/platform/client.js.map +1 -0
  141. package/dist/types/platform/error.d.ts +20 -0
  142. package/dist/types/platform/error.js.map +1 -0
  143. package/dist/types/platform/fetch.d.ts +34 -0
  144. package/dist/types/platform/fetch.js.map +1 -0
  145. package/dist/types/platform/index.d.ts +4 -0
  146. package/dist/types/platform/index.js.map +1 -0
  147. package/dist/types/platform/types.d.ts +135 -0
  148. package/dist/types/platform/types.js.map +1 -0
  149. package/dist/types/utils/hast-fast.d.ts +101 -0
  150. package/dist/types/utils/hast-fast.js.map +1 -0
  151. package/dist/utils/hast-fast.js +246 -0
  152. package/package.json +10 -3
  153. package/dist/types/cli/commands/webforai/index.d.ts +0 -6
  154. package/dist/types/cli/commands/webforai/index.js.map +0 -1
@@ -0,0 +1,425 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var sanitize_exports = {};
20
+ __export(sanitize_exports, {
21
+ cleanContent: () => cleanContent,
22
+ dropEmptyElements: () => dropEmptyElements,
23
+ findCleanupTargets: () => findCleanupTargets,
24
+ findLandmarkChrome: () => findLandmarkChrome,
25
+ findUnlikelyElements: () => findUnlikelyElements,
26
+ hoistNoscriptImages: () => hoistNoscriptImages,
27
+ stripNonContent: () => stripNonContent,
28
+ stripUnlikely: () => stripUnlikely
29
+ });
30
+ module.exports = __toCommonJS(sanitize_exports);
31
+ var import_hast_fast = require("../../utils/hast-fast");
32
+ var import_constants = require("./constants");
33
+ var import_ui_chrome = require("./ui-chrome");
34
+ const INVISIBLE_STYLE = /(^|;)\s*(display\s*:\s*none|visibility\s*:\s*hidden)\s*(;|$)/i;
35
+ const hasHiddenClass = (element) => {
36
+ const classes = (0, import_hast_fast.classList)(element);
37
+ if (!classes.some((name) => import_constants.HIDDEN_CLASS_NAMES.has(name))) {
38
+ return classes.some((name) => import_constants.REGEXPS.hidden.test(name));
39
+ }
40
+ return !classes.some((name) => import_constants.RESPONSIVE_DISPLAY_OVERRIDE.test(name));
41
+ };
42
+ const isNonContent = (node) => {
43
+ if (node.type === "comment" || node.type === "doctype") {
44
+ return true;
45
+ }
46
+ return (0, import_hast_fast.isElement)(node) && import_constants.NON_CONTENT_TAGS.has(node.tagName);
47
+ };
48
+ const isHidden = (element) => {
49
+ if (containsMath(element)) {
50
+ return false;
51
+ }
52
+ if ((0, import_hast_fast.isTruthyAttribute)(element, "hidden")) {
53
+ return true;
54
+ }
55
+ const style = (0, import_hast_fast.stringProperty)(element, "style");
56
+ if (style && INVISIBLE_STYLE.test(style)) {
57
+ return true;
58
+ }
59
+ if (hasHiddenClass(element)) {
60
+ return true;
61
+ }
62
+ const width = (0, import_hast_fast.numericProperty)(element, "data-rwidth");
63
+ const height = (0, import_hast_fast.numericProperty)(element, "data-rheight");
64
+ if (width === 0 && height === 0) {
65
+ return true;
66
+ }
67
+ return false;
68
+ };
69
+ const PLACEHOLDER_IMAGE_SRC = /^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
70
+ const isPlaceholderImage = (element) => {
71
+ if (element.tagName === "img") {
72
+ const src = (0, import_hast_fast.stringProperty)(element, "src");
73
+ return src !== void 0 && PLACEHOLDER_IMAGE_SRC.test(src);
74
+ }
75
+ if (element.tagName !== "picture") {
76
+ return false;
77
+ }
78
+ for (const child of element.children) {
79
+ if ((0, import_hast_fast.isElement)(child) && child.tagName === "img") {
80
+ return isPlaceholderImage(child);
81
+ }
82
+ }
83
+ return false;
84
+ };
85
+ const hoistNoscriptImages = (tree) => {
86
+ const visit = (parent) => {
87
+ for (let index = 0; index < parent.children.length; index++) {
88
+ const child = parent.children[index];
89
+ if (!(0, import_hast_fast.isElement)(child)) {
90
+ continue;
91
+ }
92
+ if (child.tagName !== "noscript") {
93
+ visit(child);
94
+ continue;
95
+ }
96
+ const hasImage = child.children.some(
97
+ (node) => (0, import_hast_fast.isElement)(node) && (node.tagName === "img" || node.tagName === "picture")
98
+ );
99
+ if (!hasImage) {
100
+ continue;
101
+ }
102
+ const rescued = child.children;
103
+ parent.children.splice(index, 1, ...rescued);
104
+ const previous = parent.children[index - 1];
105
+ if (index > 0 && (0, import_hast_fast.isElement)(previous) && isPlaceholderImage(previous)) {
106
+ parent.children.splice(index - 1, 1);
107
+ index -= 1;
108
+ }
109
+ index += rescued.length - 1;
110
+ }
111
+ };
112
+ if ("children" in tree) {
113
+ visit(tree);
114
+ }
115
+ };
116
+ const containsMath = (element) => {
117
+ if (element.tagName === "math") {
118
+ return true;
119
+ }
120
+ for (const child of element.children) {
121
+ if ((0, import_hast_fast.isElement)(child) && containsMath(child)) {
122
+ return true;
123
+ }
124
+ }
125
+ return false;
126
+ };
127
+ const stripNonContent = (tree) => {
128
+ hoistNoscriptImages(tree);
129
+ return (0, import_hast_fast.pruneInPlace)(tree, (node) => {
130
+ if (isNonContent(node)) {
131
+ return false;
132
+ }
133
+ if ((0, import_hast_fast.isElement)(node) && isHidden(node)) {
134
+ return false;
135
+ }
136
+ return true;
137
+ });
138
+ };
139
+ const findLandmarkChrome = (tree) => {
140
+ const chrome = /* @__PURE__ */ new Set();
141
+ const visit = (node, depth, insideContent) => {
142
+ if (!("children" in node)) {
143
+ return;
144
+ }
145
+ for (const child of node.children) {
146
+ if (!(0, import_hast_fast.isElement)(child)) {
147
+ continue;
148
+ }
149
+ const nowInsideContent = insideContent || child.tagName === "article" || child.tagName === "main" || (0, import_hast_fast.stringProperty)(child, "role") === "main";
150
+ if (import_constants.CONTEXTUAL_LANDMARK_TAGS.has(child.tagName) && !insideContent && depth <= 3) {
151
+ chrome.add(child);
152
+ continue;
153
+ }
154
+ visit(child, depth + 1, nowInsideContent);
155
+ }
156
+ };
157
+ visit(tree, 0, false);
158
+ return chrome;
159
+ };
160
+ const isUnlikelyCandidate = (element) => {
161
+ if (element.tagName === "article" || element.tagName === "main" || element.tagName === "body") {
162
+ return false;
163
+ }
164
+ const role = (0, import_hast_fast.stringProperty)(element, "role");
165
+ if (role && import_constants.UNLIKELY_ROLES.has(role)) {
166
+ return true;
167
+ }
168
+ if (import_constants.CHROME_TAGS.has(element.tagName)) {
169
+ return true;
170
+ }
171
+ const match = (0, import_hast_fast.matchString)(element);
172
+ if (import_constants.REGEXPS.specialUnlikelyCandidates.test(match)) {
173
+ return true;
174
+ }
175
+ for (const name of [...(0, import_hast_fast.classList)(element), (0, import_hast_fast.stringProperty)(element, "id") ?? ""]) {
176
+ if (import_constants.REGEXPS.stronglyUnlikely.test(name)) {
177
+ return true;
178
+ }
179
+ }
180
+ if (import_constants.REGEXPS.unlikelyCandidates.test(match) && !import_constants.REGEXPS.okMaybeItsaCandidate.test(match)) {
181
+ return true;
182
+ }
183
+ return false;
184
+ };
185
+ const findUnlikelyElements = (tree) => {
186
+ const landmarks = findLandmarkChrome(tree);
187
+ const doomed = [];
188
+ const visit = (node) => {
189
+ if (!("children" in node)) {
190
+ return;
191
+ }
192
+ for (const child of node.children) {
193
+ if (!(0, import_hast_fast.isElement)(child)) {
194
+ continue;
195
+ }
196
+ if (landmarks.has(child) || isUnlikelyCandidate(child)) {
197
+ doomed.push(child);
198
+ continue;
199
+ }
200
+ visit(child);
201
+ }
202
+ };
203
+ visit(tree);
204
+ return doomed;
205
+ };
206
+ const stripUnlikely = (tree) => {
207
+ const doomed = new Set(findUnlikelyElements(tree));
208
+ return (0, import_hast_fast.pruneInPlace)(tree, (node) => !((0, import_hast_fast.isElement)(node) && doomed.has(node)));
209
+ };
210
+ const isEmbeddedContent = (element) => {
211
+ if (import_constants.EMBEDDED_CONTENT_TAGS.has(element.tagName)) {
212
+ return true;
213
+ }
214
+ return element.tagName === "br" || element.tagName === "hr" || element.tagName === "input";
215
+ };
216
+ const collectPreformattedElements = (tree) => {
217
+ const protectedElements = /* @__PURE__ */ new Set();
218
+ const markSubtree = (node) => {
219
+ if (!("children" in node)) {
220
+ return;
221
+ }
222
+ for (const child of node.children) {
223
+ if ((0, import_hast_fast.isElement)(child)) {
224
+ protectedElements.add(child);
225
+ markSubtree(child);
226
+ }
227
+ }
228
+ };
229
+ (0, import_hast_fast.walk)(tree, (node) => {
230
+ if ((0, import_hast_fast.isElement)(node) && (node.tagName === "pre" || node.tagName === "code")) {
231
+ protectedElements.add(node);
232
+ markSubtree(node);
233
+ return false;
234
+ }
235
+ });
236
+ return protectedElements;
237
+ };
238
+ const dropEmptyElements = (tree, collector) => {
239
+ const preformatted = collectPreformattedElements(tree);
240
+ const visit = (parent) => {
241
+ for (let index = 0; index < parent.children.length; index++) {
242
+ const child = parent.children[index];
243
+ if (!(0, import_hast_fast.isElement)(child)) {
244
+ continue;
245
+ }
246
+ if (preformatted.has(child) || !import_constants.DROP_WHEN_EMPTY_TAGS.has(child.tagName)) {
247
+ visit(child);
248
+ continue;
249
+ }
250
+ if (collector.metrics(child).text > 0 || hasEmbeddedDescendant(child)) {
251
+ visit(child);
252
+ continue;
253
+ }
254
+ if (holdsWhitespace(child)) {
255
+ parent.children[index] = { type: "text", value: " " };
256
+ continue;
257
+ }
258
+ parent.children.splice(index, 1);
259
+ index -= 1;
260
+ }
261
+ };
262
+ if ("children" in tree) {
263
+ visit(tree);
264
+ }
265
+ return tree;
266
+ };
267
+ const holdsWhitespace = (element) => {
268
+ for (const child of element.children) {
269
+ if (child.type === "text" && /\s/.test(child.value)) {
270
+ return true;
271
+ }
272
+ if ((0, import_hast_fast.isElement)(child) && holdsWhitespace(child)) {
273
+ return true;
274
+ }
275
+ }
276
+ return false;
277
+ };
278
+ const hasEmbeddedDescendant = (element) => {
279
+ for (const child of element.children) {
280
+ if (!(0, import_hast_fast.isElement)(child)) {
281
+ continue;
282
+ }
283
+ if (isEmbeddedContent(child) || hasEmbeddedDescendant(child)) {
284
+ return true;
285
+ }
286
+ }
287
+ return false;
288
+ };
289
+ const REFERENCE_HEADING = /^(see also|references?|further reading|external links?|bibliography|sources|footnotes|citations|関連項目|参考文献|外部リンク|脚注|出典|参考リンク|参考資料)$/i;
290
+ const MAX_DANGLING_HEADING_LENGTH = 60;
291
+ const isHeadingTag = (element) => /^h[1-6]$/.test(element.tagName);
292
+ const previousElement = (siblings, index) => {
293
+ for (let cursor = index - 1; cursor >= 0; cursor--) {
294
+ const sibling = siblings[cursor];
295
+ if ((0, import_hast_fast.isElement)(sibling)) {
296
+ return sibling;
297
+ }
298
+ if (sibling.type === "text" && sibling.value.trim().length > 0) {
299
+ return void 0;
300
+ }
301
+ }
302
+ return void 0;
303
+ };
304
+ const condemnLinkBlock = (block, siblings, index, collector, doomed) => {
305
+ const heading = previousElement(siblings, index);
306
+ const isHeading = heading !== void 0 && isHeadingTag(heading);
307
+ if (isHeading && REFERENCE_HEADING.test(elementText(heading).trim())) {
308
+ return;
309
+ }
310
+ if (introducedByColon(siblings, index, collector)) {
311
+ return;
312
+ }
313
+ doomed.push(block);
314
+ if (isHeading && elementText(heading).trim().length <= MAX_DANGLING_HEADING_LENGTH) {
315
+ doomed.push(heading);
316
+ }
317
+ };
318
+ const introducedByColon = (siblings, index, collector) => {
319
+ const before = previousElement(siblings, index);
320
+ if (!before || collector.metrics(before).text === 0) {
321
+ return false;
322
+ }
323
+ return /[::]\s*$/.test(elementText(before).trim());
324
+ };
325
+ const findCleanupTargets = (tree, collector) => {
326
+ const doomed = [];
327
+ const visit = (node) => {
328
+ if (!("children" in node)) {
329
+ return;
330
+ }
331
+ const siblings = node.children;
332
+ for (let index = 0; index < siblings.length; index++) {
333
+ const child = siblings[index];
334
+ if (!(0, import_hast_fast.isElement)(child)) {
335
+ continue;
336
+ }
337
+ if (child.tagName === "article" || child.tagName === "main") {
338
+ visit(child);
339
+ continue;
340
+ }
341
+ if (isLinkOnlyBlock(child, collector)) {
342
+ condemnLinkBlock(child, siblings, index, collector, doomed);
343
+ continue;
344
+ }
345
+ if (isMidArticleWidget(child)) {
346
+ doomed.push(child);
347
+ continue;
348
+ }
349
+ visit(child);
350
+ }
351
+ };
352
+ visit(tree);
353
+ return doomed;
354
+ };
355
+ const LINK_BLOCK_TAGS = /* @__PURE__ */ new Set(["ul", "ol", "nav", "div", "p"]);
356
+ const MIN_LINK_BLOCK_ANCHORS = 3;
357
+ const isLinkOnlyBlock = (element, collector) => {
358
+ if (!LINK_BLOCK_TAGS.has(element.tagName)) {
359
+ return false;
360
+ }
361
+ const metrics = collector.metrics(element);
362
+ if (metrics.text === 0 || metrics.commas > 0) {
363
+ return false;
364
+ }
365
+ if (metrics.link / metrics.text <= 0.9) {
366
+ return false;
367
+ }
368
+ return countAnchors(element) >= MIN_LINK_BLOCK_ANCHORS;
369
+ };
370
+ const countAnchors = (element) => {
371
+ let count = 0;
372
+ const stack = [element];
373
+ while (stack.length > 0 && count < MIN_LINK_BLOCK_ANCHORS) {
374
+ const node = stack.pop();
375
+ if ((0, import_hast_fast.isElement)(node) && node.tagName === "a") {
376
+ count += 1;
377
+ }
378
+ if ("children" in node) {
379
+ for (const child of node.children) {
380
+ stack.push(child);
381
+ }
382
+ }
383
+ }
384
+ return count;
385
+ };
386
+ const isMidArticleWidget = (element) => {
387
+ if (element.tagName === "form" || element.tagName === "fieldset") {
388
+ return true;
389
+ }
390
+ if (element.tagName === "button") {
391
+ return true;
392
+ }
393
+ return (0, import_ui_chrome.isUiChromeText)(elementText(element));
394
+ };
395
+ const cleanContent = (tree, collector) => {
396
+ const doomed = new Set(findCleanupTargets(tree, collector));
397
+ if (doomed.size > 0) {
398
+ (0, import_hast_fast.pruneInPlace)(tree, (node) => !((0, import_hast_fast.isElement)(node) && doomed.has(node)));
399
+ }
400
+ return dropEmptyElements(tree, new import_hast_fast.MetricsCollector());
401
+ };
402
+ const elementText = (node) => {
403
+ if (node.type === "text") {
404
+ return node.value;
405
+ }
406
+ if (!("children" in node)) {
407
+ return "";
408
+ }
409
+ let text = "";
410
+ for (const child of node.children) {
411
+ text += elementText(child);
412
+ }
413
+ return text;
414
+ };
415
+ // Annotate the CommonJS export names for ESM import in node:
416
+ 0 && (module.exports = {
417
+ cleanContent,
418
+ dropEmptyElements,
419
+ findCleanupTargets,
420
+ findLandmarkChrome,
421
+ findUnlikelyElements,
422
+ hoistNoscriptImages,
423
+ stripNonContent,
424
+ stripUnlikely
425
+ });
@@ -0,0 +1,264 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var score_exports = {};
20
+ __export(score_exports, {
21
+ buildScoringContext: () => buildScoringContext,
22
+ classWeight: () => classWeight,
23
+ climbToArticleBoundary: () => climbToArticleBoundary,
24
+ collectArticleNodes: () => collectArticleNodes,
25
+ geometryMultiplier: () => geometryMultiplier,
26
+ scoreCandidates: () => scoreCandidates,
27
+ semanticBonus: () => semanticBonus
28
+ });
29
+ module.exports = __toCommonJS(score_exports);
30
+ var import_hast_fast = require("../../utils/hast-fast");
31
+ var import_constants = require("./constants");
32
+ const TAG_BASE_SCORE = {
33
+ div: 5,
34
+ pre: 3,
35
+ td: 3,
36
+ blockquote: 3,
37
+ article: 10,
38
+ main: 10,
39
+ section: 5,
40
+ address: -3,
41
+ ol: -3,
42
+ ul: -3,
43
+ dl: -3,
44
+ dd: -3,
45
+ dt: -3,
46
+ li: -3,
47
+ form: -3,
48
+ h1: -5,
49
+ h2: -5,
50
+ h3: -5,
51
+ h4: -5,
52
+ h5: -5,
53
+ h6: -5,
54
+ th: -5
55
+ };
56
+ const MAX_ANCESTOR_DEPTH = 5;
57
+ const MIN_SCOREABLE_TEXT = 25;
58
+ const buildScoringContext = (root, collector, lang) => {
59
+ const parents = /* @__PURE__ */ new WeakMap();
60
+ const scoreable = [];
61
+ let hasGeometry = false;
62
+ (0, import_hast_fast.walk)(root, (node, parent) => {
63
+ if (parent) {
64
+ parents.set(node, parent);
65
+ }
66
+ if (!(0, import_hast_fast.isElement)(node)) {
67
+ return;
68
+ }
69
+ if (!hasGeometry && (0, import_hast_fast.numericProperty)(node, "data-rheight") !== void 0) {
70
+ hasGeometry = true;
71
+ }
72
+ if (isParagraphLike(node)) {
73
+ scoreable.push(node);
74
+ }
75
+ });
76
+ return { collector, parents, lang, hasGeometry, scoreable };
77
+ };
78
+ const classWeight = (element) => {
79
+ let weight = 0;
80
+ const id = (0, import_hast_fast.stringProperty)(element, "id") ?? "";
81
+ const classes = (0, import_hast_fast.classList)(element).join(" ");
82
+ if (import_constants.REGEXPS.negative.test(classes)) {
83
+ weight -= 25;
84
+ }
85
+ if (import_constants.REGEXPS.positive.test(classes)) {
86
+ weight += 25;
87
+ }
88
+ if (import_constants.REGEXPS.negative.test(id)) {
89
+ weight -= 25;
90
+ }
91
+ if (import_constants.REGEXPS.positive.test(id)) {
92
+ weight += 25;
93
+ }
94
+ return weight;
95
+ };
96
+ const semanticBonus = (element) => {
97
+ let bonus = 0;
98
+ if (element.tagName === "article") {
99
+ bonus += 60;
100
+ }
101
+ if (element.tagName === "main") {
102
+ bonus += 50;
103
+ }
104
+ const role = (0, import_hast_fast.stringProperty)(element, "role");
105
+ if (role && import_constants.CONTENT_ROLES.has(role)) {
106
+ bonus += 40;
107
+ }
108
+ const itemprop = (0, import_hast_fast.stringProperty)(element, "itemprop");
109
+ if (itemprop?.includes("articleBody")) {
110
+ bonus += 80;
111
+ }
112
+ const match = (0, import_hast_fast.matchString)(element);
113
+ if (/(^|[\s_-])(markdown|prose|article-?content|post-?content|entry-?content|doc-?content|md-?content)([\s_-]|$)/i.test(
114
+ match
115
+ )) {
116
+ bonus += 35;
117
+ }
118
+ return bonus;
119
+ };
120
+ const geometryMultiplier = (element, context) => {
121
+ if (!context.hasGeometry) {
122
+ return 1;
123
+ }
124
+ const height = (0, import_hast_fast.numericProperty)(element, "data-rheight");
125
+ const width = (0, import_hast_fast.numericProperty)(element, "data-rwidth");
126
+ if (height === void 0 || width === void 0) {
127
+ return 1;
128
+ }
129
+ if (height === 0 || width === 0) {
130
+ return 0;
131
+ }
132
+ if (width < 220) {
133
+ return 0.35;
134
+ }
135
+ return 1;
136
+ };
137
+ const paragraphScore = (textLength, commas, densityDivisor) => 1 + commas + Math.min(Math.floor(textLength / densityDivisor), 3);
138
+ const ancestorWeight = (level) => {
139
+ if (level === 0) {
140
+ return 1;
141
+ }
142
+ if (level === 1) {
143
+ return 2;
144
+ }
145
+ return level * 3;
146
+ };
147
+ const isParagraphLike = (element) => {
148
+ if (!import_constants.SCOREABLE_TAGS.has(element.tagName)) {
149
+ return false;
150
+ }
151
+ if (element.tagName !== "div" && element.tagName !== "section") {
152
+ return true;
153
+ }
154
+ return !element.children.some((child) => (0, import_hast_fast.isElement)(child) && import_constants.CANDIDATE_TAGS.has(child.tagName));
155
+ };
156
+ const scoreCandidates = (_root, context) => {
157
+ const { collector, parents } = context;
158
+ const scores = /* @__PURE__ */ new Map();
159
+ const densityDivisor = (0, import_constants.minContentLength)(context.lang) / 4;
160
+ for (const node of context.scoreable) {
161
+ const metrics = collector.metrics(node);
162
+ if (metrics.text < MIN_SCOREABLE_TEXT) {
163
+ continue;
164
+ }
165
+ const contribution = paragraphScore(metrics.text, metrics.commas, densityDivisor);
166
+ let current = node;
167
+ for (let level = 0; level < MAX_ANCESTOR_DEPTH && current; level++) {
168
+ const parent = parents.get(current);
169
+ if (!(parent && (0, import_hast_fast.isElement)(parent))) {
170
+ break;
171
+ }
172
+ if (import_constants.CANDIDATE_TAGS.has(parent.tagName) || parent.tagName === "article" || parent.tagName === "main") {
173
+ const base = scores.get(parent) ?? (TAG_BASE_SCORE[parent.tagName] ?? 0) + classWeight(parent) + semanticBonus(parent);
174
+ scores.set(parent, base + contribution / ancestorWeight(level));
175
+ }
176
+ current = parent;
177
+ }
178
+ }
179
+ const candidates = [];
180
+ for (const [element, rawScore] of scores) {
181
+ const density = collector.linkDensity(element);
182
+ const score = rawScore * (1 - density) * geometryMultiplier(element, context);
183
+ if (score > 0) {
184
+ candidates.push({ element, score });
185
+ }
186
+ }
187
+ candidates.sort((a, b) => b.score - a.score);
188
+ return candidates;
189
+ };
190
+ const collectArticleNodes = (top, candidates, context) => {
191
+ const { collector, parents } = context;
192
+ const parent = parents.get(top.element);
193
+ if (!(parent && (0, import_hast_fast.isElement)(parent))) {
194
+ return [top.element];
195
+ }
196
+ const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
197
+ const threshold = Math.max(10, top.score * 0.2);
198
+ const selected = [];
199
+ for (const child of parent.children) {
200
+ if (!(0, import_hast_fast.isElement)(child)) {
201
+ continue;
202
+ }
203
+ if (child === top.element) {
204
+ selected.push(child);
205
+ continue;
206
+ }
207
+ const siblingScore = scoreOf.get(child) ?? 0;
208
+ if (siblingScore >= threshold) {
209
+ selected.push(child);
210
+ continue;
211
+ }
212
+ if (child.tagName === "p" || child.tagName === "section") {
213
+ const metrics = collector.metrics(child);
214
+ const density = collector.linkDensity(child);
215
+ if (metrics.text > 80 && density < 0.25) {
216
+ selected.push(child);
217
+ }
218
+ }
219
+ }
220
+ return selected.length > 0 ? selected : [top.element];
221
+ };
222
+ const MAX_CLIMB_HOPS = 5;
223
+ const MIN_CLIMB_RELATIVE_GAIN = 0.08;
224
+ const MIN_CLIMB_ABSOLUTE_GAIN = 500;
225
+ const MAX_CLIMB_GAIN_LINK_DENSITY = 1 / 3;
226
+ const CLIMB_SCORE_FLOOR = 1 / 3;
227
+ const climbToArticleBoundary = (top, candidates, context) => {
228
+ const { collector, parents } = context;
229
+ const scoreOf = new Map(candidates.map((candidate) => [candidate.element, candidate.score]));
230
+ const scoreFloor = top.score * CLIMB_SCORE_FLOOR;
231
+ let current = top.element;
232
+ for (let hop = 0; hop < MAX_CLIMB_HOPS; hop++) {
233
+ const parent = parents.get(current);
234
+ if (!(parent && (0, import_hast_fast.isElement)(parent))) {
235
+ break;
236
+ }
237
+ const parentScore = scoreOf.get(parent);
238
+ if (parentScore === void 0 || parentScore < scoreFloor) {
239
+ break;
240
+ }
241
+ const currentMetrics = collector.metrics(current);
242
+ const parentMetrics = collector.metrics(parent);
243
+ const gainedText = parentMetrics.text - currentMetrics.text;
244
+ if (gainedText < Math.max(MIN_CLIMB_ABSOLUTE_GAIN, currentMetrics.text * MIN_CLIMB_RELATIVE_GAIN)) {
245
+ break;
246
+ }
247
+ const gainedLink = parentMetrics.link - currentMetrics.link;
248
+ if (gainedLink / gainedText > MAX_CLIMB_GAIN_LINK_DENSITY) {
249
+ break;
250
+ }
251
+ current = parent;
252
+ }
253
+ return current;
254
+ };
255
+ // Annotate the CommonJS export names for ESM import in node:
256
+ 0 && (module.exports = {
257
+ buildScoringContext,
258
+ classWeight,
259
+ climbToArticleBoundary,
260
+ collectArticleNodes,
261
+ geometryMultiplier,
262
+ scoreCandidates,
263
+ semanticBonus
264
+ });