webforai 3.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/README.md +47 -5
  2. package/dist/bin.js +3109 -490
  3. package/dist/cjs/detect-client-shell.js +78 -0
  4. package/dist/cjs/extract-mdast.js +58 -0
  5. package/dist/cjs/extract-meta-refresh.js +71 -0
  6. package/dist/cjs/extractors/index.js +5 -0
  7. package/dist/cjs/extractors/lib/block-features.js +465 -0
  8. package/dist/cjs/extractors/lib/block-model.generated.js +45 -0
  9. package/dist/cjs/extractors/lib/block-model.js +72 -0
  10. package/dist/cjs/extractors/lib/blocks.js +212 -0
  11. package/dist/cjs/extractors/lib/code-tabs.js +191 -0
  12. package/dist/cjs/extractors/lib/constants.js +1 -1
  13. package/dist/cjs/extractors/lib/layout-tables.js +123 -0
  14. package/dist/cjs/extractors/lib/sanitize.js +146 -26
  15. package/dist/cjs/extractors/lib/takumi-signal.js +76 -0
  16. package/dist/cjs/extractors/lib/ui-chrome.js +7 -0
  17. package/dist/cjs/extractors/presets/auto.js +3 -3
  18. package/dist/cjs/extractors/presets/kiwame.js +231 -0
  19. package/dist/cjs/extractors/presets/takumi.js +34 -3
  20. package/dist/cjs/html-to-markdown.js +58 -4
  21. package/dist/cjs/html-to-mdast.js +4 -3
  22. package/dist/cjs/index.js +16 -0
  23. package/dist/cjs/link-replacer.js +19 -13
  24. package/dist/cjs/loaders/cf-puppeteer.js +14 -7
  25. package/dist/cjs/loaders/fetch.js +14 -2
  26. package/dist/cjs/loaders/playwright.js +32 -30
  27. package/dist/cjs/loaders/puppeteer.js +14 -7
  28. package/dist/cjs/md-splitter.js +24 -11
  29. package/dist/cjs/mdast-handlers/custom-code-handler.js +32 -9
  30. package/dist/cjs/mdast-handlers/custom-div-handler.js +14 -35
  31. package/dist/cjs/mdast-handlers/custom-img-handler.js +10 -1
  32. package/dist/cjs/mdast-handlers/custom-table-handler.js +18 -1
  33. package/dist/cjs/mdast-handlers/definition-list-handler.js +17 -6
  34. package/dist/cjs/mdast-handlers/table-cell-flatten.js +195 -0
  35. package/dist/cjs/mdast-handlers/twoslash.js +71 -0
  36. package/dist/cjs/mdast-to-markdown.js +2 -5
  37. package/dist/cjs/metadata/index.js +4 -3
  38. package/dist/cjs/normalize/index.js +11 -2
  39. package/dist/cjs/platform/client.js +73 -16
  40. package/dist/cjs/platform/index.js +4 -0
  41. package/dist/cjs/platform/types.js +7 -1
  42. package/dist/cjs/utils/hast-fast.js +1 -1
  43. package/dist/cjs/utils/parse-html.js +113 -0
  44. package/dist/cjs/utils/pre-text.js +104 -0
  45. package/dist/detect-client-shell.js +53 -0
  46. package/dist/extract-mdast.js +58 -0
  47. package/dist/extract-meta-refresh.js +47 -0
  48. package/dist/extractors/index.js +3 -0
  49. package/dist/extractors/lib/block-features.js +437 -0
  50. package/dist/extractors/lib/block-model.generated.js +21 -0
  51. package/dist/extractors/lib/block-model.js +48 -0
  52. package/dist/extractors/lib/blocks.js +185 -0
  53. package/dist/extractors/lib/code-tabs.js +168 -0
  54. package/dist/extractors/lib/constants.js +1 -1
  55. package/dist/extractors/lib/layout-tables.js +99 -0
  56. package/dist/extractors/lib/sanitize.js +144 -26
  57. package/dist/extractors/lib/takumi-signal.js +53 -0
  58. package/dist/extractors/lib/ui-chrome.js +5 -0
  59. package/dist/extractors/presets/auto.js +3 -3
  60. package/dist/extractors/presets/kiwame.js +206 -0
  61. package/dist/extractors/presets/takumi.js +34 -3
  62. package/dist/html-to-markdown.js +57 -4
  63. package/dist/html-to-mdast.js +4 -3
  64. package/dist/index.js +18 -0
  65. package/dist/link-replacer.js +19 -13
  66. package/dist/loaders/cf-puppeteer.js +15 -8
  67. package/dist/loaders/fetch.js +14 -2
  68. package/dist/loaders/playwright.js +32 -30
  69. package/dist/loaders/puppeteer.js +15 -8
  70. package/dist/md-splitter.js +24 -11
  71. package/dist/mdast-handlers/custom-code-handler.js +30 -9
  72. package/dist/mdast-handlers/custom-div-handler.js +14 -35
  73. package/dist/mdast-handlers/custom-img-handler.js +10 -1
  74. package/dist/mdast-handlers/custom-table-handler.js +19 -2
  75. package/dist/mdast-handlers/definition-list-handler.js +17 -6
  76. package/dist/mdast-handlers/table-cell-flatten.js +170 -0
  77. package/dist/mdast-handlers/twoslash.js +48 -0
  78. package/dist/mdast-to-markdown.js +2 -5
  79. package/dist/metadata/index.js +4 -3
  80. package/dist/normalize/index.js +12 -2
  81. package/dist/platform/client.js +73 -16
  82. package/dist/platform/index.js +4 -0
  83. package/dist/platform/types.js +5 -1
  84. package/dist/types/cli/bin.js.map +1 -1
  85. package/dist/types/cli/commands/jobs/files.d.ts +28 -0
  86. package/dist/types/cli/commands/jobs/files.js.map +1 -0
  87. package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
  88. package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
  89. package/dist/types/cli/commands/jobs/options.d.ts +58 -0
  90. package/dist/types/cli/commands/jobs/options.js.map +1 -0
  91. package/dist/types/cli/commands/jobs/run.d.ts +44 -0
  92. package/dist/types/cli/commands/jobs/run.js.map +1 -0
  93. package/dist/types/cli/commands/skill/content.js.map +1 -1
  94. package/dist/types/cli/commands/webforai/interactive.d.ts +1 -1
  95. package/dist/types/cli/commands/webforai/interactive.js.map +1 -1
  96. package/dist/types/cli/commands/webforai/options.d.ts +9 -2
  97. package/dist/types/cli/commands/webforai/options.js.map +1 -1
  98. package/dist/types/cli/commands/webforai/run.js.map +1 -1
  99. package/dist/types/cli/constants.d.ts +3 -1
  100. package/dist/types/cli/constants.js.map +1 -1
  101. package/dist/types/cli/utils.d.ts +8 -1
  102. package/dist/types/cli/utils.js.map +1 -1
  103. package/dist/types/detect-client-shell.d.ts +24 -0
  104. package/dist/types/detect-client-shell.js.map +1 -0
  105. package/dist/types/extract-mdast.js.map +1 -1
  106. package/dist/types/extract-meta-refresh.d.ts +29 -0
  107. package/dist/types/extract-meta-refresh.js.map +1 -0
  108. package/dist/types/extractors/index.d.ts +1 -0
  109. package/dist/types/extractors/index.js.map +1 -1
  110. package/dist/types/extractors/lib/block-features.d.ts +43 -0
  111. package/dist/types/extractors/lib/block-features.js.map +1 -0
  112. package/dist/types/extractors/lib/block-model.d.ts +28 -0
  113. package/dist/types/extractors/lib/block-model.generated.d.ts +5 -0
  114. package/dist/types/extractors/lib/block-model.generated.js.map +1 -0
  115. package/dist/types/extractors/lib/block-model.js.map +1 -0
  116. package/dist/types/extractors/lib/blocks.d.ts +53 -0
  117. package/dist/types/extractors/lib/blocks.js.map +1 -0
  118. package/dist/types/extractors/lib/code-tabs.d.ts +30 -0
  119. package/dist/types/extractors/lib/code-tabs.js.map +1 -0
  120. package/dist/types/extractors/lib/constants.js.map +1 -1
  121. package/dist/types/extractors/lib/layout-tables.d.ts +17 -0
  122. package/dist/types/extractors/lib/layout-tables.js.map +1 -0
  123. package/dist/types/extractors/lib/sanitize.d.ts +33 -3
  124. package/dist/types/extractors/lib/sanitize.js.map +1 -1
  125. package/dist/types/extractors/lib/takumi-signal.d.ts +12 -0
  126. package/dist/types/extractors/lib/takumi-signal.js.map +1 -0
  127. package/dist/types/extractors/lib/ui-chrome.d.ts +8 -0
  128. package/dist/types/extractors/lib/ui-chrome.js.map +1 -1
  129. package/dist/types/extractors/presets/auto.d.ts +6 -1
  130. package/dist/types/extractors/presets/auto.js.map +1 -1
  131. package/dist/types/extractors/presets/kiwame.d.ts +31 -0
  132. package/dist/types/extractors/presets/kiwame.js.map +1 -0
  133. package/dist/types/extractors/presets/takumi.js.map +1 -1
  134. package/dist/types/html-to-markdown.d.ts +9 -0
  135. package/dist/types/html-to-markdown.js.map +1 -1
  136. package/dist/types/html-to-mdast.js.map +1 -1
  137. package/dist/types/index.d.ts +3 -1
  138. package/dist/types/index.js.map +1 -1
  139. package/dist/types/link-replacer.d.ts +3 -1
  140. package/dist/types/link-replacer.js.map +1 -1
  141. package/dist/types/loaders/cf-puppeteer.js.map +1 -1
  142. package/dist/types/loaders/fetch.d.ts +4 -0
  143. package/dist/types/loaders/fetch.js.map +1 -1
  144. package/dist/types/loaders/playwright.js.map +1 -1
  145. package/dist/types/loaders/puppeteer.js.map +1 -1
  146. package/dist/types/md-splitter.d.ts +1 -1
  147. package/dist/types/md-splitter.js.map +1 -1
  148. package/dist/types/mdast-handlers/custom-code-handler.d.ts +9 -0
  149. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
  150. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  151. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  152. package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
  153. package/dist/types/mdast-handlers/definition-list-handler.d.ts +0 -11
  154. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -1
  155. package/dist/types/mdast-handlers/table-cell-flatten.d.ts +18 -0
  156. package/dist/types/mdast-handlers/table-cell-flatten.js.map +1 -0
  157. package/dist/types/mdast-handlers/twoslash.d.ts +14 -0
  158. package/dist/types/mdast-handlers/twoslash.js.map +1 -0
  159. package/dist/types/mdast-to-markdown.js.map +1 -1
  160. package/dist/types/metadata/index.js.map +1 -1
  161. package/dist/types/normalize/index.js.map +1 -1
  162. package/dist/types/platform/client.d.ts +2 -1
  163. package/dist/types/platform/client.js.map +1 -1
  164. package/dist/types/platform/error.d.ts +5 -3
  165. package/dist/types/platform/error.js.map +1 -1
  166. package/dist/types/platform/fetch.d.ts +2 -0
  167. package/dist/types/platform/fetch.js.map +1 -1
  168. package/dist/types/platform/index.d.ts +1 -1
  169. package/dist/types/platform/index.js.map +1 -1
  170. package/dist/types/platform/types.d.ts +35 -6
  171. package/dist/types/platform/types.js.map +1 -1
  172. package/dist/types/utils/hast-fast.d.ts +1 -1
  173. package/dist/types/utils/hast-fast.js.map +1 -1
  174. package/dist/types/utils/parse-html.d.ts +33 -0
  175. package/dist/types/utils/parse-html.js.map +1 -0
  176. package/dist/types/utils/pre-text.d.ts +12 -0
  177. package/dist/types/utils/pre-text.js.map +1 -0
  178. package/dist/utils/hast-fast.js +1 -1
  179. package/dist/utils/parse-html.js +87 -0
  180. package/dist/utils/pre-text.js +81 -0
  181. package/package.json +18 -5
@@ -0,0 +1,168 @@
1
+ // src/extractors/lib/code-tabs.ts
2
+ import { isElement, stringProperty } from "../../utils/hast-fast.js";
3
+ var MAX_LABEL_LENGTH = 80;
4
+ var CodeTabCollector = class {
5
+ #groups = [];
6
+ /** Groups whose tab list's parent is still being traversed, innermost last. */
7
+ #open = [];
8
+ #isHidden;
9
+ #radioStrips = [];
10
+ /** @param isHidden - Elements whose text a reader never sees, excluded from labels. */
11
+ constructor(isHidden) {
12
+ this.#isHidden = isHidden;
13
+ }
14
+ #applyRadioStrip(strip, parent) {
15
+ const labels = strip.children.filter((child) => isElement(child) && child.tagName === "label");
16
+ const panels = radioStripPanels(strip, parent);
17
+ if (!panels || panels.length !== labels.length) {
18
+ return;
19
+ }
20
+ panels.forEach((panel, index) => labelCode(panel, tabLabel(labels[index], this.#isHidden)));
21
+ parent.children.splice(parent.children.indexOf(strip), 1);
22
+ }
23
+ /**
24
+ * Call for every element, in document order, with its depth.
25
+ *
26
+ * A group only collects panels inside the tab list's parent: a page-level tab strip must not
27
+ * lend its labels to an unrelated code switcher further down, and an inner group closes before
28
+ * the outer group's next panel.
29
+ */
30
+ visit(element, depth, parent) {
31
+ if (isRadioTabStrip(element)) {
32
+ this.#radioStrips.push({ strip: element, parent });
33
+ return;
34
+ }
35
+ while (this.#open.length > 0 && depth < (this.#open.at(-1)?.depth ?? 0)) {
36
+ this.#open.pop();
37
+ }
38
+ const role = stringProperty(element, "role");
39
+ if (role === "tablist") {
40
+ const group2 = { tabs: [], panels: [] };
41
+ this.#groups.push(group2);
42
+ this.#open.push({ group: group2, depth });
43
+ return;
44
+ }
45
+ const group = this.#open.at(-1)?.group;
46
+ if (!group) {
47
+ return;
48
+ }
49
+ if (role === "tab") {
50
+ group.tabs.push(element);
51
+ } else if (role === "tabpanel") {
52
+ group.panels.push(element);
53
+ }
54
+ }
55
+ /** Labels the code block of every panel whose tab could be identified. */
56
+ apply() {
57
+ for (const { strip, parent } of this.#radioStrips) {
58
+ this.#applyRadioStrip(strip, parent);
59
+ }
60
+ for (const group of this.#groups) {
61
+ if (group.tabs.length < 2 || group.panels.length === 0) {
62
+ continue;
63
+ }
64
+ for (const [panel, tab] of matchTabs(group)) {
65
+ labelCode(panel, tabLabel(tab, this.#isHidden));
66
+ }
67
+ }
68
+ }
69
+ };
70
+ var radioStripPanels = (strip, parent) => {
71
+ const index = parent.children.indexOf(strip);
72
+ for (let cursor = index + 1; cursor < parent.children.length; cursor++) {
73
+ const sibling = parent.children[cursor];
74
+ if (isElement(sibling)) {
75
+ return sibling.children.filter(isElement);
76
+ }
77
+ }
78
+ return void 0;
79
+ };
80
+ var isRadioTabStrip = (element) => {
81
+ const first = element.children.find(isElement);
82
+ if (!(first?.tagName === "input" && stringProperty(first, "type") === "radio")) {
83
+ return false;
84
+ }
85
+ let labels = 0;
86
+ for (const child of element.children) {
87
+ if (!isElement(child)) {
88
+ continue;
89
+ }
90
+ if (child.tagName === "label") {
91
+ labels += 1;
92
+ } else if (!(child.tagName === "input" && stringProperty(child, "type") === "radio")) {
93
+ return false;
94
+ }
95
+ }
96
+ return labels >= 2;
97
+ };
98
+ var indexById = (elements) => {
99
+ const byId = /* @__PURE__ */ new Map();
100
+ for (const element of elements) {
101
+ const id = stringProperty(element, "id");
102
+ if (id) {
103
+ byId.set(id, element);
104
+ }
105
+ }
106
+ return byId;
107
+ };
108
+ var matchTabs = ({ tabs, panels }) => {
109
+ const tabById = indexById(tabs);
110
+ const panelById = indexById(panels);
111
+ const tabOf = /* @__PURE__ */ new Map();
112
+ for (const panel of panels) {
113
+ const tab = tabById.get(stringProperty(panel, "ariaLabelledBy") ?? "");
114
+ if (tab) {
115
+ tabOf.set(panel, tab);
116
+ }
117
+ }
118
+ for (const tab of tabs) {
119
+ const panel = panelById.get(stringProperty(tab, "ariaControls") ?? "");
120
+ if (panel && !tabOf.has(panel)) {
121
+ tabOf.set(panel, tab);
122
+ }
123
+ }
124
+ if (tabOf.size === 0 && tabs.length === panels.length) {
125
+ panels.forEach((panel, index) => tabOf.set(panel, tabs[index]));
126
+ }
127
+ return tabOf;
128
+ };
129
+ var tabLabel = (tab, isHidden) => {
130
+ const label = (textOf(tab, isHidden) || stringProperty(tab, "dataTitle") || "").replace(/\s+/g, " ").trim();
131
+ return label.length > 0 && label.length <= MAX_LABEL_LENGTH ? label : void 0;
132
+ };
133
+ var labelCode = (panel, label) => {
134
+ if (!label) {
135
+ return;
136
+ }
137
+ const blocks = [];
138
+ collectPre(panel, blocks);
139
+ if (blocks.length === 1 && stringProperty(blocks[0], "dataTitle") === void 0) {
140
+ blocks[0].properties.dataTitle = label;
141
+ }
142
+ };
143
+ var collectPre = (element, into) => {
144
+ for (const child of element.children) {
145
+ if (!isElement(child) || into.length > 1) {
146
+ continue;
147
+ }
148
+ if (child.tagName === "pre") {
149
+ into.push(child);
150
+ continue;
151
+ }
152
+ collectPre(child, into);
153
+ }
154
+ };
155
+ var textOf = (element, isHidden) => {
156
+ let text = "";
157
+ for (const child of element.children) {
158
+ if (child.type === "text") {
159
+ text += child.value;
160
+ } else if (isElement(child) && !isHidden(child)) {
161
+ text += textOf(child, isHidden);
162
+ }
163
+ }
164
+ return text;
165
+ };
166
+ export {
167
+ CodeTabCollector
168
+ };
@@ -8,7 +8,7 @@ var REGEXPS = {
8
8
  * `class="post-header"`), so matching them here deletes the page's headings. The landmark
9
9
  * versions of those elements are handled structurally instead.
10
10
  */
11
- unlikelyCandidates: /-ad-|ai2html|banner|breadcrumb|combx|comment(?!ary)|community|cover-wrap|disqus|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|site-index|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar|subscribe|newsletter|paywall|consent|cookie|toolbar|masthead|promo|share|sharing|widget|tooltip|dropdown|modal|overlay|backdrop|carousel|announcement|notification|toast|skip-link|screen-reader|visually-hidden|sr-only/i,
11
+ unlikelyCandidates: /-ad-|ai2html|banner|breadcrumb|combx|comment(?!ary)|community|cover-wrap|disqus|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|site-index|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar|subscribe|newsletter|paywall|consent|cookie|toolbar|masthead|promo|share|sharing|widget|tooltip|dropdown|modal|overlay|backdrop|carousel|announcement|notification|toast|skip-link|screen-reader|(?<!not-)visually-hidden|(?<!not-)sr-only/i,
12
12
  /** Rescues containers whose class merely happens to contain an unlikely substring. */
13
13
  okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code|post|entry|story|markdown|prose|doc/i,
14
14
  /** Very high-confidence boilerplate, not rescued by the clause above. */
@@ -0,0 +1,99 @@
1
+ // src/extractors/lib/layout-tables.ts
2
+ import { isElement, stringProperty } from "../../utils/hast-fast.js";
3
+ var DATA_MARKERS = /* @__PURE__ */ new Set(["caption", "thead", "tfoot", "th", "col", "colgroup"]);
4
+ var BLOCK_IN_CELL = /* @__PURE__ */ new Set(["h1", "h2", "h3", "h4", "h5", "h6", "table", "form"]);
5
+ var TABLE_PARTS = /* @__PURE__ */ new Set(["tbody", "thead", "tfoot", "tr", "td", "th"]);
6
+ var TABLE_CONTEXT = /* @__PURE__ */ new Set(["table", "tbody", "thead", "tfoot", "tr"]);
7
+ var shapeOf = (table) => {
8
+ const shape = { rows: 0, columns: 0, dataMarker: false, blockInCell: false };
9
+ const visit = (node, inCell) => {
10
+ for (const child of node.children) {
11
+ if (!isElement(child)) {
12
+ continue;
13
+ }
14
+ const tag = child.tagName;
15
+ if (tag === "table") {
16
+ shape.blockInCell = shape.blockInCell || inCell;
17
+ continue;
18
+ }
19
+ if (DATA_MARKERS.has(tag)) {
20
+ shape.dataMarker = true;
21
+ }
22
+ if (tag === "tr") {
23
+ shape.rows += 1;
24
+ const cells = child.children.filter(
25
+ (cell) => isElement(cell) && (cell.tagName === "td" || cell.tagName === "th")
26
+ ).length;
27
+ shape.columns = Math.max(shape.columns, cells);
28
+ }
29
+ if (inCell && BLOCK_IN_CELL.has(tag)) {
30
+ shape.blockInCell = true;
31
+ }
32
+ visit(child, inCell || tag === "td" || tag === "th");
33
+ }
34
+ };
35
+ visit(table, false);
36
+ return shape;
37
+ };
38
+ var isLayoutTable = (table) => {
39
+ const role = stringProperty(table, "role");
40
+ if (role === "presentation" || role === "none" || stringProperty(table, "datatable") === "0") {
41
+ return true;
42
+ }
43
+ const shape = shapeOf(table);
44
+ if (shape.blockInCell) {
45
+ return true;
46
+ }
47
+ if (shape.dataMarker) {
48
+ return false;
49
+ }
50
+ if (shape.rows <= 1 || shape.columns <= 1) {
51
+ return true;
52
+ }
53
+ if (shape.rows >= 10 || shape.columns > 4) {
54
+ return false;
55
+ }
56
+ return shape.rows * shape.columns <= 10;
57
+ };
58
+ var unwrap = (table) => {
59
+ const cells = [];
60
+ const collect = (node) => {
61
+ for (const child of node.children) {
62
+ if (!isElement(child)) {
63
+ continue;
64
+ }
65
+ if (child.tagName === "td" || child.tagName === "th") {
66
+ cells.push({ type: "element", tagName: "div", properties: {}, children: child.children });
67
+ } else if (child.tagName !== "table") {
68
+ collect(child);
69
+ }
70
+ }
71
+ };
72
+ collect(table);
73
+ return { type: "element", tagName: "div", properties: {}, children: cells };
74
+ };
75
+ var unwrapLayoutTables = (root) => {
76
+ const visit = (parent) => {
77
+ const parentTag = isElement(parent) ? parent.tagName : "";
78
+ for (let index = 0; index < parent.children.length; index++) {
79
+ const child = parent.children[index];
80
+ if (!isElement(child)) {
81
+ continue;
82
+ }
83
+ if (TABLE_PARTS.has(child.tagName) && !TABLE_CONTEXT.has(parentTag)) {
84
+ child.tagName = "div";
85
+ }
86
+ visit(child);
87
+ if (child.tagName === "table" && isLayoutTable(child)) {
88
+ parent.children[index] = unwrap(child);
89
+ }
90
+ }
91
+ };
92
+ if ("children" in root) {
93
+ visit(root);
94
+ }
95
+ };
96
+ export {
97
+ isLayoutTable,
98
+ unwrapLayoutTables
99
+ };
@@ -10,6 +10,7 @@ import {
10
10
  stringProperty,
11
11
  walk
12
12
  } from "../../utils/hast-fast.js";
13
+ import { CodeTabCollector } from "./code-tabs.js";
13
14
  import {
14
15
  CHROME_TAGS,
15
16
  CONTEXTUAL_LANDMARK_TAGS,
@@ -21,25 +22,39 @@ import {
21
22
  RESPONSIVE_DISPLAY_OVERRIDE,
22
23
  UNLIKELY_ROLES
23
24
  } from "./constants.js";
24
- import { isUiChromeText } from "./ui-chrome.js";
25
+ import { MAX_CONSENT_PLACEHOLDER_LENGTH, isConsentPlaceholderText, isUiChromeText } from "./ui-chrome.js";
25
26
  var INVISIBLE_STYLE = /(^|;)\s*(display\s*:\s*none|visibility\s*:\s*hidden)\s*(;|$)/i;
26
27
  var hasHiddenClass = (element) => {
27
28
  const classes = classList(element);
28
29
  if (!classes.some((name) => HIDDEN_CLASS_NAMES.has(name))) {
29
- return classes.some((name) => REGEXPS.hidden.test(name));
30
+ return classes.some((name) => REGEXPS.hidden.test(name) || isScreenReaderOnlyClass(name));
30
31
  }
31
32
  return !classes.some((name) => RESPONSIVE_DISPLAY_OVERRIDE.test(name));
32
33
  };
34
+ var SCREEN_READER_ONLY = /^(sronly|visuallyhidden|screenreader(only|text))(focusable|styles\w*)?$/;
35
+ var isScreenReaderOnlyClass = (name) => {
36
+ const first = name.charCodeAt(0) | 32;
37
+ if (first !== 115 && first !== 118) {
38
+ return false;
39
+ }
40
+ return SCREEN_READER_ONLY.test(name.toLowerCase().replace(/[-_]/g, ""));
41
+ };
33
42
  var isNonContent = (node) => {
34
43
  if (node.type === "comment" || node.type === "doctype") {
35
44
  return true;
36
45
  }
37
46
  return isElement(node) && NON_CONTENT_TAGS.has(node.tagName);
38
47
  };
39
- var isHidden = (element) => {
48
+ var isHidden = (element, parent, siblingCode) => {
49
+ if (!looksHidden(element)) {
50
+ return false;
51
+ }
40
52
  if (containsMath(element)) {
41
53
  return false;
42
54
  }
55
+ return !isHiddenCodePanel(element, parent, siblingCode);
56
+ };
57
+ var looksHidden = (element) => {
43
58
  if (isTruthyAttribute(element, "hidden")) {
44
59
  return true;
45
60
  }
@@ -52,10 +67,57 @@ var isHidden = (element) => {
52
67
  }
53
68
  const width = numericProperty(element, "data-rwidth");
54
69
  const height = numericProperty(element, "data-rheight");
55
- if (width === 0 && height === 0) {
70
+ return width === 0 && height === 0;
71
+ };
72
+ var MAX_CODE_PANEL_PROSE = 200;
73
+ var panelCode = (element) => {
74
+ let code = "";
75
+ let prose = 0;
76
+ const visit = (node, insideCode) => {
77
+ for (const child of node.children) {
78
+ if (child.type === "text") {
79
+ if (insideCode) {
80
+ code += child.value;
81
+ } else {
82
+ prose += child.value.trim().length;
83
+ if (prose > MAX_CODE_PANEL_PROSE) {
84
+ return false;
85
+ }
86
+ }
87
+ } else if (isElement(child) && !visit(child, insideCode || child.tagName === "pre")) {
88
+ return false;
89
+ }
90
+ }
56
91
  return true;
92
+ };
93
+ return visit(element, element.tagName === "pre") && code.trim().length > 0 ? code : void 0;
94
+ };
95
+ var visibleSiblingCode = (parent, cache) => {
96
+ let codes = cache.get(parent);
97
+ if (!codes) {
98
+ codes = /* @__PURE__ */ new Set();
99
+ for (const child of parent.children) {
100
+ if (isElement(child) && !looksHidden(child)) {
101
+ const code = panelCode(child);
102
+ if (code !== void 0) {
103
+ codes.add(code);
104
+ }
105
+ }
106
+ }
107
+ cache.set(parent, codes);
57
108
  }
58
- return false;
109
+ return codes;
110
+ };
111
+ var isHiddenCodePanel = (element, parent, cache) => {
112
+ const code = panelCode(element);
113
+ if (code === void 0) {
114
+ return false;
115
+ }
116
+ const siblings = visibleSiblingCode(parent, cache);
117
+ if (siblings.has(code)) {
118
+ return false;
119
+ }
120
+ return stringProperty(element, "role") === "tabpanel" || siblings.size > 0;
59
121
  };
60
122
  var PLACEHOLDER_IMAGE_SRC = /^data:image\/|^about:blank$|spacer\.gif$|blank\.(gif|png)$/i;
61
123
  var isPlaceholderImage = (element) => {
@@ -73,15 +135,16 @@ var isPlaceholderImage = (element) => {
73
135
  }
74
136
  return false;
75
137
  };
76
- var hoistNoscriptImages = (tree) => {
77
- const visit = (parent) => {
138
+ var hoistNoscriptImages = (tree, onElement) => {
139
+ const visit = (parent, depth) => {
78
140
  for (let index = 0; index < parent.children.length; index++) {
79
141
  const child = parent.children[index];
80
142
  if (!isElement(child)) {
81
143
  continue;
82
144
  }
83
145
  if (child.tagName !== "noscript") {
84
- visit(child);
146
+ onElement?.(child, depth, parent);
147
+ visit(child, depth + 1);
85
148
  continue;
86
149
  }
87
150
  const hasImage = child.children.some(
@@ -101,7 +164,7 @@ var hoistNoscriptImages = (tree) => {
101
164
  }
102
165
  };
103
166
  if ("children" in tree) {
104
- visit(tree);
167
+ visit(tree, 0);
105
168
  }
106
169
  };
107
170
  var containsMath = (element) => {
@@ -116,12 +179,15 @@ var containsMath = (element) => {
116
179
  return false;
117
180
  };
118
181
  var stripNonContent = (tree) => {
119
- hoistNoscriptImages(tree);
120
- return pruneInPlace(tree, (node) => {
182
+ const tabs = new CodeTabCollector((element) => looksHidden(element) || NON_CONTENT_TAGS.has(element.tagName));
183
+ hoistNoscriptImages(tree, (element, depth, parent) => tabs.visit(element, depth, parent));
184
+ tabs.apply();
185
+ const siblingCode = /* @__PURE__ */ new WeakMap();
186
+ return pruneInPlace(tree, (node, parent) => {
121
187
  if (isNonContent(node)) {
122
188
  return false;
123
189
  }
124
- if (isElement(node) && isHidden(node)) {
190
+ if (isElement(node) && isHidden(node, parent, siblingCode)) {
125
191
  return false;
126
192
  }
127
193
  return true;
@@ -148,32 +214,32 @@ var findLandmarkChrome = (tree) => {
148
214
  visit(tree, 0, false);
149
215
  return chrome;
150
216
  };
151
- var isUnlikelyCandidate = (element) => {
217
+ var unlikelyTier = (element) => {
152
218
  if (element.tagName === "article" || element.tagName === "main" || element.tagName === "body") {
153
- return false;
219
+ return void 0;
154
220
  }
155
221
  const role = stringProperty(element, "role");
156
222
  if (role && UNLIKELY_ROLES.has(role)) {
157
- return true;
223
+ return "strong";
158
224
  }
159
225
  if (CHROME_TAGS.has(element.tagName)) {
160
- return true;
226
+ return "strong";
161
227
  }
162
228
  const match = matchString(element);
163
229
  if (REGEXPS.specialUnlikelyCandidates.test(match)) {
164
- return true;
230
+ return "strong";
165
231
  }
166
232
  for (const name of [...classList(element), stringProperty(element, "id") ?? ""]) {
167
233
  if (REGEXPS.stronglyUnlikely.test(name)) {
168
- return true;
234
+ return "strong";
169
235
  }
170
236
  }
171
237
  if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
172
- return true;
238
+ return "weak";
173
239
  }
174
- return false;
240
+ return void 0;
175
241
  };
176
- var findUnlikelyElements = (tree) => {
242
+ var findUnlikelyElements = (tree, spareWeak) => {
177
243
  const landmarks = findLandmarkChrome(tree);
178
244
  const doomed = [];
179
245
  const visit = (node) => {
@@ -184,7 +250,15 @@ var findUnlikelyElements = (tree) => {
184
250
  if (!isElement(child)) {
185
251
  continue;
186
252
  }
187
- if (landmarks.has(child) || isUnlikelyCandidate(child)) {
253
+ if (landmarks.has(child)) {
254
+ doomed.push(child);
255
+ continue;
256
+ }
257
+ if (PREFORMATTED_TAGS.has(child.tagName)) {
258
+ continue;
259
+ }
260
+ const tier = unlikelyTier(child);
261
+ if (tier === "strong" || tier === "weak" && !spareWeak?.(child)) {
188
262
  doomed.push(child);
189
263
  continue;
190
264
  }
@@ -329,11 +403,15 @@ var findCleanupTargets = (tree, collector) => {
329
403
  visit(child);
330
404
  continue;
331
405
  }
406
+ if (LITERAL_TEXT_TAGS.has(child.tagName)) {
407
+ collectButtons(child, doomed);
408
+ continue;
409
+ }
332
410
  if (isLinkOnlyBlock(child, collector)) {
333
411
  condemnLinkBlock(child, siblings, index, collector, doomed);
334
412
  continue;
335
413
  }
336
- if (isMidArticleWidget(child)) {
414
+ if (isMidArticleWidget(child, collector)) {
337
415
  doomed.push(child);
338
416
  continue;
339
417
  }
@@ -374,15 +452,53 @@ var countAnchors = (element) => {
374
452
  }
375
453
  return count;
376
454
  };
377
- var isMidArticleWidget = (element) => {
455
+ var LITERAL_TEXT_TAGS = /* @__PURE__ */ new Set(["code", "kbd", "samp", "var", "pre"]);
456
+ var PREFORMATTED_TAGS = /* @__PURE__ */ new Set(["pre", "code"]);
457
+ var collectButtons = (element, into) => {
458
+ for (const child of element.children) {
459
+ if (!isElement(child)) {
460
+ continue;
461
+ }
462
+ if (child.tagName === "button") {
463
+ into.push(child);
464
+ continue;
465
+ }
466
+ collectButtons(child, into);
467
+ }
468
+ };
469
+ var isMidArticleWidget = (element, collector) => {
378
470
  if (element.tagName === "form" || element.tagName === "fieldset") {
379
471
  return true;
380
472
  }
381
473
  if (element.tagName === "button") {
382
474
  return true;
383
475
  }
384
- return isUiChromeText(elementText(element));
476
+ if (collector.metrics(element).text > MAX_CONSENT_PLACEHOLDER_LENGTH) {
477
+ return false;
478
+ }
479
+ const text = elementText(element);
480
+ return isUiChromeText(text) || isConsentPlaceholderText(text) && !hasBlockDescendant(element);
385
481
  };
482
+ var BLOCK_TAGS = /* @__PURE__ */ new Set([
483
+ "p",
484
+ "div",
485
+ "section",
486
+ "article",
487
+ "figure",
488
+ "figcaption",
489
+ "blockquote",
490
+ "ul",
491
+ "ol",
492
+ "table",
493
+ "pre",
494
+ "h1",
495
+ "h2",
496
+ "h3",
497
+ "h4",
498
+ "h5",
499
+ "h6"
500
+ ]);
501
+ var hasBlockDescendant = (element) => element.children.some((child) => isElement(child) && (BLOCK_TAGS.has(child.tagName) || hasBlockDescendant(child)));
386
502
  var cleanContent = (tree, collector) => {
387
503
  const doomed = new Set(findCleanupTargets(tree, collector));
388
504
  if (doomed.size > 0) {
@@ -404,6 +520,7 @@ var elementText = (node) => {
404
520
  return text;
405
521
  };
406
522
  export {
523
+ REFERENCE_HEADING,
407
524
  cleanContent,
408
525
  dropEmptyElements,
409
526
  findCleanupTargets,
@@ -411,5 +528,6 @@ export {
411
528
  findUnlikelyElements,
412
529
  hoistNoscriptImages,
413
530
  stripNonContent,
414
- stripUnlikely
531
+ stripUnlikely,
532
+ unlikelyTier
415
533
  };
@@ -0,0 +1,53 @@
1
+ // src/extractors/lib/takumi-signal.ts
2
+ import { isElement, walk } from "../../utils/hast-fast.js";
3
+ import { takumiExtractor } from "../presets/takumi.js";
4
+ var MARK = "dataWebforaiIndex";
5
+ var cloneElements = (root) => {
6
+ const copyOf = (node) => "children" in node ? {
7
+ ...node,
8
+ ...isElement(node) ? { properties: { ...node.properties } } : {},
9
+ children: [...node.children]
10
+ } : node;
11
+ const copy = copyOf(root);
12
+ const stack = [copy];
13
+ while (stack.length > 0) {
14
+ const parent = stack.pop();
15
+ if (!("children" in parent)) {
16
+ continue;
17
+ }
18
+ const children = parent.children;
19
+ for (let index = 0; index < children.length; index++) {
20
+ if ("children" in children[index]) {
21
+ children[index] = copyOf(children[index]);
22
+ stack.push(children[index]);
23
+ }
24
+ }
25
+ }
26
+ return copy;
27
+ };
28
+ var takumiSelection = (root, lang, url) => {
29
+ const elements = [];
30
+ walk(root, (node) => {
31
+ if (isElement(node)) {
32
+ node.properties[MARK] = elements.length;
33
+ elements.push(node);
34
+ }
35
+ });
36
+ const copy = cloneElements(root);
37
+ for (const element of elements) {
38
+ delete element.properties[MARK];
39
+ }
40
+ const kept = /* @__PURE__ */ new Set();
41
+ walk(takumiExtractor({ hast: copy, lang, url, owned: true }), (node) => {
42
+ if (isElement(node)) {
43
+ const index = node.properties[MARK];
44
+ if (typeof index === "number" && elements[index]) {
45
+ kept.add(elements[index]);
46
+ }
47
+ }
48
+ });
49
+ return kept;
50
+ };
51
+ export {
52
+ takumiSelection
53
+ };
@@ -91,6 +91,11 @@ var isUiChromeText = (text) => {
91
91
  }
92
92
  return UI_PHRASES.has(normalized);
93
93
  };
94
+ var CONSENT_PLACEHOLDER = /^(this content (isn['’]?t|is not) (visible|available) (due to|because of) your (cookie|privacy) (preferences|settings)|we need your (consent|permission) to load|to (view|see|load) this (content|embed|video|post)[, ]+(please )?(enable|accept|allow|update)|(please )?(accept|allow|enable) (all |marketing |functional )?cookies to (view|see|load|watch)|このコンテンツを表示するには.{0,30}(cookie|クッキー|同意|許可))/i;
95
+ var MAX_CONSENT_PLACEHOLDER_LENGTH = 400;
96
+ var isConsentPlaceholderText = (text) => text.length <= MAX_CONSENT_PLACEHOLDER_LENGTH && CONSENT_PLACEHOLDER.test(text.trimStart());
94
97
  export {
98
+ MAX_CONSENT_PLACEHOLDER_LENGTH,
99
+ isConsentPlaceholderText,
95
100
  isUiChromeText
96
101
  };
@@ -1,8 +1,8 @@
1
1
  // src/extractors/presets/auto.ts
2
2
  import { runAdapters } from "../../adapters/index.js";
3
- import { takumiExtractor } from "./takumi.js";
3
+ import { kiwameExtractor } from "./kiwame.js";
4
4
  var createAutoExtractor = (options = {}) => {
5
- const { adapters } = options;
5
+ const { adapters, fallback = kiwameExtractor } = options;
6
6
  return (params) => {
7
7
  if (adapters !== false) {
8
8
  const extracted = runAdapters({ hast: params.hast, url: params.url, lang: params.lang }, adapters ?? void 0);
@@ -10,7 +10,7 @@ var createAutoExtractor = (options = {}) => {
10
10
  return extracted;
11
11
  }
12
12
  }
13
- return takumiExtractor(params);
13
+ return fallback(params);
14
14
  };
15
15
  };
16
16
  var autoExtractor = createAutoExtractor();