webforai 3.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/README.md +47 -5
  2. package/dist/bin.js +3109 -490
  3. package/dist/cjs/detect-client-shell.js +78 -0
  4. package/dist/cjs/extract-mdast.js +58 -0
  5. package/dist/cjs/extract-meta-refresh.js +71 -0
  6. package/dist/cjs/extractors/index.js +5 -0
  7. package/dist/cjs/extractors/lib/block-features.js +465 -0
  8. package/dist/cjs/extractors/lib/block-model.generated.js +45 -0
  9. package/dist/cjs/extractors/lib/block-model.js +72 -0
  10. package/dist/cjs/extractors/lib/blocks.js +212 -0
  11. package/dist/cjs/extractors/lib/code-tabs.js +191 -0
  12. package/dist/cjs/extractors/lib/constants.js +1 -1
  13. package/dist/cjs/extractors/lib/layout-tables.js +123 -0
  14. package/dist/cjs/extractors/lib/sanitize.js +146 -26
  15. package/dist/cjs/extractors/lib/takumi-signal.js +76 -0
  16. package/dist/cjs/extractors/lib/ui-chrome.js +7 -0
  17. package/dist/cjs/extractors/presets/auto.js +3 -3
  18. package/dist/cjs/extractors/presets/kiwame.js +231 -0
  19. package/dist/cjs/extractors/presets/takumi.js +34 -3
  20. package/dist/cjs/html-to-markdown.js +58 -4
  21. package/dist/cjs/html-to-mdast.js +4 -3
  22. package/dist/cjs/index.js +16 -0
  23. package/dist/cjs/link-replacer.js +19 -13
  24. package/dist/cjs/loaders/cf-puppeteer.js +14 -7
  25. package/dist/cjs/loaders/fetch.js +14 -2
  26. package/dist/cjs/loaders/playwright.js +32 -30
  27. package/dist/cjs/loaders/puppeteer.js +14 -7
  28. package/dist/cjs/md-splitter.js +24 -11
  29. package/dist/cjs/mdast-handlers/custom-code-handler.js +32 -9
  30. package/dist/cjs/mdast-handlers/custom-div-handler.js +14 -35
  31. package/dist/cjs/mdast-handlers/custom-img-handler.js +10 -1
  32. package/dist/cjs/mdast-handlers/custom-table-handler.js +18 -1
  33. package/dist/cjs/mdast-handlers/definition-list-handler.js +17 -6
  34. package/dist/cjs/mdast-handlers/table-cell-flatten.js +195 -0
  35. package/dist/cjs/mdast-handlers/twoslash.js +71 -0
  36. package/dist/cjs/mdast-to-markdown.js +2 -5
  37. package/dist/cjs/metadata/index.js +4 -3
  38. package/dist/cjs/normalize/index.js +11 -2
  39. package/dist/cjs/platform/client.js +73 -16
  40. package/dist/cjs/platform/index.js +4 -0
  41. package/dist/cjs/platform/types.js +7 -1
  42. package/dist/cjs/utils/hast-fast.js +1 -1
  43. package/dist/cjs/utils/parse-html.js +113 -0
  44. package/dist/cjs/utils/pre-text.js +104 -0
  45. package/dist/detect-client-shell.js +53 -0
  46. package/dist/extract-mdast.js +58 -0
  47. package/dist/extract-meta-refresh.js +47 -0
  48. package/dist/extractors/index.js +3 -0
  49. package/dist/extractors/lib/block-features.js +437 -0
  50. package/dist/extractors/lib/block-model.generated.js +21 -0
  51. package/dist/extractors/lib/block-model.js +48 -0
  52. package/dist/extractors/lib/blocks.js +185 -0
  53. package/dist/extractors/lib/code-tabs.js +168 -0
  54. package/dist/extractors/lib/constants.js +1 -1
  55. package/dist/extractors/lib/layout-tables.js +99 -0
  56. package/dist/extractors/lib/sanitize.js +144 -26
  57. package/dist/extractors/lib/takumi-signal.js +53 -0
  58. package/dist/extractors/lib/ui-chrome.js +5 -0
  59. package/dist/extractors/presets/auto.js +3 -3
  60. package/dist/extractors/presets/kiwame.js +206 -0
  61. package/dist/extractors/presets/takumi.js +34 -3
  62. package/dist/html-to-markdown.js +57 -4
  63. package/dist/html-to-mdast.js +4 -3
  64. package/dist/index.js +18 -0
  65. package/dist/link-replacer.js +19 -13
  66. package/dist/loaders/cf-puppeteer.js +15 -8
  67. package/dist/loaders/fetch.js +14 -2
  68. package/dist/loaders/playwright.js +32 -30
  69. package/dist/loaders/puppeteer.js +15 -8
  70. package/dist/md-splitter.js +24 -11
  71. package/dist/mdast-handlers/custom-code-handler.js +30 -9
  72. package/dist/mdast-handlers/custom-div-handler.js +14 -35
  73. package/dist/mdast-handlers/custom-img-handler.js +10 -1
  74. package/dist/mdast-handlers/custom-table-handler.js +19 -2
  75. package/dist/mdast-handlers/definition-list-handler.js +17 -6
  76. package/dist/mdast-handlers/table-cell-flatten.js +170 -0
  77. package/dist/mdast-handlers/twoslash.js +48 -0
  78. package/dist/mdast-to-markdown.js +2 -5
  79. package/dist/metadata/index.js +4 -3
  80. package/dist/normalize/index.js +12 -2
  81. package/dist/platform/client.js +73 -16
  82. package/dist/platform/index.js +4 -0
  83. package/dist/platform/types.js +5 -1
  84. package/dist/types/cli/bin.js.map +1 -1
  85. package/dist/types/cli/commands/jobs/files.d.ts +28 -0
  86. package/dist/types/cli/commands/jobs/files.js.map +1 -0
  87. package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
  88. package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
  89. package/dist/types/cli/commands/jobs/options.d.ts +58 -0
  90. package/dist/types/cli/commands/jobs/options.js.map +1 -0
  91. package/dist/types/cli/commands/jobs/run.d.ts +44 -0
  92. package/dist/types/cli/commands/jobs/run.js.map +1 -0
  93. package/dist/types/cli/commands/skill/content.js.map +1 -1
  94. package/dist/types/cli/commands/webforai/interactive.d.ts +1 -1
  95. package/dist/types/cli/commands/webforai/interactive.js.map +1 -1
  96. package/dist/types/cli/commands/webforai/options.d.ts +9 -2
  97. package/dist/types/cli/commands/webforai/options.js.map +1 -1
  98. package/dist/types/cli/commands/webforai/run.js.map +1 -1
  99. package/dist/types/cli/constants.d.ts +3 -1
  100. package/dist/types/cli/constants.js.map +1 -1
  101. package/dist/types/cli/utils.d.ts +8 -1
  102. package/dist/types/cli/utils.js.map +1 -1
  103. package/dist/types/detect-client-shell.d.ts +24 -0
  104. package/dist/types/detect-client-shell.js.map +1 -0
  105. package/dist/types/extract-mdast.js.map +1 -1
  106. package/dist/types/extract-meta-refresh.d.ts +29 -0
  107. package/dist/types/extract-meta-refresh.js.map +1 -0
  108. package/dist/types/extractors/index.d.ts +1 -0
  109. package/dist/types/extractors/index.js.map +1 -1
  110. package/dist/types/extractors/lib/block-features.d.ts +43 -0
  111. package/dist/types/extractors/lib/block-features.js.map +1 -0
  112. package/dist/types/extractors/lib/block-model.d.ts +28 -0
  113. package/dist/types/extractors/lib/block-model.generated.d.ts +5 -0
  114. package/dist/types/extractors/lib/block-model.generated.js.map +1 -0
  115. package/dist/types/extractors/lib/block-model.js.map +1 -0
  116. package/dist/types/extractors/lib/blocks.d.ts +53 -0
  117. package/dist/types/extractors/lib/blocks.js.map +1 -0
  118. package/dist/types/extractors/lib/code-tabs.d.ts +30 -0
  119. package/dist/types/extractors/lib/code-tabs.js.map +1 -0
  120. package/dist/types/extractors/lib/constants.js.map +1 -1
  121. package/dist/types/extractors/lib/layout-tables.d.ts +17 -0
  122. package/dist/types/extractors/lib/layout-tables.js.map +1 -0
  123. package/dist/types/extractors/lib/sanitize.d.ts +33 -3
  124. package/dist/types/extractors/lib/sanitize.js.map +1 -1
  125. package/dist/types/extractors/lib/takumi-signal.d.ts +12 -0
  126. package/dist/types/extractors/lib/takumi-signal.js.map +1 -0
  127. package/dist/types/extractors/lib/ui-chrome.d.ts +8 -0
  128. package/dist/types/extractors/lib/ui-chrome.js.map +1 -1
  129. package/dist/types/extractors/presets/auto.d.ts +6 -1
  130. package/dist/types/extractors/presets/auto.js.map +1 -1
  131. package/dist/types/extractors/presets/kiwame.d.ts +31 -0
  132. package/dist/types/extractors/presets/kiwame.js.map +1 -0
  133. package/dist/types/extractors/presets/takumi.js.map +1 -1
  134. package/dist/types/html-to-markdown.d.ts +9 -0
  135. package/dist/types/html-to-markdown.js.map +1 -1
  136. package/dist/types/html-to-mdast.js.map +1 -1
  137. package/dist/types/index.d.ts +3 -1
  138. package/dist/types/index.js.map +1 -1
  139. package/dist/types/link-replacer.d.ts +3 -1
  140. package/dist/types/link-replacer.js.map +1 -1
  141. package/dist/types/loaders/cf-puppeteer.js.map +1 -1
  142. package/dist/types/loaders/fetch.d.ts +4 -0
  143. package/dist/types/loaders/fetch.js.map +1 -1
  144. package/dist/types/loaders/playwright.js.map +1 -1
  145. package/dist/types/loaders/puppeteer.js.map +1 -1
  146. package/dist/types/md-splitter.d.ts +1 -1
  147. package/dist/types/md-splitter.js.map +1 -1
  148. package/dist/types/mdast-handlers/custom-code-handler.d.ts +9 -0
  149. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
  150. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  151. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  152. package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
  153. package/dist/types/mdast-handlers/definition-list-handler.d.ts +0 -11
  154. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -1
  155. package/dist/types/mdast-handlers/table-cell-flatten.d.ts +18 -0
  156. package/dist/types/mdast-handlers/table-cell-flatten.js.map +1 -0
  157. package/dist/types/mdast-handlers/twoslash.d.ts +14 -0
  158. package/dist/types/mdast-handlers/twoslash.js.map +1 -0
  159. package/dist/types/mdast-to-markdown.js.map +1 -1
  160. package/dist/types/metadata/index.js.map +1 -1
  161. package/dist/types/normalize/index.js.map +1 -1
  162. package/dist/types/platform/client.d.ts +2 -1
  163. package/dist/types/platform/client.js.map +1 -1
  164. package/dist/types/platform/error.d.ts +5 -3
  165. package/dist/types/platform/error.js.map +1 -1
  166. package/dist/types/platform/fetch.d.ts +2 -0
  167. package/dist/types/platform/fetch.js.map +1 -1
  168. package/dist/types/platform/index.d.ts +1 -1
  169. package/dist/types/platform/index.js.map +1 -1
  170. package/dist/types/platform/types.d.ts +35 -6
  171. package/dist/types/platform/types.js.map +1 -1
  172. package/dist/types/utils/hast-fast.d.ts +1 -1
  173. package/dist/types/utils/hast-fast.js.map +1 -1
  174. package/dist/types/utils/parse-html.d.ts +33 -0
  175. package/dist/types/utils/parse-html.js.map +1 -0
  176. package/dist/types/utils/pre-text.d.ts +12 -0
  177. package/dist/types/utils/pre-text.js.map +1 -0
  178. package/dist/utils/hast-fast.js +1 -1
  179. package/dist/utils/parse-html.js +87 -0
  180. package/dist/utils/pre-text.js +81 -0
  181. package/package.json +18 -5
@@ -22,10 +22,22 @@ __export(fetch_exports, {
22
22
  loadHtml: () => loadHtml
23
23
  });
24
24
  module.exports = __toCommonJS(fetch_exports);
25
+ var import_extract_meta_refresh = require("../extract-meta-refresh");
25
26
  const USER_AGENT = "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/125.0.0.0 safari/537.36";
27
+ const MAX_META_REFRESH_HOPS = 3;
26
28
  const loadHtml = async (url, userAgent = USER_AGENT) => {
27
- const response = await fetch(url, { headers: { "User-Agent": userAgent } });
28
- return response.text();
29
+ let current = url;
30
+ let html = "";
31
+ for (let hop = 0; hop <= MAX_META_REFRESH_HOPS; hop++) {
32
+ const response = await fetch(current, { headers: { "User-Agent": userAgent } });
33
+ html = await response.text();
34
+ const refresh = (0, import_extract_meta_refresh.extractMetaRefresh)(html, response.url || current);
35
+ if (!refresh) {
36
+ break;
37
+ }
38
+ current = refresh.url;
39
+ }
40
+ return html;
29
41
  };
30
42
  // Annotate the CommonJS export names for ESM import in node:
31
43
  0 && (module.exports = {
@@ -35,40 +35,42 @@ const SUPER_BYPASS_DEVICE = {
35
35
  const loadHtml = async (url, options) => {
36
36
  const { browser, waitUntil, timeout, superBypassMode } = options ?? {};
37
37
  const _browser = browser ?? await import_playwright_core.chromium.launch({ headless: true });
38
- const context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
39
- if (superBypassMode) {
40
- await context.addInitScript(() => {
41
- Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
42
- Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
43
- Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
44
- });
45
- }
46
- const page = await context.newPage();
47
- if (superBypassMode) {
48
- await page.route("**/*.js", (route) => {
49
- if (route.request().url().includes("captcha-delivery")) {
50
- return route.abort();
38
+ let context;
39
+ try {
40
+ context = await _browser.newContext(superBypassMode ? SUPER_BYPASS_DEVICE : import_playwright_core.devices["Desktop Chrome"]);
41
+ if (superBypassMode) {
42
+ await context.addInitScript(() => {
43
+ Object.defineProperty(navigator, "webdriver", { get: () => void 0 });
44
+ Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
45
+ Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3, 4, 5] });
46
+ });
47
+ }
48
+ const page = await context.newPage();
49
+ if (superBypassMode) {
50
+ await page.route("**/*.js", (route) => {
51
+ if (route.request().url().includes("captcha-delivery")) {
52
+ return route.abort();
53
+ }
54
+ return route.continue();
55
+ });
56
+ }
57
+ await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
58
+ await page.evaluate(() => {
59
+ const elements = document.querySelectorAll("*");
60
+ for (const element of elements) {
61
+ const rect = element.getBoundingClientRect();
62
+ element.setAttribute("data-rwidth", rect.width.toString());
63
+ element.setAttribute("data-rheight", rect.height.toString());
51
64
  }
52
- return route.continue();
53
65
  });
54
- }
55
- await page.goto(url, { waitUntil: waitUntil ?? "load", timeout });
56
- await page.evaluate(() => {
57
- const elements = document.querySelectorAll("*");
58
- for (const element of elements) {
59
- const rect = element.getBoundingClientRect();
60
- element.setAttribute("data-rwidth", rect.width.toString());
61
- element.setAttribute("data-rheight", rect.height.toString());
66
+ return await page.content();
67
+ } finally {
68
+ if (browser) {
69
+ await context?.close();
70
+ } else {
71
+ await _browser.close();
62
72
  }
63
- });
64
- const html = await page.content();
65
- await page.close();
66
- if (browser) {
67
- await context.close();
68
- } else {
69
- await _browser.close();
70
73
  }
71
- return html;
72
74
  };
73
75
  // Annotate the CommonJS export names for ESM import in node:
74
76
  0 && (module.exports = {
@@ -32,7 +32,6 @@ __export(puppeteer_exports, {
32
32
  });
33
33
  module.exports = __toCommonJS(puppeteer_exports);
34
34
  var import_puppeteer = __toESM(require("puppeteer"), 1);
35
- const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
36
35
  const loadHtml = async (url, ctx) => {
37
36
  const browser = await import_puppeteer.default.launch(
38
37
  ctx || {
@@ -40,12 +39,20 @@ const loadHtml = async (url, ctx) => {
40
39
  args: ["--no-sandbox", "--disable-setuid-sandbox"]
41
40
  }
42
41
  );
43
- const page = await browser.newPage();
44
- await page.goto(url);
45
- const html = await page.content();
46
- await Promise.race([page.waitForNetworkIdle(), sleep(1e4)]);
47
- await browser.close();
48
- return html;
42
+ try {
43
+ const page = await browser.newPage();
44
+ await page.goto(url);
45
+ try {
46
+ await page.waitForNetworkIdle({ timeout: 1e4 });
47
+ } catch (error) {
48
+ if (!(error instanceof import_puppeteer.TimeoutError)) {
49
+ throw error;
50
+ }
51
+ }
52
+ return await page.content();
53
+ } finally {
54
+ await browser.close();
55
+ }
49
56
  };
50
57
  // Annotate the CommonJS export names for ESM import in node:
51
58
  0 && (module.exports = {
@@ -25,15 +25,13 @@ var import_mdast_to_markdown = require("./mdast-to-markdown");
25
25
  var import_common = require("./utils/common");
26
26
  var import_mdast_utils = require("./utils/mdast-utils");
27
27
  const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
28
- const getSplitterGenerator = function* () {
29
- for (const splitter of PRIORITY_SPLITTERS) {
30
- yield splitter;
31
- }
32
- };
33
- const _mdastSplitter = async (contents, checker, splitterGenerator) => {
34
- const splitter = splitterGenerator.next().value;
28
+ const _mdastSplitter = async (contents, checker, priority, signal) => {
29
+ signal?.throwIfAborted();
30
+ const splitter = PRIORITY_SPLITTERS[priority];
35
31
  const markdown = (0, import_mdast_to_markdown.mdastToMarkdown)((0, import_mdast_utils.warpRoot)(contents));
36
- if (await checker(markdown) || contents.length === 1) {
32
+ const accepted = await checker(markdown);
33
+ signal?.throwIfAborted();
34
+ if (accepted || contents.length === 1) {
37
35
  return [contents];
38
36
  }
39
37
  const chunked = splitter ? contents.reduce((acc, content) => {
@@ -44,11 +42,26 @@ const _mdastSplitter = async (contents, checker, splitterGenerator) => {
44
42
  acc[acc.length - 1].push(content);
45
43
  return acc;
46
44
  }, []) : (0, import_common.chunk)(contents, Math.ceil(contents.length / 2));
47
- const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
45
+ const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, priority + 1, signal));
48
46
  return Promise.all(splitting).then((chunks) => chunks.flat());
49
47
  };
50
- const mdastSplitter = (mdast, checker, _options) => {
51
- return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, getSplitterGenerator());
48
+ const mdastSplitter = async (mdast, checker, options) => {
49
+ const signal = options?.signal;
50
+ if (!signal) {
51
+ return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, 0);
52
+ }
53
+ signal.throwIfAborted();
54
+ let onAbort = () => {
55
+ };
56
+ const aborted = new Promise((_, reject) => {
57
+ onAbort = () => reject(signal.reason);
58
+ signal.addEventListener("abort", onAbort, { once: true });
59
+ });
60
+ try {
61
+ return await Promise.race([_mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, 0, signal), aborted]);
62
+ } finally {
63
+ signal.removeEventListener("abort", onAbort);
64
+ }
52
65
  };
53
66
  // Annotate the CommonJS export names for ESM import in node:
54
67
  0 && (module.exports = {
@@ -18,26 +18,49 @@ var __copyProps = (to, from, except, desc) => {
18
18
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
19
  var custom_code_handler_exports = {};
20
20
  __export(custom_code_handler_exports, {
21
+ codeLanguage: () => codeLanguage,
22
+ codeMeta: () => codeMeta,
21
23
  customCodeHandler: () => customCodeHandler
22
24
  });
23
25
  module.exports = __toCommonJS(custom_code_handler_exports);
24
- var import_hast_util_to_text = require("hast-util-to-text");
25
26
  var import_trim_trailing_lines = require("trim-trailing-lines");
26
27
  var import_detect_code_lang = require("../utils/detect-code-lang");
27
- const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
28
+ var import_hast_fast = require("../utils/hast-fast");
29
+ var import_pre_text = require("../utils/pre-text");
30
+ var import_twoslash = require("./twoslash");
31
+ const LANGUAGE_MATCH_REGEX = [/^language-(\S+)$/, /^highlight-source-(\S+)$/, /^CodeBlock--language-(\S+)$/];
32
+ const LANGUAGE_ATTRIBUTES = ["dataLanguage", "dataLang"];
33
+ const attributeLanguage = (element) => {
34
+ for (const name of LANGUAGE_ATTRIBUTES) {
35
+ const value = element.properties?.[name];
36
+ if (typeof value === "string" && /^[\w+#.-]+$/.test(value.trim()) && value.trim() !== "plaintext") {
37
+ return value.trim();
38
+ }
39
+ }
40
+ return void 0;
41
+ };
42
+ const codeLanguage = (...elements) => elements.flatMap(import_hast_fast.classList).map((className) => LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find(Boolean)?.[1]).find(Boolean) ?? elements.map(attributeLanguage).find(Boolean);
43
+ const codeMeta = (pre) => {
44
+ const title = pre.properties?.dataTitle;
45
+ if (typeof title !== "string") {
46
+ return null;
47
+ }
48
+ const clean = title.replace(/["\r\n]+/g, " ").trim();
49
+ return clean ? `title="${clean}"` : null;
50
+ };
28
51
  const customCodeHandler = (state, node) => {
29
- const classNames = node.properties?.className || [];
30
- const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(node)).trim();
31
- const classLang = classNames.map((className) => {
32
- const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
33
- return match?.[1];
34
- }).find((className) => className);
52
+ const twoslashLang = (0, import_twoslash.cleanTwoslash)(node);
53
+ const code = node.children.find((child) => (0, import_hast_fast.isElement)(child) && child.tagName === "code");
54
+ const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_pre_text.preText)(node));
55
+ const classLang = twoslashLang || codeLanguage(...(0, import_hast_fast.isElement)(code) ? [code, node] : [node]);
35
56
  const lang = classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
36
- const result = { type: "code", lang, meta: null, value: codeValue };
57
+ const result = { type: "code", lang, meta: codeMeta(node), value: codeValue };
37
58
  state.patch(node, result);
38
59
  return result;
39
60
  };
40
61
  // Annotate the CommonJS export names for ESM import in node:
41
62
  0 && (module.exports = {
63
+ codeLanguage,
64
+ codeMeta,
42
65
  customCodeHandler
43
66
  });
@@ -24,49 +24,28 @@ module.exports = __toCommonJS(custom_div_handler_exports);
24
24
  var import_hast_util_select = require("hast-util-select");
25
25
  var import_hast_util_to_mdast = require("hast-util-to-mdast");
26
26
  var import_hast_util_to_string = require("hast-util-to-string");
27
- var import_hast_util_to_text = require("hast-util-to-text");
28
27
  var import_trim_trailing_lines = require("trim-trailing-lines");
29
28
  var import_detect_code_lang = require("../utils/detect-code-lang");
29
+ var import_hast_fast = require("../utils/hast-fast");
30
+ var import_pre_text = require("../utils/pre-text");
31
+ var import_custom_code_handler = require("./custom-code-handler");
30
32
  const CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
31
33
  const CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
32
- const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
33
- const findRecursive = (array, condition, maxDepth = 3) => {
34
- if (maxDepth <= 0) {
35
- return null;
36
- }
37
- for (const value of array) {
38
- const result = condition(value);
39
- if (Array.isArray(result)) {
40
- return findRecursive(result, condition, maxDepth - 1);
41
- }
42
- if (result) {
43
- return value;
44
- }
45
- }
46
- return null;
47
- };
48
34
  const customDivHandler = (state, node) => {
49
- const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
50
- const codeBlock = findRecursive(node.children, (child) => {
51
- if (child.type !== "element") {
52
- return false;
53
- }
54
- if (child.tagName === "pre") {
55
- return true;
56
- }
57
- return child.children.filter((child2) => child2.type === "element");
58
- });
59
- if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
60
- const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
61
- const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(codeBlock)).trim();
35
+ const classNames = (0, import_hast_fast.classList)(node);
36
+ if (!classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
37
+ return import_hast_util_to_mdast.defaultHandlers.div(state, node);
38
+ }
39
+ const codeBlocks = (0, import_hast_fast.collectElements)(node, (element) => element.tagName === "pre");
40
+ const codeBlock = codeBlocks.length === 1 ? codeBlocks[0] : void 0;
41
+ if (codeBlock) {
42
+ const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_pre_text.preText)(codeBlock));
62
43
  const filenameElement = (0, import_hast_util_select.select)(CODE_FILENAME_SELECTORS, node);
63
44
  const fileLang = filenameElement ? (0, import_hast_util_to_string.toString)(filenameElement).match(/\.(\w+)$/)?.[1] : null;
64
- const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
65
- const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
66
- return match?.[1];
67
- }).find((className) => className);
45
+ const code = codeBlock.children.find((child) => (0, import_hast_fast.isElement)(child) && child.tagName === "code");
46
+ const classLang = (0, import_custom_code_handler.codeLanguage)(...(0, import_hast_fast.isElement)(code) ? [code, codeBlock, node] : [codeBlock, node]);
68
47
  const lang = fileLang || classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
69
- const result = { type: "code", lang, meta: null, value: codeValue };
48
+ const result = { type: "code", lang, meta: (0, import_custom_code_handler.codeMeta)(codeBlock), value: codeValue };
70
49
  state.patch(node, result);
71
50
  return result;
72
51
  }
@@ -34,11 +34,20 @@ const hasNoUsableSource = (node) => {
34
34
  }
35
35
  return false;
36
36
  };
37
+ const PLACEHOLDER_FILE = /^(?:[\w-]*[-_.])?placeholder[\w-]*\.(?:gif|png|svg|webp|jpe?g)$|^(?:spacer|blank|transparent|pixel)\.(?:gif|png)$/i;
38
+ const isUndescribedPlaceholder = (node) => {
39
+ if ((0, import_hast_fast.stringProperty)(node, "alt")?.trim() || (0, import_hast_fast.stringProperty)(node, "title")?.trim()) {
40
+ return false;
41
+ }
42
+ const src = (0, import_hast_fast.stringProperty)(node, "src") ?? "";
43
+ const file = src.split(/[?#]/)[0].split("/").pop() ?? "";
44
+ return PLACEHOLDER_FILE.test(file);
45
+ };
37
46
  const customImgHandler = (options) => (state, node) => {
38
47
  if (options?.hideImage) {
39
48
  return void 0;
40
49
  }
41
- if (hasNoUsableSource(node)) {
50
+ if (hasNoUsableSource(node) || isUndescribedPlaceholder(node)) {
42
51
  return void 0;
43
52
  }
44
53
  const image = import_hast_util_to_mdast.defaultHandlers.image(state, node);
@@ -24,6 +24,7 @@ module.exports = __toCommonJS(custom_table_handler_exports);
24
24
  var import_hast_util_to_mdast = require("hast-util-to-mdast");
25
25
  var import_hast_util_to_text = require("hast-util-to-text");
26
26
  var import_hast_fast = require("../utils/hast-fast");
27
+ var import_table_cell_flatten = require("./table-cell-flatten");
27
28
  const MAX_SPAN = 1e3;
28
29
  const clampSpans = (node) => {
29
30
  (0, import_hast_fast.walk)(node, (candidate) => {
@@ -38,6 +39,14 @@ const clampSpans = (node) => {
38
39
  }
39
40
  });
40
41
  };
42
+ const startsWithHeaderRow = (table) => {
43
+ const row = (0, import_hast_fast.findElement)(table, (element) => element.tagName === "tr");
44
+ if (!row) {
45
+ return false;
46
+ }
47
+ const cells = row.children.filter(import_hast_fast.isElement);
48
+ return cells.length > 0 && cells.every((cell) => cell.tagName === "th");
49
+ };
41
50
  const customTableHandler = (options) => (state, node) => {
42
51
  if (options?.asText) {
43
52
  const paragraph = { type: "paragraph", children: [{ type: "text", value: (0, import_hast_util_to_text.toText)(node) }] };
@@ -45,7 +54,15 @@ const customTableHandler = (options) => (state, node) => {
45
54
  return paragraph;
46
55
  }
47
56
  clampSpans(node);
48
- return import_hast_util_to_mdast.defaultHandlers.table(state, node);
57
+ const result = import_hast_util_to_mdast.defaultHandlers.table(state, node);
58
+ if (!result || Array.isArray(result) || result.type !== "table") {
59
+ return result;
60
+ }
61
+ if ((0, import_table_cell_flatten.tableHasCodeSamples)(result)) {
62
+ return (0, import_table_cell_flatten.unfoldTable)(result, startsWithHeaderRow(node));
63
+ }
64
+ (0, import_table_cell_flatten.flattenTableCells)(result);
65
+ return result;
49
66
  };
50
67
  // Annotate the CommonJS export names for ESM import in node:
51
68
  0 && (module.exports = {
@@ -50,6 +50,20 @@ const hoistBoundaryWhitespace = (nodes) => {
50
50
  }
51
51
  return result;
52
52
  };
53
+ const termsAndDefinitions = (parent) => {
54
+ const found = [];
55
+ for (const child of parent.children) {
56
+ if (child.type !== "element") {
57
+ continue;
58
+ }
59
+ if (child.tagName === "dt" || child.tagName === "dd") {
60
+ found.push(child);
61
+ } else if (child.tagName === "div") {
62
+ found.push(...termsAndDefinitions(child));
63
+ }
64
+ }
65
+ return found;
66
+ };
53
67
  const definitionListHandler = (state, node) => {
54
68
  const items = [];
55
69
  let current;
@@ -64,11 +78,7 @@ const definitionListHandler = (state, node) => {
64
78
  items.push({ type: "listItem", spread: false, children: [heading, ...current.definitions] });
65
79
  current = void 0;
66
80
  };
67
- for (const child of node.children) {
68
- if (child.type !== "element") {
69
- continue;
70
- }
71
- const element = child;
81
+ for (const element of termsAndDefinitions(node)) {
72
82
  if (element.tagName === "dt") {
73
83
  if (current && current.definitions.length > 0) {
74
84
  flush();
@@ -87,7 +97,8 @@ const definitionListHandler = (state, node) => {
87
97
  }
88
98
  flush();
89
99
  if (items.length === 0) {
90
- return void 0;
100
+ const fallback = state.toFlow(state.all(node));
101
+ return fallback.length > 0 ? fallback : void 0;
91
102
  }
92
103
  const result = { type: "list", ordered: false, spread: false, children: items };
93
104
  state.patch(node, result);
@@ -0,0 +1,195 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var table_cell_flatten_exports = {};
20
+ __export(table_cell_flatten_exports, {
21
+ flattenTableCells: () => flattenTableCells,
22
+ tableHasCodeSamples: () => tableHasCodeSamples,
23
+ unfoldTable: () => unfoldTable
24
+ });
25
+ module.exports = __toCommonJS(table_cell_flatten_exports);
26
+ const PHRASING_TYPES = /* @__PURE__ */ new Set([
27
+ "text",
28
+ "emphasis",
29
+ "strong",
30
+ "delete",
31
+ "inlineCode",
32
+ "link",
33
+ "linkReference",
34
+ "image",
35
+ "imageReference",
36
+ "html",
37
+ "footnoteReference",
38
+ "inlineMath"
39
+ ]);
40
+ const PHRASING_PARENTS = /* @__PURE__ */ new Set(["emphasis", "strong", "delete", "link", "linkReference"]);
41
+ const lineBreak = () => ({ type: "html", value: "<br>" });
42
+ const needsFlattening = (nodes) => nodes.some((node) => {
43
+ if (node.type === "break") {
44
+ return true;
45
+ }
46
+ if (!PHRASING_TYPES.has(node.type)) {
47
+ return true;
48
+ }
49
+ if ((node.type === "text" || node.type === "inlineCode") && node.value.includes("\n")) {
50
+ return true;
51
+ }
52
+ return PHRASING_PARENTS.has(node.type) && "children" in node && needsFlattening(node.children);
53
+ });
54
+ class CellBuilder {
55
+ constructor() {
56
+ this.out = [];
57
+ this.#pendingBlock = false;
58
+ }
59
+ #pendingBlock;
60
+ /** Starts a new visual line when the previous block left content on the current one. */
61
+ block() {
62
+ const last = this.out.at(-1);
63
+ if (this.#pendingBlock && last && !(last.type === "html" && last.value === "<br>")) {
64
+ this.out.push(lineBreak());
65
+ }
66
+ this.#pendingBlock = true;
67
+ }
68
+ add(nodes) {
69
+ for (const node of nodes) {
70
+ this.#addOne(node);
71
+ }
72
+ }
73
+ #addOne(node) {
74
+ switch (node.type) {
75
+ case "break":
76
+ this.out.push(lineBreak());
77
+ return;
78
+ case "text":
79
+ this.out.push({ type: "text", value: node.value.replace(/\s*\n\s*/g, " ") });
80
+ return;
81
+ case "inlineCode":
82
+ this.#codeLines(node.value);
83
+ return;
84
+ case "code":
85
+ this.block();
86
+ this.#codeLines(node.value);
87
+ return;
88
+ case "math":
89
+ this.block();
90
+ this.out.push({ type: "inlineMath", value: node.value.replace(/\s*\n\s*/g, " ") });
91
+ return;
92
+ case "list":
93
+ node.children.forEach((item, index) => {
94
+ this.block();
95
+ const marker = node.ordered ? `${(node.start ?? 1) + index}. ` : "- ";
96
+ this.out.push({ type: "text", value: marker });
97
+ this.#pendingBlock = false;
98
+ this.add(item.children);
99
+ });
100
+ return;
101
+ case "thematicBreak":
102
+ return;
103
+ default:
104
+ this.#addContainer(node);
105
+ }
106
+ }
107
+ #addContainer(node) {
108
+ if (PHRASING_PARENTS.has(node.type) && "children" in node) {
109
+ const inner = new CellBuilder();
110
+ inner.add(node.children);
111
+ this.out.push({ ...node, children: inner.out });
112
+ return;
113
+ }
114
+ if (PHRASING_TYPES.has(node.type)) {
115
+ this.out.push(node);
116
+ return;
117
+ }
118
+ if ("children" in node) {
119
+ this.block();
120
+ this.add(node.children);
121
+ }
122
+ }
123
+ #codeLines(value) {
124
+ value.split("\n").forEach((line, index) => {
125
+ if (index > 0) {
126
+ this.out.push(lineBreak());
127
+ }
128
+ if (line.trim().length > 0) {
129
+ this.out.push({ type: "inlineCode", value: line });
130
+ }
131
+ });
132
+ }
133
+ }
134
+ const flattenTableCells = (table) => {
135
+ for (const row of table.children) {
136
+ for (const cell of row.children) {
137
+ if (!needsFlattening(cell.children)) {
138
+ continue;
139
+ }
140
+ const builder = new CellBuilder();
141
+ builder.add(cell.children);
142
+ cell.children = builder.out;
143
+ }
144
+ }
145
+ };
146
+ const hasMultilineCode = (nodes) => nodes.some(
147
+ (node) => node.type === "code" && node.value.includes("\n") || "children" in node && hasMultilineCode(node.children)
148
+ );
149
+ const tableHasCodeSamples = (table) => table.children.some((row) => row.children.some((cell) => hasMultilineCode(cell.children)));
150
+ const unfoldTable = (table, hasHeader) => {
151
+ const [first, ...rest] = table.children;
152
+ const labelled = hasHeader && rest.length > 0;
153
+ const labels = labelled && first ? first.children.map((cell) => cell.children) : [];
154
+ const rows = labelled ? rest : table.children;
155
+ const out = [];
156
+ rows.forEach((row, rowIndex) => {
157
+ if (rowIndex > 0) {
158
+ out.push({ type: "thematicBreak" });
159
+ }
160
+ row.children.forEach((cell, cellIndex) => {
161
+ const label = labels[cellIndex];
162
+ if (label && label.length > 0) {
163
+ out.push({ type: "paragraph", children: [{ type: "strong", children: label }] });
164
+ }
165
+ out.push(...asBlocks(cell.children));
166
+ });
167
+ });
168
+ return out;
169
+ };
170
+ const asBlocks = (nodes) => {
171
+ const blocks = [];
172
+ let run = [];
173
+ const flush = () => {
174
+ if (run.some((node) => node.type !== "text" || node.value.trim().length > 0)) {
175
+ blocks.push({ type: "paragraph", children: run });
176
+ }
177
+ run = [];
178
+ };
179
+ for (const node of nodes) {
180
+ if (PHRASING_TYPES.has(node.type) || node.type === "break") {
181
+ run.push(node);
182
+ } else {
183
+ flush();
184
+ blocks.push(node);
185
+ }
186
+ }
187
+ flush();
188
+ return blocks;
189
+ };
190
+ // Annotate the CommonJS export names for ESM import in node:
191
+ 0 && (module.exports = {
192
+ flattenTableCells,
193
+ tableHasCodeSamples,
194
+ unfoldTable
195
+ });