webforai 1.2.0 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/cjs/extract-hast/index.js +35 -3
  2. package/dist/cjs/extract-hast/readability.js +57 -29
  3. package/dist/cjs/extract-hast/strongly.js +6 -3
  4. package/dist/cjs/extract-hast/utils.js +8 -4
  5. package/dist/cjs/html-to-mdast.js +6 -19
  6. package/dist/cjs/index.js +18 -9
  7. package/dist/cjs/link-replacer.js +2 -1
  8. package/dist/cjs/loaders/playwright.js +8 -16
  9. package/dist/cjs/md-splitter.js +5 -3
  10. package/dist/cjs/mdast-handlers/custom-a-handler.js +2 -1
  11. package/dist/cjs/mdast-handlers/custom-code-handler.js +43 -0
  12. package/dist/cjs/mdast-handlers/custom-div-handler.js +15 -9
  13. package/dist/cjs/mdast-handlers/custom-img-handler.js +2 -1
  14. package/dist/cjs/mdast-handlers/empty-handler.js +29 -0
  15. package/dist/cjs/mdast-to-markdown.js +2 -1
  16. package/dist/cjs/utils/mdast-utils.js +6 -3
  17. package/dist/extract-hast/index.js +28 -1
  18. package/dist/extract-hast/readability.js +62 -34
  19. package/dist/extract-hast/strongly.js +6 -3
  20. package/dist/extract-hast/utils.js +8 -4
  21. package/dist/html-to-mdast.js +7 -20
  22. package/dist/index.js +10 -4
  23. package/dist/link-replacer.js +2 -1
  24. package/dist/loaders/playwright.js +8 -6
  25. package/dist/md-splitter.js +5 -3
  26. package/dist/mdast-handlers/custom-a-handler.js +4 -3
  27. package/dist/mdast-handlers/custom-code-handler.js +20 -0
  28. package/dist/mdast-handlers/custom-div-handler.js +17 -11
  29. package/dist/mdast-handlers/custom-img-handler.js +2 -1
  30. package/dist/mdast-handlers/empty-handler.js +6 -0
  31. package/dist/mdast-to-markdown.js +2 -1
  32. package/dist/types/extract-hast/index.d.ts +10 -1
  33. package/dist/types/extract-hast/index.js.map +1 -1
  34. package/dist/types/extract-hast/readability.js.map +1 -1
  35. package/dist/types/extract-hast/strongly.js.map +1 -1
  36. package/dist/types/extract-hast/utils.js.map +1 -1
  37. package/dist/types/html-to-markdown.d.ts +1 -1
  38. package/dist/types/html-to-markdown.js.map +1 -1
  39. package/dist/types/html-to-mdast.d.ts +2 -4
  40. package/dist/types/html-to-mdast.js.map +1 -1
  41. package/dist/types/index.d.ts +4 -4
  42. package/dist/types/index.js.map +1 -1
  43. package/dist/types/link-replacer.js.map +1 -1
  44. package/dist/types/loaders/playwright.d.ts +2 -2
  45. package/dist/types/loaders/playwright.js.map +1 -1
  46. package/dist/types/md-splitter.d.ts +1 -1
  47. package/dist/types/md-splitter.js.map +1 -1
  48. package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
  49. package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
  50. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -0
  51. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  52. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  53. package/dist/types/mdast-handlers/empty-handler.d.ts +1 -0
  54. package/dist/types/mdast-handlers/empty-handler.js.map +1 -0
  55. package/dist/types/mdast-to-markdown.js.map +1 -1
  56. package/dist/types/utils/mdast-utils.js.map +1 -1
  57. package/dist/utils/mdast-utils.js +6 -3
  58. package/package.json +2 -2
@@ -3,6 +3,10 @@ var __defProp = Object.defineProperty;
3
3
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
4
  var __getOwnPropNames = Object.getOwnPropertyNames;
5
5
  var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
6
10
  var __copyProps = (to, from, except, desc) => {
7
11
  if (from && typeof from === "object" || typeof from === "function") {
8
12
  for (let key of __getOwnPropNames(from))
@@ -11,12 +15,40 @@ var __copyProps = (to, from, except, desc) => {
11
15
  }
12
16
  return to;
13
17
  };
14
- var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
15
18
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
16
19
  var extract_hast_exports = {};
20
+ __export(extract_hast_exports, {
21
+ DEFAULT_EXTRACT_HAST: () => DEFAULT_EXTRACT_HAST,
22
+ PRESET_EXTRACT_HAST: () => PRESET_EXTRACT_HAST,
23
+ extractHast: () => extractHast
24
+ });
17
25
  module.exports = __toCommonJS(extract_hast_exports);
18
- __reExport(extract_hast_exports, require("./strongly"), module.exports);
26
+ var import_readability = require("./readability");
27
+ var import_strongly = require("./strongly");
28
+ const PRESET_EXTRACT_HAST = {
29
+ strongly: import_strongly.stronglyExtractHast,
30
+ readability: import_readability.readabilityExtractHast
31
+ };
32
+ const DEFAULT_EXTRACT_HAST = ["readability"];
33
+ const extractHast = (hast, extractors = DEFAULT_EXTRACT_HAST) => {
34
+ const _extractors = Array.isArray(extractors) ? extractors : [extractors];
35
+ const extracted = _extractors.reduce((acc, extractor) => {
36
+ if (extractor === false) {
37
+ return acc;
38
+ }
39
+ if (typeof extractor === "string" && extractor in DEFAULT_EXTRACT_HAST) {
40
+ return PRESET_EXTRACT_HAST[extractor](acc);
41
+ }
42
+ if (typeof extractor === "function") {
43
+ return extractor(acc);
44
+ }
45
+ throw new Error(`Invalid extractor: ${extractor}`);
46
+ }, hast) || hast;
47
+ return extracted;
48
+ };
19
49
  // Annotate the CommonJS export names for ESM import in node:
20
50
  0 && (module.exports = {
21
- ...require("./strongly")
51
+ DEFAULT_EXTRACT_HAST,
52
+ PRESET_EXTRACT_HAST,
53
+ extractHast
22
54
  });
@@ -31,7 +31,7 @@ const REGEXPS = {
31
31
  hidden: /hidden|invisible|fallback-image/i,
32
32
  byline: /byline|author|dateline|writtenby|p-author/i,
33
33
  unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore/i,
34
- okMaybeItsACandidate: /and|article|body|column|content|main|shadow|code/i
34
+ okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i
35
35
  };
36
36
  const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry", "main"];
37
37
  const PARAGRAPH_TAGS = ["p", "div", "section", "article", "main", "ul", "ol", "li"];
@@ -40,60 +40,79 @@ const BASE_MINIMAL_LENGTH = {
40
40
  en: 500
41
41
  };
42
42
  const tryGetLang = (node) => {
43
- if (node.type !== "element")
43
+ if (node.type !== "element") {
44
44
  return;
45
+ }
45
46
  const element = node;
46
- if (element.tagName !== "html")
47
+ if (element.tagName !== "html") {
47
48
  return;
49
+ }
48
50
  const langAttr = element.properties.lang || element.properties["xml:lang"];
49
- if (langAttr)
51
+ if (langAttr) {
50
52
  return langAttr;
53
+ }
51
54
  };
52
55
  const metadataFilter = (node) => !["comment", "doctype"].includes(node.type);
53
56
  const universalElementFilter = (node) => {
54
- if (node.type !== "element")
57
+ if (node.type !== "element") {
55
58
  return true;
59
+ }
56
60
  const element = node;
57
- if (["script", "aside"].includes(element.tagName))
61
+ if (["script", "aside"].includes(element.tagName)) {
58
62
  return false;
59
- if (["hidden", "aria-hidden"].some((key) => element.properties[key]))
63
+ }
64
+ if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
60
65
  return false;
61
- if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname)))
66
+ }
67
+ if ((0, import_utils.classnames)(element).some((classname) => REGEXPS.hidden.test(classname))) {
62
68
  return false;
63
- if (element.tagName === "dialog")
69
+ }
70
+ if (element.tagName === "dialog") {
64
71
  return false;
65
- if (element.properties.role === "dialog" && element.properties["aria-modal"])
72
+ }
73
+ if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
66
74
  return false;
67
- if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author"))
75
+ }
76
+ if (element.properties.rel === "author" && (0, import_utils.isStrInclude)(element.properties.itemprop, "author")) {
68
77
  return false;
69
- if (REGEXPS.byline.test((0, import_utils.matchString)(element)))
78
+ }
79
+ if (REGEXPS.byline.test((0, import_utils.matchString)(element))) {
70
80
  return false;
71
- if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role))
81
+ }
82
+ if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role)) {
72
83
  return false;
84
+ }
73
85
  return true;
74
86
  };
75
87
  const unlikelyElementFilter = (node) => {
76
- if (node.type !== "element")
88
+ if (node.type !== "element") {
77
89
  return true;
90
+ }
78
91
  const element = node;
79
92
  const match = (0, import_utils.matchString)(element);
80
- if (["body", "article", "main", "section", "a"].includes(element.tagName))
93
+ if (["body", "article", "main", "section", "a"].includes(element.tagName)) {
81
94
  return true;
82
- if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3))
95
+ }
96
+ if ((0, import_utils.hasAncestors)(element, ["table", "code"], 3)) {
83
97
  return true;
84
- if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsACandidate.test(match))
98
+ }
99
+ if (REGEXPS.unlikelyCandidates.test(match) && !REGEXPS.okMaybeItsaCandidate.test(match)) {
85
100
  return false;
101
+ }
86
102
  return true;
87
103
  };
88
104
  const removeEmptyFilter = (node) => {
89
- if (node.type !== "element")
105
+ if (node.type !== "element") {
90
106
  return true;
107
+ }
91
108
  const element = node;
92
- if (!PARAGRAPH_TAGS.includes(element.tagName))
109
+ if (!PARAGRAPH_TAGS.includes(element.tagName)) {
93
110
  return true;
111
+ }
94
112
  const text = (0, import_hast_util_to_string.toString)(element);
95
- if (text.length < 10)
113
+ if (text.length < 10) {
96
114
  return false;
115
+ }
97
116
  return true;
98
117
  };
99
118
  const readabilityExtractHast = (hast) => {
@@ -101,45 +120,54 @@ const readabilityExtractHast = (hast) => {
101
120
  const body = (0, import_hast_util_select.select)("body", hast) ?? hast;
102
121
  const proxiedHast = (0, import_unist_util_parents.parents)(body);
103
122
  let baseFilterd = (0, import_unist_util_filter.filter)(proxiedHast, (node) => {
104
- if (!metadataFilter(node))
123
+ if (!metadataFilter(node)) {
105
124
  return false;
106
- if (!universalElementFilter(node))
125
+ }
126
+ if (!universalElementFilter(node)) {
107
127
  return false;
128
+ }
108
129
  return true;
109
130
  });
110
- if (!baseFilterd)
131
+ if (!baseFilterd) {
111
132
  return { type: "root", children: [] };
133
+ }
112
134
  const baseText = (0, import_hast_util_to_string.toString)(baseFilterd);
113
135
  let minimalLength = lang in BASE_MINIMAL_LENGTH ? BASE_MINIMAL_LENGTH[lang] : 500;
114
136
  if (baseText.length > minimalLength) {
115
137
  const filterd = (0, import_unist_util_filter.filter)(baseFilterd, (node) => {
116
- if (!unlikelyElementFilter(node))
138
+ if (!unlikelyElementFilter(node)) {
117
139
  return false;
140
+ }
118
141
  return true;
119
142
  });
120
- if (filterd)
143
+ if (filterd) {
121
144
  baseFilterd = filterd;
122
- } else
145
+ }
146
+ } else {
123
147
  minimalLength = Math.max(0, baseText.length - 200);
148
+ }
124
149
  let bodyTree = baseFilterd;
125
150
  for (const selector of BODY_SELECTORS) {
126
151
  const body2 = { type: "root", children: (0, import_hast_util_select.selectAll)(selector, baseFilterd) };
127
152
  const bodyText = (0, import_hast_util_to_string.toString)(body2);
128
- if (bodyText.length < 25)
153
+ if (bodyText.length < 25) {
129
154
  continue;
155
+ }
130
156
  const links = (0, import_hast_util_select.selectAll)("a", body2);
131
157
  const linkText = links.map((link) => (0, import_hast_util_to_string.toString)(link)).join("");
132
158
  const linkDensity = linkText.length / bodyText.length;
133
- if (linkDensity > 0.4)
159
+ if (linkDensity > 0.4) {
134
160
  continue;
161
+ }
135
162
  if (bodyText.length > minimalLength) {
136
163
  bodyTree = body2;
137
164
  break;
138
165
  }
139
166
  }
140
167
  const finalTree = (0, import_unist_util_filter.filter)(bodyTree, (node) => {
141
- if (!removeEmptyFilter(node))
168
+ if (!removeEmptyFilter(node)) {
142
169
  return false;
170
+ }
143
171
  return true;
144
172
  });
145
173
  return finalTree;
@@ -35,14 +35,17 @@ const stronglyExtractHast = (hast) => {
35
35
  }
36
36
  }
37
37
  const removedElements = (0, import_unist_util_filter.filter)(selectedElement, (node) => {
38
- if (node.type === "comment")
38
+ if (node.type === "comment") {
39
39
  return false;
40
- if (node.type !== "element")
40
+ }
41
+ if (node.type !== "element") {
41
42
  return true;
43
+ }
42
44
  const element = node;
43
45
  const elementName = `${element.tagName} ${element.properties.id} ${element.properties.className}`;
44
- if (REMOVE_REGEX.test(elementName))
46
+ if (REMOVE_REGEX.test(elementName)) {
45
47
  return false;
48
+ }
46
49
  return true;
47
50
  });
48
51
  return removedElements;
@@ -26,22 +26,26 @@ __export(utils_exports, {
26
26
  module.exports = __toCommonJS(utils_exports);
27
27
  const matchString = (element) => `${element.tagName} ${element.properties.id} ${classnames(element).join(" ")} }`;
28
28
  const classnames = (element) => {
29
- if (Array.isArray(element.properties.className))
29
+ if (Array.isArray(element.properties.className)) {
30
30
  return element.properties.className;
31
+ }
31
32
  return [];
32
33
  };
33
34
  const isStrInclude = (value, match) => {
34
- if (typeof value === "string")
35
+ if (typeof value === "string") {
35
36
  return value.includes(match);
37
+ }
36
38
  return false;
37
39
  };
38
40
  const hasAncestors = (element, tagNames, depth = 3) => {
39
41
  let parent = element.parent;
40
42
  for (let i = 0; i < depth; i++) {
41
- if (!parent || parent.type !== "element")
43
+ if (!parent || parent.type !== "element") {
42
44
  return false;
43
- if (tagNames.includes(parent.tagName))
45
+ }
46
+ if (tagNames.includes(parent.tagName)) {
44
47
  return true;
48
+ }
45
49
  parent = parent.parent;
46
50
  }
47
51
  return false;
@@ -24,39 +24,26 @@ module.exports = __toCommonJS(html_to_mdast_exports);
24
24
  var import_hast_util_from_html = require("hast-util-from-html");
25
25
  var import_hast_util_to_mdast = require("hast-util-to-mdast");
26
26
  var import_extract_hast = require("./extract-hast");
27
- var import_readability = require("./extract-hast/readability");
28
27
  var import_custom_a_handler = require("./mdast-handlers/custom-a-handler");
28
+ var import_custom_code_handler = require("./mdast-handlers/custom-code-handler");
29
29
  var import_custom_div_handler = require("./mdast-handlers/custom-div-handler");
30
30
  var import_custom_img_handler = require("./mdast-handlers/custom-img-handler");
31
31
  var import_custom_table_handler = require("./mdast-handlers/custom-table-handler");
32
+ var import_empty_handler = require("./mdast-handlers/empty-handler");
32
33
  var import_math_handler = require("./mdast-handlers/math-handler");
33
- const PRESET_EXTRACT_HAST = {
34
- strongly: import_extract_hast.stronglyExtractHast,
35
- readability: import_readability.readabilityExtractHast
36
- };
37
- const DEFAULT_EXTRACT_HAST = ["readability"];
38
34
  const htmlToMdast = (html, options) => {
39
- const { extractHast = DEFAULT_EXTRACT_HAST } = options || {};
40
- const _extractHast = Array.isArray(extractHast) ? extractHast : [extractHast];
35
+ const { extractHast: _extractHast } = options || {};
41
36
  const hast = (0, import_hast_util_from_html.fromHtml)(html, { fragment: true });
42
- const extractedHast = _extractHast.reduce((acc, extractHast2) => {
43
- if (extractHast2 === false)
44
- return acc;
45
- if (typeof extractHast2 === "string" && extractHast2 in PRESET_EXTRACT_HAST)
46
- return PRESET_EXTRACT_HAST[extractHast2](acc);
47
- if (typeof extractHast2 === "function")
48
- return extractHast2(acc);
49
- throw new Error(`Invalid extractHast: ${extractHast2}`);
50
- }, hast) || hast;
37
+ const extractedHast = (0, import_extract_hast.extractHast)(hast, _extractHast);
51
38
  const mdast = (0, import_hast_util_to_mdast.toMdast)(extractedHast, {
52
39
  handlers: {
53
40
  math: import_math_handler.mathHandler,
54
41
  div: import_custom_div_handler.customDivHandler,
42
+ pre: import_custom_code_handler.customCodeHandler,
55
43
  a: (0, import_custom_a_handler.customAHandler)({ asText: options?.linkAsText }),
56
44
  img: (0, import_custom_img_handler.customImgHandler)({ hideImage: options?.hideImage }),
57
45
  table: (0, import_custom_table_handler.customTableHandler)({ asText: options?.tableAsText }),
58
- br: () => {
59
- }
46
+ br: import_empty_handler.emptyHandler
60
47
  }
61
48
  });
62
49
  return mdast;
package/dist/cjs/index.js CHANGED
@@ -3,6 +3,10 @@ var __defProp = Object.defineProperty;
3
3
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
4
  var __getOwnPropNames = Object.getOwnPropertyNames;
5
5
  var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
6
10
  var __copyProps = (to, from, except, desc) => {
7
11
  if (from && typeof from === "object" || typeof from === "function") {
8
12
  for (let key of __getOwnPropNames(from))
@@ -11,18 +15,23 @@ var __copyProps = (to, from, except, desc) => {
11
15
  }
12
16
  return to;
13
17
  };
14
- var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
15
18
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
16
19
  var src_exports = {};
20
+ __export(src_exports, {
21
+ htmlToMarkdown: () => import_html_to_markdown.htmlToMarkdown,
22
+ htmlToMdast: () => import_html_to_mdast.htmlToMdast,
23
+ mdastSplitter: () => import_md_splitter.mdastSplitter,
24
+ mdastToMarkdown: () => import_mdast_to_markdown.mdastToMarkdown
25
+ });
17
26
  module.exports = __toCommonJS(src_exports);
18
- __reExport(src_exports, require("./html-to-markdown"), module.exports);
19
- __reExport(src_exports, require("./md-splitter"), module.exports);
20
- __reExport(src_exports, require("./html-to-mdast"), module.exports);
21
- __reExport(src_exports, require("./mdast-to-markdown"), module.exports);
27
+ var import_html_to_markdown = require("./html-to-markdown");
28
+ var import_md_splitter = require("./md-splitter");
29
+ var import_html_to_mdast = require("./html-to-mdast");
30
+ var import_mdast_to_markdown = require("./mdast-to-markdown");
22
31
  // Annotate the CommonJS export names for ESM import in node:
23
32
  0 && (module.exports = {
24
- ...require("./html-to-markdown"),
25
- ...require("./md-splitter"),
26
- ...require("./html-to-mdast"),
27
- ...require("./mdast-to-markdown")
33
+ htmlToMarkdown,
34
+ htmlToMdast,
35
+ mdastSplitter,
36
+ mdastToMarkdown
28
37
  });
@@ -24,8 +24,9 @@ module.exports = __toCommonJS(link_replacer_exports);
24
24
  const linkReplacer = (markdown, link) => {
25
25
  const regex = /(!?\[.*?\]\()(.+?)(\))/g;
26
26
  return markdown.replace(regex, (match, pre, thisUrl, post) => {
27
- if (thisUrl.startsWith("http") || thisUrl.startsWith("#"))
27
+ if (thisUrl.startsWith("http") || thisUrl.startsWith("#")) {
28
28
  return match;
29
+ }
29
30
  return `${pre}${new URL(thisUrl, link).href}${post}`;
30
31
  });
31
32
  };
@@ -1,9 +1,7 @@
1
1
  "use strict";
2
- var __create = Object.create;
3
2
  var __defProp = Object.defineProperty;
4
3
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
4
  var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
5
  var __hasOwnProp = Object.prototype.hasOwnProperty;
8
6
  var __export = (target, all) => {
9
7
  for (var name in all)
@@ -17,37 +15,31 @@ var __copyProps = (to, from, except, desc) => {
17
15
  }
18
16
  return to;
19
17
  };
20
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
- // If the importer is in node compatibility mode or this is not an ESM
22
- // file that has been converted to a CommonJS file using a Babel-
23
- // compatible transform (i.e. "__esModule" has not been set), then set
24
- // "default" to the CommonJS "module.exports" for node compatibility.
25
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
- mod
27
- ));
28
18
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
19
  var playwright_exports = {};
30
20
  __export(playwright_exports, {
31
21
  loadHtml: () => loadHtml
32
22
  });
33
23
  module.exports = __toCommonJS(playwright_exports);
34
- var playwright = __toESM(require("playwright"), 1);
24
+ var import_playwright = require("playwright");
35
25
  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
36
26
  const loadHtml = async (url, context) => {
37
27
  let _browser = null;
38
28
  let _context;
39
- if (!context) {
40
- _browser = await playwright.chromium.launch({ headless: true });
41
- _context = await _browser.newContext();
42
- } else
29
+ if (context) {
43
30
  _context = context;
31
+ } else {
32
+ _browser = await import_playwright.chromium.launch({ headless: true });
33
+ _context = await _browser.newContext();
34
+ }
44
35
  const page = await _context.newPage();
45
36
  await page.goto(url);
46
37
  await Promise.race([page.waitForLoadState("networkidle"), sleep(1e4)]);
47
38
  const html = await page.content();
48
39
  await page.close();
49
- if (_browser)
40
+ if (_browser) {
50
41
  await _browser.close();
42
+ }
51
43
  return html;
52
44
  };
53
45
  // Annotate the CommonJS export names for ESM import in node:
@@ -26,14 +26,16 @@ var import_common = require("./utils/common");
26
26
  var import_mdast_utils = require("./utils/mdast-utils");
27
27
  const PRIORITY_SPLITTERS = ["h1", "h2", "h3", "h4", "h5", "h6", "list", "table", "code"];
28
28
  const getSplitterGenerator = function* () {
29
- for (const splitter of PRIORITY_SPLITTERS)
29
+ for (const splitter of PRIORITY_SPLITTERS) {
30
30
  yield splitter;
31
+ }
31
32
  };
32
33
  const _mdastSplitter = async (contents, checker, splitterGenerator) => {
33
34
  const splitter = splitterGenerator.next().value;
34
35
  const markdown = (0, import_mdast_to_markdown.mdastToMarkdown)((0, import_mdast_utils.warpRoot)(contents));
35
- if (await checker(markdown) || contents.length === 1)
36
+ if (await checker(markdown) || contents.length === 1) {
36
37
  return [contents];
38
+ }
37
39
  const chunked = splitter ? contents.reduce((acc, content) => {
38
40
  if ((0, import_mdast_utils.internalType)(content) === splitter || acc.length === 0) {
39
41
  acc.push([content]);
@@ -45,7 +47,7 @@ const _mdastSplitter = async (contents, checker, splitterGenerator) => {
45
47
  const splitting = chunked.map((chunk2) => _mdastSplitter(chunk2, checker, splitterGenerator));
46
48
  return Promise.all(splitting).then((chunks) => chunks.flat());
47
49
  };
48
- const mdastSplitter = async (mdast, checker, options) => {
50
+ const mdastSplitter = (mdast, checker, _options) => {
49
51
  return _mdastSplitter((0, import_mdast_utils.unwarpRoot)(mdast), checker, getSplitterGenerator());
50
52
  };
51
53
  // Annotate the CommonJS export names for ESM import in node:
@@ -30,8 +30,9 @@ const customAHandler = (options) => (state, node) => {
30
30
  return link2;
31
31
  }
32
32
  const link = import_hast_util_to_mdast.defaultHandlers.a(state, node);
33
- if (link.children.length > 0)
33
+ if (link.children.length > 0) {
34
34
  return link;
35
+ }
35
36
  return void 0;
36
37
  };
37
38
  // Annotate the CommonJS export names for ESM import in node:
@@ -0,0 +1,43 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var custom_code_handler_exports = {};
20
+ __export(custom_code_handler_exports, {
21
+ customCodeHandler: () => customCodeHandler
22
+ });
23
+ module.exports = __toCommonJS(custom_code_handler_exports);
24
+ var import_hast_util_to_text = require("hast-util-to-text");
25
+ var import_trim_trailing_lines = require("trim-trailing-lines");
26
+ var import_detect_code_lang = require("../utils/detect-code-lang");
27
+ const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
28
+ const customCodeHandler = (state, node) => {
29
+ const classNames = node.properties?.className || [];
30
+ const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(node)).trim();
31
+ const classLang = classNames.map((className) => {
32
+ const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
33
+ return match?.[1];
34
+ }).find((className) => className);
35
+ const lang = classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
36
+ const result = { type: "code", lang, meta: null, value: codeValue };
37
+ state.patch(node, result);
38
+ return result;
39
+ };
40
+ // Annotate the CommonJS export names for ESM import in node:
41
+ 0 && (module.exports = {
42
+ customCodeHandler
43
+ });
@@ -29,35 +29,41 @@ var import_trim_trailing_lines = require("trim-trailing-lines");
29
29
  var import_detect_code_lang = require("../utils/detect-code-lang");
30
30
  const CODE_BLOCK_REGEX = /highlight-source|language-|codegroup|codeblock|code-block/i;
31
31
  const CODE_FILENAME_SELECTORS = "[class*='fileName'],[class*='fileName'],[class*='title'],[class*='Title']";
32
- const LANGUAGE_MATCH_REGEX = /language-(\w+)|highlight-source-(\w+)/;
32
+ const LANGUAGE_MATCH_REGEX = [/language-(\w+)/, /highlight-source-(\w+)/, /CodeBlock--language-(\w+)/];
33
33
  const findRecursive = (array, condition, maxDepth = 3) => {
34
- if (maxDepth <= 0)
34
+ if (maxDepth <= 0) {
35
35
  return null;
36
+ }
36
37
  for (const value of array) {
37
38
  const result = condition(value);
38
- if (Array.isArray(result))
39
+ if (Array.isArray(result)) {
39
40
  return findRecursive(result, condition, maxDepth - 1);
40
- if (result)
41
+ }
42
+ if (result) {
41
43
  return value;
44
+ }
42
45
  }
43
46
  return null;
44
47
  };
45
48
  const customDivHandler = (state, node) => {
46
49
  const classNames = Array.isArray(node.properties.className) ? node.properties.className : [];
47
50
  const codeBlock = findRecursive(node.children, (child) => {
48
- if (child.type !== "element")
51
+ if (child.type !== "element") {
49
52
  return false;
50
- if (child.tagName === "pre")
53
+ }
54
+ if (child.tagName === "pre") {
51
55
  return true;
56
+ }
52
57
  return child.children.filter((child2) => child2.type === "element");
53
58
  });
54
59
  if (codeBlock && classNames.some((className) => CODE_BLOCK_REGEX.test(className))) {
60
+ const codeBlockClassNames = codeBlock.type === "element" ? codeBlock.properties.className ?? [] : [];
55
61
  const codeValue = (0, import_trim_trailing_lines.trimTrailingLines)((0, import_hast_util_to_text.toText)(codeBlock)).trim();
56
62
  const filenameElement = (0, import_hast_util_select.select)(CODE_FILENAME_SELECTORS, node);
57
63
  const fileLang = filenameElement ? (0, import_hast_util_to_string.toString)(filenameElement).match(/\.(\w+)$/)?.[1] : null;
58
- const classLang = classNames.map((className) => {
59
- const match = className.match(LANGUAGE_MATCH_REGEX);
60
- return match?.[1] || match?.[2];
64
+ const classLang = [...classNames, ...codeBlockClassNames].map((className) => {
65
+ const match = LANGUAGE_MATCH_REGEX.map((regex) => className.match(regex)).find((match2) => match2);
66
+ return match?.[1];
61
67
  }).find((className) => className);
62
68
  const lang = fileLang || classLang || (0, import_detect_code_lang.detectLanguage)(codeValue) || null;
63
69
  const result = { type: "code", lang, meta: null, value: codeValue };
@@ -23,8 +23,9 @@ __export(custom_img_handler_exports, {
23
23
  module.exports = __toCommonJS(custom_img_handler_exports);
24
24
  var import_hast_util_to_mdast = require("hast-util-to-mdast");
25
25
  const customImgHandler = (options) => (state, node) => {
26
- if (options?.hideImage)
26
+ if (options?.hideImage) {
27
27
  return void 0;
28
+ }
28
29
  return import_hast_util_to_mdast.defaultHandlers.image(state, node);
29
30
  };
30
31
  // Annotate the CommonJS export names for ESM import in node:
@@ -0,0 +1,29 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
19
+ var empty_handler_exports = {};
20
+ __export(empty_handler_exports, {
21
+ emptyHandler: () => emptyHandler
22
+ });
23
+ module.exports = __toCommonJS(empty_handler_exports);
24
+ const emptyHandler = () => {
25
+ };
26
+ // Annotate the CommonJS export names for ESM import in node:
27
+ 0 && (module.exports = {
28
+ emptyHandler
29
+ });
@@ -30,8 +30,9 @@ const mdastToMarkdown = (mdast, options) => {
30
30
  let markdown = (0, import_mdast_util_to_markdown.toMarkdown)((0, import_mdast_utils.warpRoot)(mdast), {
31
31
  extensions: [(0, import_mdast_util_gfm.gfmToMarkdown)(), (0, import_mdast_util_math.mathToMarkdown)()]
32
32
  });
33
- if (options?.url)
33
+ if (options?.url) {
34
34
  markdown = (0, import_link_replacer.linkReplacer)(markdown, options.url);
35
+ }
35
36
  return markdown;
36
37
  };
37
38
  // Annotate the CommonJS export names for ESM import in node:
@@ -24,18 +24,21 @@ __export(mdast_utils_exports, {
24
24
  });
25
25
  module.exports = __toCommonJS(mdast_utils_exports);
26
26
  const unwarpRoot = (mdast) => {
27
- if (mdast.type === "root")
27
+ if (mdast.type === "root") {
28
28
  return mdast.children;
29
+ }
29
30
  return [mdast];
30
31
  };
31
32
  const warpRoot = (mdast) => {
32
- if (Array.isArray(mdast))
33
+ if (Array.isArray(mdast)) {
33
34
  return { type: "root", children: mdast };
35
+ }
34
36
  return mdast;
35
37
  };
36
38
  const internalType = (content) => {
37
- if (content.type === "heading")
39
+ if (content.type === "heading") {
38
40
  return `h${content.depth}`;
41
+ }
39
42
  return content.type;
40
43
  };
41
44
  // Annotate the CommonJS export names for ESM import in node: