webforai 2.1.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/README.md +86 -4
  2. package/dist/adapters/index.js +58 -0
  3. package/dist/adapters/registry.js +149 -0
  4. package/dist/adapters/sites/declarative.js +178 -0
  5. package/dist/adapters/sites/hackernews.js +94 -0
  6. package/dist/adapters/sites/youtube.js +165 -0
  7. package/dist/adapters/types.js +0 -0
  8. package/dist/bin.js +4142 -549
  9. package/dist/cjs/adapters/index.js +86 -0
  10. package/dist/cjs/adapters/registry.js +178 -0
  11. package/dist/cjs/adapters/sites/declarative.js +202 -0
  12. package/dist/cjs/adapters/sites/hackernews.js +117 -0
  13. package/dist/cjs/adapters/sites/youtube.js +191 -0
  14. package/dist/cjs/adapters/types.js +16 -0
  15. package/dist/cjs/constants.js +2 -2
  16. package/dist/cjs/detect-client-shell.js +78 -0
  17. package/dist/cjs/extract-meta-refresh.js +71 -0
  18. package/dist/cjs/extractors/index.js +5 -0
  19. package/dist/cjs/extractors/lib/constants.js +186 -0
  20. package/dist/cjs/extractors/lib/sanitize.js +429 -0
  21. package/dist/cjs/extractors/lib/score.js +264 -0
  22. package/dist/cjs/extractors/lib/terminators.js +131 -0
  23. package/dist/cjs/extractors/lib/ui-chrome.js +119 -0
  24. package/dist/cjs/extractors/pipeExtractors.js +12 -9
  25. package/dist/cjs/extractors/presets/auto.js +44 -0
  26. package/dist/cjs/extractors/presets/minimal-filter.js +27 -48
  27. package/dist/cjs/extractors/presets/takumi.js +84 -122
  28. package/dist/cjs/html-to-markdown.js +44 -7
  29. package/dist/cjs/html-to-mdast.js +10 -1
  30. package/dist/cjs/index.js +43 -2
  31. package/dist/cjs/link-replacer.js +19 -13
  32. package/dist/cjs/loaders/cf-puppeteer.js +14 -7
  33. package/dist/cjs/loaders/fetch.js +14 -2
  34. package/dist/cjs/loaders/playwright.js +32 -30
  35. package/dist/cjs/loaders/puppeteer.js +14 -7
  36. package/dist/cjs/md-splitter.js +24 -11
  37. package/dist/cjs/mdast-handlers/custom-a-handler.js +25 -9
  38. package/dist/cjs/mdast-handlers/custom-code-handler.js +18 -7
  39. package/dist/cjs/mdast-handlers/custom-div-handler.js +12 -33
  40. package/dist/cjs/mdast-handlers/custom-img-handler.js +23 -1
  41. package/dist/cjs/mdast-handlers/custom-table-handler.js +16 -0
  42. package/dist/cjs/mdast-handlers/definition-list-handler.js +110 -0
  43. package/dist/cjs/mdast-handlers/math-handler.js +12 -3
  44. package/dist/cjs/mdast-handlers/sup-sub-handler.js +39 -0
  45. package/dist/cjs/mdast-to-markdown.js +2 -5
  46. package/dist/cjs/metadata/index.js +223 -0
  47. package/dist/cjs/normalize/index.js +177 -0
  48. package/dist/cjs/platform/client.js +204 -0
  49. package/dist/cjs/platform/error.js +36 -0
  50. package/dist/cjs/platform/fetch.js +37 -0
  51. package/dist/cjs/platform/index.js +44 -0
  52. package/dist/cjs/platform/types.js +40 -0
  53. package/dist/cjs/utils/hast-fast.js +284 -0
  54. package/dist/constants.js +2 -2
  55. package/dist/detect-client-shell.js +53 -0
  56. package/dist/extract-meta-refresh.js +47 -0
  57. package/dist/extractors/index.js +3 -0
  58. package/dist/extractors/lib/constants.js +150 -0
  59. package/dist/extractors/lib/sanitize.js +419 -0
  60. package/dist/extractors/lib/score.js +242 -0
  61. package/dist/extractors/lib/terminators.js +107 -0
  62. package/dist/extractors/lib/ui-chrome.js +96 -0
  63. package/dist/extractors/pipeExtractors.js +12 -9
  64. package/dist/extractors/presets/auto.js +20 -0
  65. package/dist/extractors/presets/minimal-filter.js +35 -47
  66. package/dist/extractors/presets/takumi.js +91 -122
  67. package/dist/html-to-markdown.js +42 -6
  68. package/dist/html-to-mdast.js +10 -1
  69. package/dist/index.js +45 -2
  70. package/dist/link-replacer.js +19 -13
  71. package/dist/loaders/cf-puppeteer.js +15 -8
  72. package/dist/loaders/fetch.js +14 -2
  73. package/dist/loaders/playwright.js +32 -30
  74. package/dist/loaders/puppeteer.js +15 -8
  75. package/dist/md-splitter.js +24 -11
  76. package/dist/mdast-handlers/custom-a-handler.js +25 -9
  77. package/dist/mdast-handlers/custom-code-handler.js +17 -7
  78. package/dist/mdast-handlers/custom-div-handler.js +12 -33
  79. package/dist/mdast-handlers/custom-img-handler.js +23 -1
  80. package/dist/mdast-handlers/custom-table-handler.js +16 -0
  81. package/dist/mdast-handlers/definition-list-handler.js +87 -0
  82. package/dist/mdast-handlers/math-handler.js +12 -3
  83. package/dist/mdast-handlers/sup-sub-handler.js +16 -0
  84. package/dist/mdast-to-markdown.js +2 -5
  85. package/dist/metadata/index.js +198 -0
  86. package/dist/normalize/index.js +159 -0
  87. package/dist/platform/client.js +182 -0
  88. package/dist/platform/error.js +13 -0
  89. package/dist/platform/fetch.js +14 -0
  90. package/dist/platform/index.js +23 -0
  91. package/dist/platform/types.js +13 -0
  92. package/dist/types/adapters/index.d.ts +24 -0
  93. package/dist/types/adapters/index.js.map +1 -0
  94. package/dist/types/adapters/registry.d.ts +35 -0
  95. package/dist/types/adapters/registry.js.map +1 -0
  96. package/dist/types/adapters/sites/declarative.d.ts +13 -0
  97. package/dist/types/adapters/sites/declarative.js.map +1 -0
  98. package/dist/types/adapters/sites/hackernews.d.ts +13 -0
  99. package/dist/types/adapters/sites/hackernews.js.map +1 -0
  100. package/dist/types/adapters/sites/youtube.d.ts +31 -0
  101. package/dist/types/adapters/sites/youtube.js.map +1 -0
  102. package/dist/types/adapters/types.d.ts +99 -0
  103. package/dist/types/adapters/types.js.map +1 -0
  104. package/dist/types/cli/bin.js.map +1 -1
  105. package/dist/types/cli/commands/jobs/files.d.ts +28 -0
  106. package/dist/types/cli/commands/jobs/files.js.map +1 -0
  107. package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
  108. package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
  109. package/dist/types/cli/commands/jobs/options.d.ts +58 -0
  110. package/dist/types/cli/commands/jobs/options.js.map +1 -0
  111. package/dist/types/cli/commands/jobs/run.d.ts +44 -0
  112. package/dist/types/cli/commands/jobs/run.js.map +1 -0
  113. package/dist/types/cli/commands/skill/content.d.ts +9 -0
  114. package/dist/types/cli/commands/skill/content.js.map +1 -0
  115. package/dist/types/cli/commands/skill/index.d.ts +19 -0
  116. package/dist/types/cli/commands/skill/index.js.map +1 -0
  117. package/dist/types/cli/commands/webforai/interactive.d.ts +7 -0
  118. package/dist/types/cli/commands/webforai/interactive.js.map +1 -0
  119. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -1
  120. package/dist/types/cli/commands/webforai/options.d.ts +62 -0
  121. package/dist/types/cli/commands/webforai/options.js.map +1 -0
  122. package/dist/types/cli/commands/webforai/run.d.ts +10 -0
  123. package/dist/types/cli/commands/webforai/run.js.map +1 -0
  124. package/dist/types/cli/constants.d.ts +14 -4
  125. package/dist/types/cli/constants.js.map +1 -1
  126. package/dist/types/cli/helpers/selectExtractMode.js.map +1 -1
  127. package/dist/types/cli/helpers/selectLoader.js.map +1 -1
  128. package/dist/types/cli/utils.d.ts +8 -1
  129. package/dist/types/cli/utils.js.map +1 -1
  130. package/dist/types/constants.d.ts +7 -1
  131. package/dist/types/constants.js.map +1 -1
  132. package/dist/types/detect-client-shell.d.ts +24 -0
  133. package/dist/types/detect-client-shell.js.map +1 -0
  134. package/dist/types/extract-meta-refresh.d.ts +29 -0
  135. package/dist/types/extract-meta-refresh.js.map +1 -0
  136. package/dist/types/extractors/index.d.ts +1 -0
  137. package/dist/types/extractors/index.js.map +1 -1
  138. package/dist/types/extractors/lib/constants.d.ts +81 -0
  139. package/dist/types/extractors/lib/constants.js.map +1 -0
  140. package/dist/types/extractors/lib/sanitize.d.ts +66 -0
  141. package/dist/types/extractors/lib/sanitize.js.map +1 -0
  142. package/dist/types/extractors/lib/score.d.ts +85 -0
  143. package/dist/types/extractors/lib/score.js.map +1 -0
  144. package/dist/types/extractors/lib/terminators.d.ts +26 -0
  145. package/dist/types/extractors/lib/terminators.js.map +1 -0
  146. package/dist/types/extractors/lib/ui-chrome.d.ts +18 -0
  147. package/dist/types/extractors/lib/ui-chrome.js.map +1 -0
  148. package/dist/types/extractors/pipeExtractors.d.ts +5 -0
  149. package/dist/types/extractors/pipeExtractors.js.map +1 -1
  150. package/dist/types/extractors/presets/auto.d.ts +30 -0
  151. package/dist/types/extractors/presets/auto.js.map +1 -0
  152. package/dist/types/extractors/presets/minimal-filter.d.ts +10 -2
  153. package/dist/types/extractors/presets/minimal-filter.js.map +1 -1
  154. package/dist/types/extractors/presets/takumi.d.ts +19 -3
  155. package/dist/types/extractors/presets/takumi.js.map +1 -1
  156. package/dist/types/extractors/types.d.ts +8 -0
  157. package/dist/types/html-to-markdown.d.ts +38 -0
  158. package/dist/types/html-to-markdown.js.map +1 -1
  159. package/dist/types/html-to-mdast.d.ts +11 -0
  160. package/dist/types/html-to-mdast.js.map +1 -1
  161. package/dist/types/index.d.ts +8 -3
  162. package/dist/types/index.js.map +1 -1
  163. package/dist/types/link-replacer.d.ts +3 -1
  164. package/dist/types/link-replacer.js.map +1 -1
  165. package/dist/types/loaders/cf-puppeteer.js.map +1 -1
  166. package/dist/types/loaders/fetch.d.ts +4 -0
  167. package/dist/types/loaders/fetch.js.map +1 -1
  168. package/dist/types/loaders/playwright.js.map +1 -1
  169. package/dist/types/loaders/puppeteer.js.map +1 -1
  170. package/dist/types/md-splitter.d.ts +1 -1
  171. package/dist/types/md-splitter.js.map +1 -1
  172. package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
  173. package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
  174. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
  175. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  176. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  177. package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
  178. package/dist/types/mdast-handlers/definition-list-handler.d.ts +2 -0
  179. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -0
  180. package/dist/types/mdast-handlers/math-handler.js.map +1 -1
  181. package/dist/types/mdast-handlers/sup-sub-handler.d.ts +10 -0
  182. package/dist/types/mdast-handlers/sup-sub-handler.js.map +1 -0
  183. package/dist/types/mdast-to-markdown.js.map +1 -1
  184. package/dist/types/metadata/index.d.ts +47 -0
  185. package/dist/types/metadata/index.js.map +1 -0
  186. package/dist/types/normalize/index.d.ts +53 -0
  187. package/dist/types/normalize/index.js.map +1 -0
  188. package/dist/types/platform/client.d.ts +55 -0
  189. package/dist/types/platform/client.js.map +1 -0
  190. package/dist/types/platform/error.d.ts +22 -0
  191. package/dist/types/platform/error.js.map +1 -0
  192. package/dist/types/platform/fetch.d.ts +36 -0
  193. package/dist/types/platform/fetch.js.map +1 -0
  194. package/dist/types/platform/index.d.ts +4 -0
  195. package/dist/types/platform/index.js.map +1 -0
  196. package/dist/types/platform/types.d.ts +164 -0
  197. package/dist/types/platform/types.js.map +1 -0
  198. package/dist/types/utils/hast-fast.d.ts +101 -0
  199. package/dist/types/utils/hast-fast.js.map +1 -0
  200. package/dist/utils/hast-fast.js +246 -0
  201. package/package.json +25 -7
  202. package/dist/types/cli/commands/webforai/index.d.ts +0 -6
  203. package/dist/types/cli/commands/webforai/index.js.map +0 -1
@@ -0,0 +1 @@
1
+ {"version":3,"file":"client.js","sourceRoot":"","sources":["../../../src/platform/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAA0C,YAAY,EAAE,MAAM,SAAS,CAAC;AAC/E,OAAO,EAYN,gBAAgB,GAChB,MAAM,SAAS,CAAC;AAEjB,MAAM,CAAC,MAAM,gBAAgB,GAAG,+BAA+B,CAAC;AAEhE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;AAsDzD,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,MAAoB,EAAiB,EAAE,CACjE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;IAC/B,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QAC9E,OAAO;IACR,CAAC;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;QAC7B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9C,OAAO,EAAE,CAAC;IACX,CAAC,EAAE,EAAE,CAAC,CAAC;IACP,MAAM,OAAO,GAAG,GAAG,EAAE;QACpB,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,MAAM,CAAC,MAAM,EAAE,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IAChF,CAAC,CAAC;IACF,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC5D,CAAC,CAAC,CAAC;AAEJ,MAAM,eAAe,GAAG,CAAC,IAAa,EAAE,QAA2B,EAAsB,EAAE;IAC1F,MAAM,QAAQ,GAAI,IAAyD,EAAE,KAAK,EAAE,UAAU,CAAC;IAC/F,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,QAAQ,CAAC;IACjB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG,CAAC,MAAc,EAAE,YAAoB,EAAsB,EAAE;IAClF,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACpB,OAAO,+BAA+B,YAAY,EAAE,CAAC;IACtD,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACpB,OAAO,gDAAgD,YAAY,EAAE,CAAC;IACvE,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,QAA2B,EAAE,IAAa,EAAE,YAAoB,EAAoB,EAAE;IACzG,MAAM,QAAQ,GAAG,IAAqE,CAAC;IACvF,MAAM,IAAI,GAAG,OAAO,QAAQ,EAAE,KAAK,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAClG,IAAI,OAAO,GACV,OAAO,QAAQ,EAAE,KAAK,EAAE,OAAO,KAAK,QAAQ;QAC3C,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO;QACxB,CAAC,CAAC,6BAA6B,QAAQ,CAAC,MAAM,GAAG,CAAC;IACpD,MAAM,IAAI,GAAG,IAAI,KAAK,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACpG,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7C,OAAO,GAAG,GAAG,OAAO,KAAK,IAAI,GAAG,CAAC;IAClC,CAAC;IACD,OAAO,IAAI,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;AAC9F,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAAiC,EAAE,EAAkB,EAAE;IAC3F,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,gBAAgB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1E,MAAM,YAAY,GAAG,GAAG,OAAO,YAAY,CAAC;IAC5C,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAE9C,MAAM,OAAO,GAAG,KAAK,EACpB,IAAY,EACZ,IAAqF,EACpF,EAAE;QACH,MAAM,OAAO,GAA2B,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;QACvE,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;QAC9C,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,IAAI,gBAAgB,CACzB,iBAAiB,EACjB,0FAA0F,YAAY,8BAA8B,EACpI,CAAC,CACD,CAAC;YACH,CAAC;YACD,OAAO,CAAC,aAAa,GAAG,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC;QACpD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,GAAG,IAAI,EAAE,EAAE;YACrD,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;YACrE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/C,CAAC,CAAC;QAEH,MAAM,IAAI,GAAY,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,2BAA2B,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,IAAS,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,MAAoB,EAAsB,EAAE,CAC1E,OAAO,CAAC,YAAY,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAEzF,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,IAA0B,EAA2B,EAAE;QAC5F,MAAM,KAAK,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,OAAO,OAAO,CAAC,YAAY,kBAAkB,CAAC,KAAK,CAAC,WAAW,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACxG,CAAC,CAAC;IAEF,MAAM,aAAa,GAAG,KAAK,EAAE,SAAiB,EAAwB,EAAE;QACvE,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,gBAAgB,CACzB,uBAAuB,EACvB,oCAAoC,QAAQ,CAAC,MAAM,mCAAmC,EACtF,QAAQ,CAAC,MAAM,CACf,CAAC;QACH,CAAC;QACD,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAgB,CAAC;IAC/C,CAAC,CAAC;IAEF;;;OAGG;IACH,MAAM,eAAe,GAAG,KAAK,EAC5B,KAAa,EACb,MAAmB,EACnB,QAAgB,EACiB,EAAE;QACnC,IAAI,CAAC;YACJ,OAAO,MAAM,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,CAAC,KAAK,YAAY,gBAAgB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,CAAC,EAAE,CAAC;gBAClE,MAAM,KAAK,CAAC;YACb,CAAC;YACD,MAAM,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACzF,OAAO,SAAS,CAAC;QAClB,CAAC;IACF,CAAC,CAAC;IAEF,OAAO;QACN,MAAM,EAAE,CAAC,IAAmB,EAAE,EAAE,CAAC,OAAO,CAAe,YAAY,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEhH,WAAW,EAAE,CAAC,IAAmB,EAAE,EAAE,CACpC,OAAO,CAAS,YAAY,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAE9F,KAAK,EAAE,CAAC,IAAkB,EAAE,EAAE,CAAC,OAAO,CAAS,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEvG,KAAK,EAAE,CAAC,IAAkB,EAAE,EAAE,CAAC,OAAO,CAAS,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEvG,MAAM;QACN,aAAa;QAEb,UAAU,EAAE,KAAK,EAAE,KAAa,EAAE,OAA0B,EAAE,EAAsB,EAAE;YACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;YACjD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,UAAa,EAAE,CAAC;gBAC5E,MAAM,IAAI,UAAU,CAAC,iDAAiD,CAAC,CAAC;YACzE,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,OAAO,GAAG,UAAa,EAAE,CAAC;gBAC1E,MAAM,IAAI,UAAU,CAAC,yDAAyD,CAAC,CAAC;YACjF,CAAC;YACD,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;YAE9B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;YAC9B,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC5D,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,MAAM,KAAK,GAAG,UAAU,CACvB,GAAG,EAAE,CACJ,UAAU,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,cAAc,EAAE,OAAO,KAAK,0BAA0B,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,EAC7G,OAAO,CACP,CAAC;YACF,IAAI,aAAa,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;gBACvD,aAAa,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5C,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,KAAK,IAAwB,EAAE;gBAC3C,SAAS,CAAC;oBACT,MAAM,CAAC,cAAc,EAAE,CAAC;oBACxB,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;oBAC9D,MAAM,CAAC,cAAc,EAAE,CAAC;oBACxB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;wBAC1B,SAAS;oBACV,CAAC;oBACD,IAAI,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC;oBACxB,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;wBACxC,OAAO,MAAM,CAAC;oBACf,CAAC;oBACD,MAAM,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBAC/B,CAAC;YACF,CAAC,CAAC;YACF,IAAI,CAAC;gBACJ,uFAAuF;gBACvF,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;YAC9C,CAAC;oBAAS,CAAC;gBACV,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,IAAI,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBACnD,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YACpD,CAAC;QACF,CAAC;QAED,UAAU,EAAE,KAAK,SAAS,CAAC,EAAE,KAAa;YACzC,IAAI,MAA0B,CAAC;YAC/B,GAAG,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;gBACpD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjC,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC3E,CAAC;gBACD,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YACtB,CAAC,QAAQ,MAAM,EAAE;QAClB,CAAC;QAED,UAAU,EAAE,CAAC,IAAuB,EAAE,EAAE,CACvC,OAAO,CAAa,iBAAiB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;KACpF,CAAC;AACH,CAAC,CAAC"}
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Error thrown for any non-2xx platform response, mirroring the API's uniform
3
+ * `{ error: { code, message } }` envelope.
4
+ *
5
+ * `code` values worth branching on: `invalid_request` (400), `invalid_api_key` (401),
6
+ * `payment_required` / `spend_cap_reached` (402), `robots_disallowed` (403), `job_not_found`
7
+ * (404), `rate_limited` / `too_many_jobs` (429, `retryAfter` set when the server sends it),
8
+ * `engine_unavailable` / `result_unavailable` / `scheduling_unknown` (503). 401 and 402
9
+ * messages end with a hint naming the deployment's dashboard. The client never retries on its
10
+ * own; a caller that wants backoff should honour `retryAfter` when present.
11
+ *
12
+ * Network-level failures (DNS, refused connections) are NOT wrapped — they propagate as the
13
+ * runtime's own `fetch` errors. `code` is `invalid_response` when the server answered with
14
+ * something that is not the JSON envelope (usually a wrong `baseUrl`).
15
+ */
16
+ export declare class PlatformApiError extends Error {
17
+ readonly code: string;
18
+ readonly status: number;
19
+ /** Seconds to wait before retrying; set on rate-limited responses that provide it. */
20
+ readonly retryAfter?: number;
21
+ constructor(code: string, message: string, status: number, retryAfter?: number);
22
+ }
@@ -0,0 +1 @@
1
+ {"version":3,"file":"error.js","sourceRoot":"","sources":["../../../src/platform/error.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAM1C,YAAY,IAAY,EAAE,OAAe,EAAE,MAAc,EAAE,UAAmB;QAC7E,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC9B,CAAC;CACD"}
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Structural fetch contract — everything the client needs from an HTTP implementation, and
3
+ * nothing more.
4
+ *
5
+ * Deliberately NOT typed as `typeof fetch`: that resolves against whichever global lib the
6
+ * consumer compiles with (lib.dom, @types/node, @cloudflare/workers-types), and the three
7
+ * disagree. A structural type keeps the published d.ts independent of any of them, so the
8
+ * global fetch, a bound Cloudflare service binding (`env.MY_WORKER.fetch`), undici,
9
+ * node-fetch, or a hand-rolled test stub all satisfy it as long as they accept a URL string
10
+ * plus a plain init object and resolve to something response-shaped.
11
+ */
12
+ /** The request init the client sends. A plain object — always JSON bodies, never streams. */
13
+ export interface FetchRequestInit {
14
+ method: string;
15
+ headers?: Record<string, string>;
16
+ body?: string;
17
+ /** Cancels an in-flight status request when polling stops or times out. */
18
+ signal?: AbortSignal;
19
+ }
20
+ /** The subset of a Response the client reads. */
21
+ export interface FetchResponseLike {
22
+ ok: boolean;
23
+ status: number;
24
+ headers: {
25
+ get(name: string): string | null;
26
+ };
27
+ json(): Promise<unknown>;
28
+ }
29
+ export type FetchLike = (url: string, init: FetchRequestInit) => Promise<FetchResponseLike>;
30
+ /**
31
+ * Picks the fetch implementation once, at client creation, with a clear failure instead of a
32
+ * `Cannot read properties of undefined` deep inside the first request on runtimes without a
33
+ * global fetch (Node < 18). The global is bound to `globalThis` because browsers throw
34
+ * `Illegal invocation` when `window.fetch` is called unbound.
35
+ */
36
+ export declare const resolveFetch: (custom?: FetchLike) => FetchLike;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../../src/platform/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAqBH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAkB,EAAa,EAAE;IAC7D,IAAI,MAAM,EAAE,CAAC;QACZ,OAAO,MAAM,CAAC;IACf,CAAC;IACD,MAAM,WAAW,GAAI,UAAoC,CAAC,KAAK,CAAC;IAChE,IAAI,OAAO,WAAW,KAAK,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,kGAAkG,CAAC,CAAC;IACrH,CAAC;IACD,OAAO,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC,CAAC"}
@@ -0,0 +1,4 @@
1
+ export { DEFAULT_BASE_URL, createPlatformClient, type PlatformClient, type PlatformClientOptions, type WaitForJobOptions, } from "./client";
2
+ export { PlatformApiError } from "./error";
3
+ export type { FetchLike, FetchRequestInit, FetchResponseLike } from "./fetch";
4
+ export { CRAWL_SITEMAP_MODES, ENGINES, REGIONS, REQUESTED_ENGINES, isStoredPageStub, type BatchOptions, type ConvertOptions, type CrawlOptions, type CrawlSitemapMode, type DemoResult, type DemoScrapeOptions, type Engine, type ExtractorPreset, type JobRef, type JobResultItem, type JobResultsPage, type JobState, type JobStatus, type JobType, type PageError, type PageFailure, type PageResult, type PageSuccess, type Region, type RequestedEngine, type ScrapeOptions, type ScrapeResult, type StoredPageStub, } from "./types";
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/platform/index.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,OAAO,EACN,gBAAgB,EAChB,oBAAoB,GAIpB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,OAAO,EACN,mBAAmB,EACnB,OAAO,EACP,OAAO,EACP,iBAAiB,EACjB,gBAAgB,GAwBhB,MAAM,SAAS,CAAC"}
@@ -0,0 +1,164 @@
1
+ /**
2
+ * Wire types for the webforai platform HTTP API (`/v1`).
3
+ *
4
+ * These mirror the server contract exactly (`apps/platform/src/routes/*` in the webforai
5
+ * repository is the source of truth). Request bodies are validated strictly server-side:
6
+ * unknown keys are a `400`, so the client only ever sends fields the caller provided.
7
+ */
8
+ /** How the platform acquires the page's HTML. */
9
+ export declare const ENGINES: readonly ["fetch", "browser", "proxy-fetch", "proxy-browser"];
10
+ export type Engine = (typeof ENGINES)[number];
11
+ /**
12
+ * What a request may ask for: a concrete engine, or `auto` (the server default) — the
13
+ * cheapest engine for the request, escalated to browser rendering when the fetched HTML is
14
+ * a client-rendered shell. Responses report the concrete engine that ran.
15
+ */
16
+ export declare const REQUESTED_ENGINES: readonly ["fetch", "browser", "proxy-fetch", "proxy-browser", "auto"];
17
+ export type RequestedEngine = (typeof REQUESTED_ENGINES)[number];
18
+ /** Coarse egress location, honoured by the proxy engines only. */
19
+ /** Egress regions the hosted platform can honour (Japan, or no geo-targeting). */
20
+ export declare const REGIONS: readonly ["auto", "jp"];
21
+ export type Region = (typeof REGIONS)[number];
22
+ /** Extraction presets understood by the platform (passed through to the webforai library). */
23
+ export type ExtractorPreset = "auto" | "takumi" | "minimal" | "none";
24
+ export interface ConvertOptions {
25
+ extractor?: ExtractorPreset;
26
+ /** Prepend YAML front matter built from the page's metadata. */
27
+ frontmatter?: boolean;
28
+ /** Base URL used to absolutize relative links; defaults to the fetched URL. */
29
+ baseUrl?: string;
30
+ }
31
+ interface CommonRequestOptions {
32
+ /** Defaults to `auto` server-side. */
33
+ engine?: RequestedEngine;
34
+ /** Browser engines only (`browser`, `proxy-browser`); +1 credit. */
35
+ screenshot?: boolean;
36
+ /** Re-upload the page's images behind expiring URLs; +1 credit per started 5 images. */
37
+ rehostImages?: boolean;
38
+ region?: Region;
39
+ /**
40
+ * Honor the target site's robots.txt rules: each URL is checked before it is fetched, and a
41
+ * disallowed URL fails with `robots_disallowed` (never billed). Defaults to `false` for
42
+ * scrape and batch and to `true` for crawl (pass `false` to turn it off).
43
+ */
44
+ respectRobotsTxt?: boolean;
45
+ convert?: ConvertOptions;
46
+ }
47
+ export interface ScrapeOptions extends CommonRequestOptions {
48
+ url: string;
49
+ }
50
+ export interface BatchOptions extends CommonRequestOptions {
51
+ /** 1–100 URLs. */
52
+ urls: string[];
53
+ }
54
+ /**
55
+ * How a crawl uses the site's sitemaps (robots.txt `Sitemap:` lines, else `/sitemap.xml`):
56
+ * `skip` (default) ignores them; `include` adds their same-origin URLs at depth 1 next to the
57
+ * links found on pages; `only` crawls the seed plus sitemap URLs and follows no page links.
58
+ */
59
+ export declare const CRAWL_SITEMAP_MODES: readonly ["skip", "include", "only"];
60
+ export type CrawlSitemapMode = (typeof CRAWL_SITEMAP_MODES)[number];
61
+ export interface CrawlOptions extends CommonRequestOptions {
62
+ url: string;
63
+ /** 0–5, default 2. */
64
+ maxDepth?: number;
65
+ /** 1–500 pages, default 50. */
66
+ limit?: number;
67
+ /** Regexes matched against the pathname. */
68
+ includePaths?: string[];
69
+ excludePaths?: string[];
70
+ /** The platform currently only supports same-origin crawls; sending `false` is a 400. */
71
+ sameOrigin?: true;
72
+ /** Seed the crawl from the site's sitemaps; default `skip`. See {@link CRAWL_SITEMAP_MODES}. */
73
+ sitemap?: CrawlSitemapMode;
74
+ }
75
+ export interface ScrapeResult {
76
+ /** Final URL after redirects. */
77
+ url: string;
78
+ /** The engine that produced the result — `auto` requests resolve to a concrete one. */
79
+ engine: Engine;
80
+ markdown: string;
81
+ metadata: Record<string, unknown>;
82
+ credits: number;
83
+ /** Present when a screenshot was requested; the URL expires after ~24h. */
84
+ screenshotUrl?: string;
85
+ /** Present when `rehostImages` was requested. */
86
+ images?: {
87
+ original: string;
88
+ rehosted: string;
89
+ }[];
90
+ /** Present when the result is probably degraded (e.g. an unrendered client-side shell). */
91
+ warning?: string;
92
+ }
93
+ export interface JobRef {
94
+ jobId: string;
95
+ }
96
+ export type JobType = "batch" | "crawl";
97
+ export type JobState = "queued" | "running" | "completed" | "failed";
98
+ export interface JobStatus {
99
+ jobId: string;
100
+ type: JobType;
101
+ status: JobState;
102
+ total: number;
103
+ completed: number;
104
+ failed: number;
105
+ credits: number;
106
+ /** When the stored results disappear (ISO timestamp, 7 days after creation). */
107
+ expiresAt: string;
108
+ /** Present only when the job failed. */
109
+ error?: string;
110
+ }
111
+ export interface PageError {
112
+ code: string;
113
+ message: string;
114
+ }
115
+ export type PageSuccess = ScrapeResult & {
116
+ status: "ok";
117
+ };
118
+ export interface PageFailure {
119
+ status: "error";
120
+ url: string;
121
+ /** As requested — a failed `auto` page never resolved to a concrete engine. */
122
+ engine: RequestedEngine;
123
+ error: PageError;
124
+ }
125
+ /** One page of an async job, after the client resolves any spilled results. */
126
+ export type PageResult = PageSuccess | PageFailure;
127
+ /**
128
+ * Results larger than ~100KiB are stored out-of-band; the API returns this stub with an
129
+ * expiring `resultUrl` instead of the inline markdown. `jobResults()` resolves these
130
+ * automatically; `getJobResults()` returns them as-is.
131
+ */
132
+ export interface StoredPageStub {
133
+ status: "ok";
134
+ url: string;
135
+ engine: RequestedEngine;
136
+ credits: number;
137
+ resultUrl: string;
138
+ }
139
+ export type JobResultItem = PageResult | StoredPageStub;
140
+ export interface JobResultsPage {
141
+ jobId: string;
142
+ status: JobState;
143
+ results: JobResultItem[];
144
+ /** Absent on the last page. */
145
+ cursor?: string;
146
+ }
147
+ export interface DemoScrapeOptions {
148
+ url: string;
149
+ region?: Region;
150
+ }
151
+ export interface DemoResult {
152
+ url: string;
153
+ region: Region;
154
+ /** The engine `auto` resolved to. Absent from deployments older than 2026-09-24. */
155
+ engine?: Engine;
156
+ /** Truncated to ~40,000 characters by the demo endpoint (~8000 before 2026-09-24). */
157
+ markdown: string;
158
+ truncated: boolean;
159
+ title?: string;
160
+ metadata: Record<string, unknown>;
161
+ }
162
+ /** Narrows a job-results item to the out-of-band stub form. */
163
+ export declare const isStoredPageStub: (item: JobResultItem) => item is StoredPageStub;
164
+ export {};
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/platform/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,iDAAiD;AACjD,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,eAAe,CAAU,CAAC;AAGrF;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAG,OAAO,EAAE,MAAM,CAAU,CAAC;AAG/D,kEAAkE;AAClE,kFAAkF;AAClF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,IAAI,CAAU,CAAC;AAwC/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAU,CAAC;AAiHxE,+DAA+D;AAC/D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAmB,EAA0B,EAAE,CAC/E,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,WAAW,IAAI,IAAI,CAAC"}
@@ -0,0 +1,101 @@
1
+ /**
2
+ * Performance-critical HAST helpers.
3
+ *
4
+ * The v2 extractors leaned on `unist-util-filter` and `hast-util-to-string`, both of which
5
+ * walk (and in the case of `filter`, deep-clone) the entire tree on every call. A single run
6
+ * of `takumiExtractor` therefore cloned the document four times and re-walked it for every
7
+ * length measurement. The helpers here replace both with a single mutation pass and a
8
+ * memoised post-order metric walk.
9
+ */
10
+ import type { Element, Nodes as Hast, Parent, RootContent } from "hast";
11
+ /** Text metrics for a subtree, gathered in one post-order pass. */
12
+ export interface TextMetrics {
13
+ /** Length of the whitespace-normalised text content. */
14
+ text: number;
15
+ /** Length of text that sits inside an `<a>` descendant. */
16
+ link: number;
17
+ /** Number of sentence-separating commas (ASCII, CJK and Arabic). */
18
+ commas: number;
19
+ /** Number of `<p>`, `<pre>` and `<li>` descendants. */
20
+ paragraphs: number;
21
+ /** Number of element descendants (self excluded). */
22
+ elements: number;
23
+ }
24
+ export declare const isElement: (node: unknown) => node is Element;
25
+ export declare const isParent: (node: unknown) => node is Parent;
26
+ /** Normalised class list, tolerating both the array and the raw-string shapes hast can produce. */
27
+ export declare const classList: (element: Element) => string[];
28
+ /** `"tagname#id .class1 .class2"` — the string the heuristic regexps are matched against. */
29
+ export declare const matchString: (element: Element) => string;
30
+ /**
31
+ * A memoised metric collector bound to one snapshot of a tree.
32
+ *
33
+ * The cache is only sound while the tree is unchanged, so callers create a fresh collector
34
+ * after every mutation pass rather than invalidating entries individually.
35
+ */
36
+ export declare class MetricsCollector {
37
+ #private;
38
+ metrics(node: Hast): TextMetrics;
39
+ textLength(node: Hast): number;
40
+ /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
41
+ linkDensity(node: Hast): number;
42
+ }
43
+ /**
44
+ * Removes every node for which `keep` returns `false`, mutating `tree` in place.
45
+ *
46
+ * Unlike `unist-util-filter` this performs no cloning, so callers own the responsibility of
47
+ * copying the tree beforehand when the input must stay untouched.
48
+ *
49
+ * `keep` is called top-down; returning `false` for a node skips its subtree entirely, which is
50
+ * what makes a single pass sufficient.
51
+ */
52
+ export declare const pruneInPlace: (tree: Hast, keep: (node: Hast) => boolean) => Hast;
53
+ /**
54
+ * Depth-first pre-order walk. Return `false` from `visit` to skip a subtree.
55
+ *
56
+ * `boolean | void` is deliberate: it is the standard visitor signature (unist-util-visit uses
57
+ * the same shape) and lets callers that only observe nodes omit a return entirely.
58
+ */
59
+ export declare const walk: (tree: Hast, visit: (node: Hast, parent: Parent | undefined) => boolean | void) => void;
60
+ /**
61
+ * Collects every element matching `predicate`.
62
+ *
63
+ * This exists to replace repeated `hast-util-select` calls: the CSS engine re-parses the
64
+ * selector and re-walks the tree per call, while callers here usually only need a tag-name or
65
+ * attribute test that a single traversal can answer.
66
+ */
67
+ export declare const collectElements: (tree: Hast, predicate: (element: Element) => boolean) => Element[];
68
+ /** First element matching `predicate`, in document order. */
69
+ export declare const findElement: (tree: Hast, predicate: (element: Element) => boolean) => Element | undefined;
70
+ /**
71
+ * Structural clone of a HAST tree.
72
+ *
73
+ * `structuredClone` is available on every runtime the package targets (Node 18+, Deno, Bun,
74
+ * Workers) and is markedly faster than a JSON round trip, but it rejects the `Object.create(null)`
75
+ * prototypes that some HAST producers use for `properties`, so we fall back on a manual copy.
76
+ */
77
+ export declare const cloneHast: <T extends Hast>(tree: T) => T;
78
+ /** Wraps loose nodes into a root so downstream utilities always receive a single tree. */
79
+ export declare const asRoot: (nodes: Hast | RootContent[]) => Hast;
80
+ /**
81
+ * Reads a raw property, accepting the attribute spelling as written in HTML.
82
+ *
83
+ * `hast-util-from-html` stores attributes under their DOM property names, so `aria-hidden`
84
+ * becomes `ariaHidden` and `data-rwidth` becomes `dataRwidth`. Looking up the hyphenated form
85
+ * directly always misses — a mistake that silently disabled several v2 filters — so this
86
+ * helper checks the verbatim key first and then the camel-cased spelling.
87
+ */
88
+ export declare const property: (element: Element, key: string) => unknown;
89
+ /** Reads a property as a finite number, tolerating the string form attributes arrive in. */
90
+ export declare const numericProperty: (element: Element, key: string) => number | undefined;
91
+ /** Reads a property as a string, joining the array form hast uses for token lists. */
92
+ export declare const stringProperty: (element: Element, key: string) => string | undefined;
93
+ /** True when the attribute is present at all, regardless of its value. */
94
+ export declare const hasProperty: (element: Element, key: string) => boolean;
95
+ /**
96
+ * True for attributes that follow the ARIA/HTML "true unless explicitly false" convention.
97
+ *
98
+ * Boolean HTML attributes (`hidden`) arrive as `true`, while ARIA states arrive as the strings
99
+ * `"true"`/`"false"`, and an empty string means present-and-true.
100
+ */
101
+ export declare const isTruthyAttribute: (element: Element, key: string) => boolean;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hast-fast.js","sourceRoot":"","sources":["../../../src/utils/hast-fast.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;;;;;;;AAkBH,MAAM,aAAa,GAAgB,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;AAE/F,+EAA+E;AAC/E,MAAM,WAAW,GAAG,SAAS,CAAC;AAE9B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAE3F;;;GAGG;AACH,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE/G,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAa,EAAmB,EAAE,CAC3D,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAK,IAAa,CAAC,IAAI,KAAK,SAAS,CAAC;AAEhF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAkB,EAAE,CACzD,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,IAAe,CAAC,QAAQ,CAAC,CAAC;AAEvF,mGAAmG;AACnG,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,OAAgB,EAAY,EAAE;IACvD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,EAAE,SAAS,CAAC;IAChD,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO,SAAqB,CAAC;IAC9B,CAAC;IACD,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAU,EAAE;IACvD,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,OAAO,CAAC,OAAO,CAAC;IACxB,CAAC;IACD,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACtF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,OAAO,gBAAgB;IAA7B;QACU,kCAAS,IAAI,OAAO,EAAuB,EAAC;IA0EtD,CAAC;IAxEA,OAAO,CAAC,IAAU;QACjB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,aAAa,CAAC;YACtB,CAAC;YACD,OAAO;gBACN,IAAI,EAAE,KAAK,CAAC,MAAM;gBAClB,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM;gBAC/C,UAAU,EAAE,CAAC;gBACb,QAAQ,EAAE,CAAC;aACX,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,MAAM,MAAM,GAAG,uBAAA,IAAI,+BAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,MAAM,EAAE,CAAC;YACZ,OAAO,MAAM,CAAC;QACf,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzD,uBAAA,IAAI,+BAAO,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;YACrC,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,KAAa,CAAC,CAAC;YACjD,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC;YAC1B,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC;YAC1B,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC;YAC9B,UAAU,IAAI,YAAY,CAAC,UAAU,CAAC;YACtC,QAAQ,IAAI,YAAY,CAAC,QAAQ,CAAC;YAClC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,QAAQ,IAAI,CAAC,CAAC;gBACd,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvC,UAAU,IAAI,CAAC,CAAC;gBACjB,CAAC;YACF,CAAC;QACF,CAAC;QAED,iFAAiF;QACjF,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;YAC7C,IAAI,GAAG,IAAI,CAAC;QACb,CAAC;QAED,MAAM,OAAO,GAAgB,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QAC1E,uBAAA,IAAI,+BAAO,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/B,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,UAAU,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;IAChC,CAAC;IAED,qFAAqF;IACrF,WAAW,CAAC,IAAU;QACrB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YAChB,OAAO,CAAC,CAAC;QACV,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IACjC,CAAC;CACD;;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAU,EAAE,IAA6B,EAAQ,EAAE;IAC/E,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,4EAA4E;QAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,0FAA0F;QAC1F,6DAA6D;QAC7D,uEAAuE;QACvE,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAS,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClB,SAAS;YACV,CAAC;YACD,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,KAAoB,CAAC;YACzC,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC;IACzB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;GAKG;AACH,0FAA0F;AAC1F,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAU,EAAE,KAAiE,EAAQ,EAAE;IAC3G,MAAM,KAAK,GAAsD,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAErG,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,4EAA4E;QAC5E,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QACtC,IAAI,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,KAAK,EAAE,CAAC;YACnC,SAAS;QACV,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,0EAA0E;YAC1E,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAU,EAAE,SAAwC,EAAa,EAAE;IAClG,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACF,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,6DAA6D;AAC7D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAU,EAAE,SAAwC,EAAuB,EAAE;IACxG,IAAI,KAA0B,CAAC;IAC/B,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,KAAK,GAAG,IAAI,CAAC;YACb,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAiB,IAAO,EAAK,EAAE;IACvD,IAAI,CAAC;QACJ,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAI,IAAO,EAAK,EAAE;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,CAAM,CAAC;IACnC,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAS,CAAC;AAClB,CAAC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,KAA2B,EAAQ,EAAE;IAC3D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAW,EAAE;IAClE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACtC,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IACf,CAAC;IACD,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,OAAO,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,kBAAkB,GAAG,CAAC,GAAW,EAAU,EAAE,CAClD,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,EAAE,SAAiB,EAAE,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;AAEhF,4FAA4F;AAC5F,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAsB,EAAE;IACpF,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/C,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACtC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IACrD,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,sFAAsF;AACtF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAsB,EAAE;IACnF,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC;IACZ,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,CAAC;IACX,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,0EAA0E;AAC1E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,SAAS,CAAC;AAE5G;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,OAAgB,EAAE,GAAW,EAAW,EAAE;IAC3E,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
@@ -0,0 +1,246 @@
1
+ // src/utils/hast-fast.ts
2
+ var EMPTY_METRICS = { text: 0, link: 0, commas: 0, paragraphs: 0, elements: 0 };
3
+ var COMMA_REGEX = /[,、,،]/g;
4
+ var PARAGRAPH_LIKE = /* @__PURE__ */ new Set(["p", "pre", "li", "blockquote", "figcaption", "dd", "dt"]);
5
+ var NON_PROSE_TAGS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "object"]);
6
+ var isElement = (node) => typeof node === "object" && node !== null && node.type === "element";
7
+ var isParent = (node) => typeof node === "object" && node !== null && Array.isArray(node.children);
8
+ var classList = (element) => {
9
+ const className = element.properties?.className;
10
+ if (Array.isArray(className)) {
11
+ return className;
12
+ }
13
+ if (typeof className === "string") {
14
+ return className.split(/\s+/).filter(Boolean);
15
+ }
16
+ return [];
17
+ };
18
+ var matchString = (element) => {
19
+ const id = element.properties?.id;
20
+ const classes = classList(element);
21
+ if (!id && classes.length === 0) {
22
+ return element.tagName;
23
+ }
24
+ return `${element.tagName} ${typeof id === "string" ? id : ""} ${classes.join(" ")}`;
25
+ };
26
+ var MetricsCollector = class {
27
+ #cache = /* @__PURE__ */ new WeakMap();
28
+ metrics(node) {
29
+ if (node.type === "text") {
30
+ const value = node.value;
31
+ if (value.trim().length === 0) {
32
+ return EMPTY_METRICS;
33
+ }
34
+ return {
35
+ text: value.length,
36
+ link: 0,
37
+ commas: (value.match(COMMA_REGEX) ?? []).length,
38
+ paragraphs: 0,
39
+ elements: 0
40
+ };
41
+ }
42
+ if (!isParent(node)) {
43
+ return EMPTY_METRICS;
44
+ }
45
+ const cached = this.#cache.get(node);
46
+ if (cached) {
47
+ return cached;
48
+ }
49
+ if (isElement(node) && NON_PROSE_TAGS.has(node.tagName)) {
50
+ this.#cache.set(node, EMPTY_METRICS);
51
+ return EMPTY_METRICS;
52
+ }
53
+ let text = 0;
54
+ let link = 0;
55
+ let commas = 0;
56
+ let paragraphs = 0;
57
+ let elements = 0;
58
+ for (const child of node.children) {
59
+ const childMetrics = this.metrics(child);
60
+ text += childMetrics.text;
61
+ link += childMetrics.link;
62
+ commas += childMetrics.commas;
63
+ paragraphs += childMetrics.paragraphs;
64
+ elements += childMetrics.elements;
65
+ if (isElement(child)) {
66
+ elements += 1;
67
+ if (PARAGRAPH_LIKE.has(child.tagName)) {
68
+ paragraphs += 1;
69
+ }
70
+ }
71
+ }
72
+ if (isElement(node) && node.tagName === "a") {
73
+ link = text;
74
+ }
75
+ const metrics = { text, link, commas, paragraphs, elements };
76
+ this.#cache.set(node, metrics);
77
+ return metrics;
78
+ }
79
+ textLength(node) {
80
+ return this.metrics(node).text;
81
+ }
82
+ /** Fraction of a subtree's text that is link anchor text. `0` for empty subtrees. */
83
+ linkDensity(node) {
84
+ const { text, link } = this.metrics(node);
85
+ if (text === 0) {
86
+ return 0;
87
+ }
88
+ return Math.min(1, link / text);
89
+ }
90
+ };
91
+ var pruneInPlace = (tree, keep) => {
92
+ if (!isParent(tree)) {
93
+ return tree;
94
+ }
95
+ const stack = [tree];
96
+ while (stack.length > 0) {
97
+ const parent = stack.pop();
98
+ const children = parent.children;
99
+ let write = 0;
100
+ for (let read = 0; read < children.length; read++) {
101
+ const child = children[read];
102
+ if (!keep(child)) {
103
+ continue;
104
+ }
105
+ children[write++] = child;
106
+ if (isParent(child)) {
107
+ stack.push(child);
108
+ }
109
+ }
110
+ children.length = write;
111
+ }
112
+ return tree;
113
+ };
114
+ var walk = (tree, visit) => {
115
+ const stack = [{ node: tree, parent: void 0 }];
116
+ while (stack.length > 0) {
117
+ const { node, parent } = stack.pop();
118
+ if (visit(node, parent) === false) {
119
+ continue;
120
+ }
121
+ if (isParent(node)) {
122
+ for (let i = node.children.length - 1; i >= 0; i--) {
123
+ stack.push({ node: node.children[i], parent: node });
124
+ }
125
+ }
126
+ }
127
+ };
128
+ var collectElements = (tree, predicate) => {
129
+ const found = [];
130
+ walk(tree, (node) => {
131
+ if (isElement(node) && predicate(node)) {
132
+ found.push(node);
133
+ }
134
+ });
135
+ return found;
136
+ };
137
+ var findElement = (tree, predicate) => {
138
+ let found;
139
+ walk(tree, (node) => {
140
+ if (found) {
141
+ return false;
142
+ }
143
+ if (isElement(node) && predicate(node)) {
144
+ found = node;
145
+ return false;
146
+ }
147
+ });
148
+ return found;
149
+ };
150
+ var cloneHast = (tree) => {
151
+ try {
152
+ return structuredClone(tree);
153
+ } catch {
154
+ return manualClone(tree);
155
+ }
156
+ };
157
+ var manualClone = (node) => {
158
+ if (Array.isArray(node)) {
159
+ return node.map(manualClone);
160
+ }
161
+ if (typeof node !== "object" || node === null) {
162
+ return node;
163
+ }
164
+ const copy = {};
165
+ for (const [key, value] of Object.entries(node)) {
166
+ copy[key] = manualClone(value);
167
+ }
168
+ return copy;
169
+ };
170
+ var asRoot = (nodes) => {
171
+ if (Array.isArray(nodes)) {
172
+ return { type: "root", children: nodes };
173
+ }
174
+ return nodes;
175
+ };
176
+ var property = (element, key) => {
177
+ const properties = element.properties;
178
+ if (!properties) {
179
+ return void 0;
180
+ }
181
+ const direct = properties[key];
182
+ if (direct !== void 0) {
183
+ return direct;
184
+ }
185
+ if (!(key.includes("-") || key.includes(":"))) {
186
+ return void 0;
187
+ }
188
+ return properties[camelCaseAttribute(key)];
189
+ };
190
+ var camelCaseAttribute = (key) => key.replace(/[-:]([a-z])/g, (_, character) => character.toUpperCase());
191
+ var numericProperty = (element, key) => {
192
+ const raw = property(element, key);
193
+ if (typeof raw === "number") {
194
+ return Number.isFinite(raw) ? raw : void 0;
195
+ }
196
+ if (typeof raw === "string" && raw.length > 0) {
197
+ const parsed = Number.parseFloat(raw);
198
+ return Number.isFinite(parsed) ? parsed : void 0;
199
+ }
200
+ return void 0;
201
+ };
202
+ var stringProperty = (element, key) => {
203
+ const raw = property(element, key);
204
+ if (typeof raw === "string") {
205
+ return raw;
206
+ }
207
+ if (Array.isArray(raw)) {
208
+ return raw.join(" ");
209
+ }
210
+ if (raw === true) {
211
+ return "";
212
+ }
213
+ return void 0;
214
+ };
215
+ var hasProperty = (element, key) => property(element, key) !== void 0;
216
+ var isTruthyAttribute = (element, key) => {
217
+ const raw = property(element, key);
218
+ if (raw === void 0 || raw === null || raw === false) {
219
+ return false;
220
+ }
221
+ if (raw === true) {
222
+ return true;
223
+ }
224
+ if (typeof raw === "string") {
225
+ return raw.toLowerCase() !== "false";
226
+ }
227
+ return true;
228
+ };
229
+ export {
230
+ MetricsCollector,
231
+ asRoot,
232
+ classList,
233
+ cloneHast,
234
+ collectElements,
235
+ findElement,
236
+ hasProperty,
237
+ isElement,
238
+ isParent,
239
+ isTruthyAttribute,
240
+ matchString,
241
+ numericProperty,
242
+ property,
243
+ pruneInPlace,
244
+ stringProperty,
245
+ walk
246
+ };