webforai 2.1.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/README.md +86 -4
  2. package/dist/adapters/index.js +58 -0
  3. package/dist/adapters/registry.js +149 -0
  4. package/dist/adapters/sites/declarative.js +178 -0
  5. package/dist/adapters/sites/hackernews.js +94 -0
  6. package/dist/adapters/sites/youtube.js +165 -0
  7. package/dist/adapters/types.js +0 -0
  8. package/dist/bin.js +4142 -549
  9. package/dist/cjs/adapters/index.js +86 -0
  10. package/dist/cjs/adapters/registry.js +178 -0
  11. package/dist/cjs/adapters/sites/declarative.js +202 -0
  12. package/dist/cjs/adapters/sites/hackernews.js +117 -0
  13. package/dist/cjs/adapters/sites/youtube.js +191 -0
  14. package/dist/cjs/adapters/types.js +16 -0
  15. package/dist/cjs/constants.js +2 -2
  16. package/dist/cjs/detect-client-shell.js +78 -0
  17. package/dist/cjs/extract-meta-refresh.js +71 -0
  18. package/dist/cjs/extractors/index.js +5 -0
  19. package/dist/cjs/extractors/lib/constants.js +186 -0
  20. package/dist/cjs/extractors/lib/sanitize.js +429 -0
  21. package/dist/cjs/extractors/lib/score.js +264 -0
  22. package/dist/cjs/extractors/lib/terminators.js +131 -0
  23. package/dist/cjs/extractors/lib/ui-chrome.js +119 -0
  24. package/dist/cjs/extractors/pipeExtractors.js +12 -9
  25. package/dist/cjs/extractors/presets/auto.js +44 -0
  26. package/dist/cjs/extractors/presets/minimal-filter.js +27 -48
  27. package/dist/cjs/extractors/presets/takumi.js +84 -122
  28. package/dist/cjs/html-to-markdown.js +44 -7
  29. package/dist/cjs/html-to-mdast.js +10 -1
  30. package/dist/cjs/index.js +43 -2
  31. package/dist/cjs/link-replacer.js +19 -13
  32. package/dist/cjs/loaders/cf-puppeteer.js +14 -7
  33. package/dist/cjs/loaders/fetch.js +14 -2
  34. package/dist/cjs/loaders/playwright.js +32 -30
  35. package/dist/cjs/loaders/puppeteer.js +14 -7
  36. package/dist/cjs/md-splitter.js +24 -11
  37. package/dist/cjs/mdast-handlers/custom-a-handler.js +25 -9
  38. package/dist/cjs/mdast-handlers/custom-code-handler.js +18 -7
  39. package/dist/cjs/mdast-handlers/custom-div-handler.js +12 -33
  40. package/dist/cjs/mdast-handlers/custom-img-handler.js +23 -1
  41. package/dist/cjs/mdast-handlers/custom-table-handler.js +16 -0
  42. package/dist/cjs/mdast-handlers/definition-list-handler.js +110 -0
  43. package/dist/cjs/mdast-handlers/math-handler.js +12 -3
  44. package/dist/cjs/mdast-handlers/sup-sub-handler.js +39 -0
  45. package/dist/cjs/mdast-to-markdown.js +2 -5
  46. package/dist/cjs/metadata/index.js +223 -0
  47. package/dist/cjs/normalize/index.js +177 -0
  48. package/dist/cjs/platform/client.js +204 -0
  49. package/dist/cjs/platform/error.js +36 -0
  50. package/dist/cjs/platform/fetch.js +37 -0
  51. package/dist/cjs/platform/index.js +44 -0
  52. package/dist/cjs/platform/types.js +40 -0
  53. package/dist/cjs/utils/hast-fast.js +284 -0
  54. package/dist/constants.js +2 -2
  55. package/dist/detect-client-shell.js +53 -0
  56. package/dist/extract-meta-refresh.js +47 -0
  57. package/dist/extractors/index.js +3 -0
  58. package/dist/extractors/lib/constants.js +150 -0
  59. package/dist/extractors/lib/sanitize.js +419 -0
  60. package/dist/extractors/lib/score.js +242 -0
  61. package/dist/extractors/lib/terminators.js +107 -0
  62. package/dist/extractors/lib/ui-chrome.js +96 -0
  63. package/dist/extractors/pipeExtractors.js +12 -9
  64. package/dist/extractors/presets/auto.js +20 -0
  65. package/dist/extractors/presets/minimal-filter.js +35 -47
  66. package/dist/extractors/presets/takumi.js +91 -122
  67. package/dist/html-to-markdown.js +42 -6
  68. package/dist/html-to-mdast.js +10 -1
  69. package/dist/index.js +45 -2
  70. package/dist/link-replacer.js +19 -13
  71. package/dist/loaders/cf-puppeteer.js +15 -8
  72. package/dist/loaders/fetch.js +14 -2
  73. package/dist/loaders/playwright.js +32 -30
  74. package/dist/loaders/puppeteer.js +15 -8
  75. package/dist/md-splitter.js +24 -11
  76. package/dist/mdast-handlers/custom-a-handler.js +25 -9
  77. package/dist/mdast-handlers/custom-code-handler.js +17 -7
  78. package/dist/mdast-handlers/custom-div-handler.js +12 -33
  79. package/dist/mdast-handlers/custom-img-handler.js +23 -1
  80. package/dist/mdast-handlers/custom-table-handler.js +16 -0
  81. package/dist/mdast-handlers/definition-list-handler.js +87 -0
  82. package/dist/mdast-handlers/math-handler.js +12 -3
  83. package/dist/mdast-handlers/sup-sub-handler.js +16 -0
  84. package/dist/mdast-to-markdown.js +2 -5
  85. package/dist/metadata/index.js +198 -0
  86. package/dist/normalize/index.js +159 -0
  87. package/dist/platform/client.js +182 -0
  88. package/dist/platform/error.js +13 -0
  89. package/dist/platform/fetch.js +14 -0
  90. package/dist/platform/index.js +23 -0
  91. package/dist/platform/types.js +13 -0
  92. package/dist/types/adapters/index.d.ts +24 -0
  93. package/dist/types/adapters/index.js.map +1 -0
  94. package/dist/types/adapters/registry.d.ts +35 -0
  95. package/dist/types/adapters/registry.js.map +1 -0
  96. package/dist/types/adapters/sites/declarative.d.ts +13 -0
  97. package/dist/types/adapters/sites/declarative.js.map +1 -0
  98. package/dist/types/adapters/sites/hackernews.d.ts +13 -0
  99. package/dist/types/adapters/sites/hackernews.js.map +1 -0
  100. package/dist/types/adapters/sites/youtube.d.ts +31 -0
  101. package/dist/types/adapters/sites/youtube.js.map +1 -0
  102. package/dist/types/adapters/types.d.ts +99 -0
  103. package/dist/types/adapters/types.js.map +1 -0
  104. package/dist/types/cli/bin.js.map +1 -1
  105. package/dist/types/cli/commands/jobs/files.d.ts +28 -0
  106. package/dist/types/cli/commands/jobs/files.js.map +1 -0
  107. package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
  108. package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
  109. package/dist/types/cli/commands/jobs/options.d.ts +58 -0
  110. package/dist/types/cli/commands/jobs/options.js.map +1 -0
  111. package/dist/types/cli/commands/jobs/run.d.ts +44 -0
  112. package/dist/types/cli/commands/jobs/run.js.map +1 -0
  113. package/dist/types/cli/commands/skill/content.d.ts +9 -0
  114. package/dist/types/cli/commands/skill/content.js.map +1 -0
  115. package/dist/types/cli/commands/skill/index.d.ts +19 -0
  116. package/dist/types/cli/commands/skill/index.js.map +1 -0
  117. package/dist/types/cli/commands/webforai/interactive.d.ts +7 -0
  118. package/dist/types/cli/commands/webforai/interactive.js.map +1 -0
  119. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -1
  120. package/dist/types/cli/commands/webforai/options.d.ts +62 -0
  121. package/dist/types/cli/commands/webforai/options.js.map +1 -0
  122. package/dist/types/cli/commands/webforai/run.d.ts +10 -0
  123. package/dist/types/cli/commands/webforai/run.js.map +1 -0
  124. package/dist/types/cli/constants.d.ts +14 -4
  125. package/dist/types/cli/constants.js.map +1 -1
  126. package/dist/types/cli/helpers/selectExtractMode.js.map +1 -1
  127. package/dist/types/cli/helpers/selectLoader.js.map +1 -1
  128. package/dist/types/cli/utils.d.ts +8 -1
  129. package/dist/types/cli/utils.js.map +1 -1
  130. package/dist/types/constants.d.ts +7 -1
  131. package/dist/types/constants.js.map +1 -1
  132. package/dist/types/detect-client-shell.d.ts +24 -0
  133. package/dist/types/detect-client-shell.js.map +1 -0
  134. package/dist/types/extract-meta-refresh.d.ts +29 -0
  135. package/dist/types/extract-meta-refresh.js.map +1 -0
  136. package/dist/types/extractors/index.d.ts +1 -0
  137. package/dist/types/extractors/index.js.map +1 -1
  138. package/dist/types/extractors/lib/constants.d.ts +81 -0
  139. package/dist/types/extractors/lib/constants.js.map +1 -0
  140. package/dist/types/extractors/lib/sanitize.d.ts +66 -0
  141. package/dist/types/extractors/lib/sanitize.js.map +1 -0
  142. package/dist/types/extractors/lib/score.d.ts +85 -0
  143. package/dist/types/extractors/lib/score.js.map +1 -0
  144. package/dist/types/extractors/lib/terminators.d.ts +26 -0
  145. package/dist/types/extractors/lib/terminators.js.map +1 -0
  146. package/dist/types/extractors/lib/ui-chrome.d.ts +18 -0
  147. package/dist/types/extractors/lib/ui-chrome.js.map +1 -0
  148. package/dist/types/extractors/pipeExtractors.d.ts +5 -0
  149. package/dist/types/extractors/pipeExtractors.js.map +1 -1
  150. package/dist/types/extractors/presets/auto.d.ts +30 -0
  151. package/dist/types/extractors/presets/auto.js.map +1 -0
  152. package/dist/types/extractors/presets/minimal-filter.d.ts +10 -2
  153. package/dist/types/extractors/presets/minimal-filter.js.map +1 -1
  154. package/dist/types/extractors/presets/takumi.d.ts +19 -3
  155. package/dist/types/extractors/presets/takumi.js.map +1 -1
  156. package/dist/types/extractors/types.d.ts +8 -0
  157. package/dist/types/html-to-markdown.d.ts +38 -0
  158. package/dist/types/html-to-markdown.js.map +1 -1
  159. package/dist/types/html-to-mdast.d.ts +11 -0
  160. package/dist/types/html-to-mdast.js.map +1 -1
  161. package/dist/types/index.d.ts +8 -3
  162. package/dist/types/index.js.map +1 -1
  163. package/dist/types/link-replacer.d.ts +3 -1
  164. package/dist/types/link-replacer.js.map +1 -1
  165. package/dist/types/loaders/cf-puppeteer.js.map +1 -1
  166. package/dist/types/loaders/fetch.d.ts +4 -0
  167. package/dist/types/loaders/fetch.js.map +1 -1
  168. package/dist/types/loaders/playwright.js.map +1 -1
  169. package/dist/types/loaders/puppeteer.js.map +1 -1
  170. package/dist/types/md-splitter.d.ts +1 -1
  171. package/dist/types/md-splitter.js.map +1 -1
  172. package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
  173. package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
  174. package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
  175. package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
  176. package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
  177. package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
  178. package/dist/types/mdast-handlers/definition-list-handler.d.ts +2 -0
  179. package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -0
  180. package/dist/types/mdast-handlers/math-handler.js.map +1 -1
  181. package/dist/types/mdast-handlers/sup-sub-handler.d.ts +10 -0
  182. package/dist/types/mdast-handlers/sup-sub-handler.js.map +1 -0
  183. package/dist/types/mdast-to-markdown.js.map +1 -1
  184. package/dist/types/metadata/index.d.ts +47 -0
  185. package/dist/types/metadata/index.js.map +1 -0
  186. package/dist/types/normalize/index.d.ts +53 -0
  187. package/dist/types/normalize/index.js.map +1 -0
  188. package/dist/types/platform/client.d.ts +55 -0
  189. package/dist/types/platform/client.js.map +1 -0
  190. package/dist/types/platform/error.d.ts +22 -0
  191. package/dist/types/platform/error.js.map +1 -0
  192. package/dist/types/platform/fetch.d.ts +36 -0
  193. package/dist/types/platform/fetch.js.map +1 -0
  194. package/dist/types/platform/index.d.ts +4 -0
  195. package/dist/types/platform/index.js.map +1 -0
  196. package/dist/types/platform/types.d.ts +164 -0
  197. package/dist/types/platform/types.js.map +1 -0
  198. package/dist/types/utils/hast-fast.d.ts +101 -0
  199. package/dist/types/utils/hast-fast.js.map +1 -0
  200. package/dist/utils/hast-fast.js +246 -0
  201. package/package.json +25 -7
  202. package/dist/types/cli/commands/webforai/index.d.ts +0 -6
  203. package/dist/types/cli/commands/webforai/index.js.map +0 -1

There are too many changes on this page to be displayed.

The amount of changes on this page would crash your brower.

You can still verify the content by downloading the package file manually.