webforai 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/dist/bin.js +972 -213
  2. package/dist/cjs/cli/bin.js +33 -0
  3. package/dist/cjs/cli/commands/webforai/index.js +86 -0
  4. package/dist/cjs/cli/commands/webforai/loadHtml.js +93 -0
  5. package/dist/cjs/cli/constants.js +34 -0
  6. package/dist/cjs/cli/helpers/assertContinue.js +34 -0
  7. package/dist/cjs/cli/helpers/inputOutputPath.js +89 -0
  8. package/dist/cjs/cli/helpers/inputSourcePath.js +64 -0
  9. package/dist/cjs/cli/helpers/selectExtractMode.js +39 -0
  10. package/dist/cjs/cli/helpers/selectLoader.js +44 -0
  11. package/dist/cjs/cli/utils.js +89 -0
  12. package/dist/cjs/extract-hast/index.js +4 -3
  13. package/dist/cjs/extract-hast/readability.js +28 -61
  14. package/dist/cjs/extract-hast/strongly.js +2 -1
  15. package/dist/cjs/extract-hast/utils.js +0 -15
  16. package/dist/cjs/html-to-mdast.js +6 -6
  17. package/dist/cjs/link-replacer.js +10 -5
  18. package/dist/cjs/loaders/playwright.js +36 -7
  19. package/dist/cjs/mdast-handlers/custom-br-handler.js +31 -0
  20. package/dist/cjs/mdast-to-markdown.js +1 -1
  21. package/dist/cjs/utils/hast-utils.js +71 -0
  22. package/dist/cli/bin.js +11 -0
  23. package/dist/cli/commands/webforai/index.js +53 -0
  24. package/dist/cli/commands/webforai/loadHtml.js +60 -0
  25. package/dist/cli/constants.js +9 -0
  26. package/dist/cli/helpers/assertContinue.js +11 -0
  27. package/dist/cli/helpers/inputOutputPath.js +56 -0
  28. package/dist/cli/helpers/inputSourcePath.js +31 -0
  29. package/dist/cli/helpers/selectExtractMode.js +16 -0
  30. package/dist/cli/helpers/selectLoader.js +21 -0
  31. package/dist/cli/utils.js +52 -0
  32. package/dist/extract-hast/index.js +4 -3
  33. package/dist/extract-hast/readability.js +29 -62
  34. package/dist/extract-hast/strongly.js +2 -1
  35. package/dist/extract-hast/utils.js +0 -14
  36. package/dist/html-to-mdast.js +6 -6
  37. package/dist/link-replacer.js +10 -5
  38. package/dist/loaders/playwright.js +36 -7
  39. package/dist/mdast-handlers/custom-br-handler.js +8 -0
  40. package/dist/mdast-to-markdown.js +1 -1
  41. package/dist/types/cli/bin.js.map +1 -0
  42. package/dist/types/cli/commands/webforai/index.d.ts +6 -0
  43. package/dist/types/cli/commands/webforai/index.js.map +1 -0
  44. package/dist/types/cli/commands/webforai/loadHtml.d.ts +3 -0
  45. package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -0
  46. package/dist/types/cli/constants.d.ts +5 -0
  47. package/dist/types/cli/constants.js.map +1 -0
  48. package/dist/types/cli/helpers/assertContinue.d.ts +1 -0
  49. package/dist/types/cli/helpers/assertContinue.js.map +1 -0
  50. package/dist/types/cli/helpers/inputOutputPath.d.ts +1 -0
  51. package/dist/types/cli/helpers/inputOutputPath.js.map +1 -0
  52. package/dist/types/cli/helpers/inputSourcePath.d.ts +1 -0
  53. package/dist/types/cli/helpers/inputSourcePath.js.map +1 -0
  54. package/dist/types/cli/helpers/selectExtractMode.d.ts +1 -0
  55. package/dist/types/cli/helpers/selectExtractMode.js.map +1 -0
  56. package/dist/types/cli/helpers/selectLoader.d.ts +1 -0
  57. package/dist/types/cli/helpers/selectLoader.js.map +1 -0
  58. package/dist/types/cli/utils.d.ts +32 -0
  59. package/dist/types/cli/utils.js.map +1 -0
  60. package/dist/types/extract-hast/index.d.ts +9 -4
  61. package/dist/types/extract-hast/index.js.map +1 -1
  62. package/dist/types/extract-hast/readability.d.ts +2 -1
  63. package/dist/types/extract-hast/readability.js.map +1 -1
  64. package/dist/types/extract-hast/strongly.d.ts +2 -1
  65. package/dist/types/extract-hast/strongly.js.map +1 -1
  66. package/dist/types/extract-hast/utils.d.ts +0 -1
  67. package/dist/types/extract-hast/utils.js.map +1 -1
  68. package/dist/types/html-to-mdast.d.ts +4 -0
  69. package/dist/types/html-to-mdast.js.map +1 -1
  70. package/dist/types/link-replacer.d.ts +1 -1
  71. package/dist/types/link-replacer.js.map +1 -1
  72. package/dist/types/loaders/playwright.d.ts +8 -2
  73. package/dist/types/loaders/playwright.js.map +1 -1
  74. package/dist/types/mdast-handlers/custom-br-handler.d.ts +2 -0
  75. package/dist/types/mdast-handlers/custom-br-handler.js.map +1 -0
  76. package/dist/types/mdast-to-markdown.js.map +1 -1
  77. package/dist/types/utils/hast-utils.d.ts +4 -0
  78. package/dist/types/utils/hast-utils.js.map +1 -0
  79. package/dist/utils/hast-utils.js +46 -0
  80. package/package.json +18 -17
  81. package/dist/cjs/bin.js +0 -254
  82. package/dist/cjs/html-to-markdown.test.js +0 -146
  83. package/dist/cjs/link-replacer.test.js +0 -47
  84. package/dist/cjs/loaders/fetch.test.js +0 -9
  85. package/dist/cjs/loaders/playwright.test.js +0 -17
  86. package/dist/cjs/loaders/puppeteer.test.js +0 -17
  87. package/dist/cjs/utils/bin-utils.js +0 -124
  88. package/dist/html-to-markdown.test.js +0 -124
  89. package/dist/link-replacer.test.js +0 -47
  90. package/dist/loaders/fetch.test.js +0 -9
  91. package/dist/loaders/playwright.test.js +0 -17
  92. package/dist/loaders/puppeteer.test.js +0 -17
  93. package/dist/types/bin.js.map +0 -1
  94. package/dist/types/utils/bin-utils.d.ts +0 -5
  95. package/dist/types/utils/bin-utils.js.map +0 -1
  96. package/dist/utils/bin-utils.js +0 -87
  97. /package/dist/types/{bin.d.ts → cli/bin.d.ts} +0 -0
@@ -1 +1 @@
1
- {"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE9E,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,yQAAyQ;IAC1Q,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AAE/G,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEpF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,IAAU,EAAE,EAAE;IACjC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACR,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,QAAQ,EAAE,CAAC;QACd,OAAO,QAAkB,CAAC;IAC3B,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,8BAA8B;IAC9B,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAE7B,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IACpE,MAAM,YAAY,GAAG,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;IAEpE,OAAO,WAAW,KAAK,YAAY,IAAI,WAAW,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,EAAE;IACxC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAQ,EAAE;IAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAC1C,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;QACzC,IAAI,CAAC,cAAc,CAAC,IAAY,CAAC,EAAE,CAAC;YACnC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,sBAAsB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACvC,CAAC;IAED,MAAM,eAAe,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAClD,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,GACvB,eAAe,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI;QAC5E,CAAC,CAAE,CAAC,WAAmB,EAAE,eAAe,CAAW;QACnD,CAAC,CAAE,CAAC,IAAY,EAAE,QAAQ,CAAW,CAAC;IAExC,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACtF,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC7B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QACtD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACxC,QAAQ,GAAG,OAAO,CAAC;YACnB,QAAQ,GAAG,WAAW,CAAC;YACvB,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE;QACnD,IAAI,CAAC,iBAAiB,CAAC,IAAY,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,MAAM,SAAS,GAAG,YAAY,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC9G,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
1
+ {"version":3,"file":"readability.js","sourceRoot":"","sources":["../../../src/extract-hast/readability.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEhE,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;AAE5G,MAAM,OAAO,GAAG;IACf,MAAM,EAAE,kCAAkC;IAC1C,MAAM,EAAE,4CAA4C;IACpD,yBAAyB,EAAE,8BAA8B;IACzD,kBAAkB,EACjB,wRAAwR;IACzR,oBAAoB,EAAE,mDAAmD;CACzE,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEvG,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEzF,MAAM,mBAAmB,GAAG;IAC3B,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACP,CAAC;AAEF,yBAAyB;AACzB,UAAU;AACV,WAAW;AACX,KAAK;AAEL,MAAM,cAAc,GAAG,CAAC,IAAU,EAAE,EAAE;IACrC,OAAO,CAAC,CACP,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAC1C,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CACnH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yCAAyC;IACzC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,KAAK,CAAC;IACd,CAAC;IAED,yBAAyB;IACzB,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,KAAK,CAAC;IACd,CAAC;IAED,wBAAwB;IACxB,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,IAAc,CAAC,EAAE,CAAC;QAC3F,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAU,EAAE,EAAE;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,6BAA6B;IAC7B,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/G,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACd,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzF,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAU,EAAE,KAAa,EAAE,EAAE;IACvD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAEhC,IAAI,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC;IACb,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,MAAqB,EAAQ,EAAE;IACrE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IACrC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;IAE1C,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAY,CAAC,CAAC,CAAC;IAClF,MAAM,wBAAwB,GAAG,oBAAoB,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC5F,IAAI,CAAC,CAAC,oBAAoB,IAAI,wBAAwB,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,oBAAoB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAY,CAAC,CAAC,CAAC;IACjG,MAAM,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACrE,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GACzB,eAAe,CAAC,MAAM,GAAG,wBAAwB,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,IAAI;QAC5F,CAAC,CAAE,CAAC,WAAmB,EAAE,eAAe,CAAW;QACnD,CAAC,CAAE,CAAC,oBAA4B,EAAE,wBAAwB,CAAW,CAAC;IAExE,IAAI,aAAa,GAAG,IAAI,IAAI,mBAAmB,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtH,IAAI,QAAQ,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;QACrC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,aAAa,GAAS,QAAQ,CAAC;IACnC,IAAI,aAAa,GAAG,QAAQ,CAAC;IAE7B,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACtF,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC7B,SAAS;QACV,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;QACzD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,SAAS;QACV,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACxC,aAAa,GAAG,OAAO,CAAC;YACxB,aAAa,GAAG,WAAW,CAAC;YAC5B,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE;QACxD,IAAI,CAAC,iBAAiB,CAAC,IAAY,EAAE,IAAI,CAAC,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC,qBAAqB,CAAC,IAAY,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC,CAAS,CAAC;IAEX,MAAM,SAAS,GACd,YAAY,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC;IACvG,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
@@ -1,2 +1,3 @@
1
1
  import type { Nodes as Hast } from "hast";
2
- export declare const stronglyExtractHast: (hast: Hast) => Hast;
2
+ import type { ExtractParams } from ".";
3
+ export declare const stronglyExtractHast: (params: ExtractParams) => Hast;
@@ -1 +1 @@
1
- {"version":3,"file":"strongly.js","sourceRoot":"","sources":["../../../src/extract-hast/strongly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,YAAY,GACjB,iPAAiP,CAAC;AAEnP,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,IAAU,EAAQ,EAAE;IACvD,MAAM,SAAS,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;IAC/E,IAAI,eAAe,GAAG,IAAI,CAAC;IAC3B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE,CAAC;YACb,eAAe,GAAG,OAAO,CAAC;YAC1B,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE;QACxD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,OAAO,GAAG,IAAe,CAAC;QAChC,MAAM,WAAW,GAAG,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAClG,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,OAAO,eAAuB,CAAC;AAChC,CAAC,CAAC"}
1
+ {"version":3,"file":"strongly.js","sourceRoot":"","sources":["../../../src/extract-hast/strongly.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAG3C,MAAM,YAAY,GACjB,iPAAiP,CAAC;AAEnP,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAqB,EAAQ,EAAE;IAClE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IACxB,MAAM,SAAS,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;IAC/E,IAAI,eAAe,GAAG,IAAI,CAAC;IAC3B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE,CAAC;YACb,eAAe,GAAG,OAAO,CAAC;YAC1B,MAAM;QACP,CAAC;IACF,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE;QACxD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,OAAO,GAAG,IAAe,CAAC;QAChC,MAAM,WAAW,GAAG,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAClG,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,OAAO,eAAuB,CAAC;AAChC,CAAC,CAAC"}
@@ -2,4 +2,3 @@ import type { Element } from "hast";
2
2
  export declare const matchString: (element: Element) => string;
3
3
  export declare const classnames: (element: Element) => string[];
4
4
  export declare const isStrInclude: (value: unknown, match: string) => boolean;
5
- export declare const hasAncestors: (element: Element, tagNames: string[], depth?: number) => boolean;
@@ -1 +1 @@
1
- {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/extract-hast/utils.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE,CAC/C,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAEhF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,OAAO,OAAO,CAAC,UAAU,CAAC,SAAqB,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,EAAE;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAE,QAAkB,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE;IAC/E,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC"}
1
+ {"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/extract-hast/utils.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE,CAC/C,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AAEhF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,OAAO,OAAO,CAAC,UAAU,CAAC,SAAqB,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,EAAE;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC,CAAC"}
@@ -13,6 +13,10 @@ export type HtmlToMdastOptions = {
13
13
  tableAsText?: boolean;
14
14
  /** Whether to hide images. */
15
15
  hideImage?: boolean;
16
+ /** The language of the HTML. */
17
+ lang?: string;
18
+ /** The URL of the HTML. */
19
+ url?: string;
16
20
  };
17
21
  /**
18
22
  * Converts an HTML string or HAST tree to an MDAST tree.
@@ -1 +1 @@
1
- {"version":3,"file":"html-to-mdast.js","sourceRoot":"","sources":["../../src/html-to-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAG7C,OAAO,EAAmB,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAgB5D;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAyB,EAAE,OAA4B,EAAS,EAAE;IAC7F,MAAM,EAAE,UAAU,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IAErC,MAAM,IAAI,GAAG,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAEpG,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAEpD,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE;QACpC,QAAQ,EAAE;YACT,IAAI,EAAE,WAAW;YACjB,GAAG,EAAE,gBAAgB;YACrB,GAAG,EAAE,iBAAiB;YACtB,CAAC,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;YAClD,GAAG,EAAE,gBAAgB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YACxD,KAAK,EAAE,kBAAkB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAC3D,EAAE,EAAE,YAAY;SAChB;KACD,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAE3C,OAAO,cAAc,CAAC;AACvB,CAAC,CAAC"}
1
+ {"version":3,"file":"html-to-mdast.js","sourceRoot":"","sources":["../../src/html-to-mdast.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAG7C,OAAO,EAAmB,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uCAAuC,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAoBrF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAyB,EAAE,OAA4B,EAAS,EAAE;IAC7F,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IAEzE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GACjB,OAAO,UAAU,KAAK,QAAQ;QAC7B,CAAC,CAAC,CAAC,WAAW,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC,WAAW,IAAI,eAAe,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,UAAU,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;IAEnE,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE;QACpC,QAAQ,EAAE;YACT,IAAI,EAAE,WAAW;YACjB,GAAG,EAAE,gBAAgB;YACrB,GAAG,EAAE,iBAAiB;YACtB,CAAC,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;YAClD,GAAG,EAAE,gBAAgB,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;YACxD,KAAK,EAAE,kBAAkB,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;SAC3D;KACD,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAE3C,OAAO,cAAc,CAAC;AACvB,CAAC,CAAC"}
@@ -1 +1 @@
1
- export declare const linkReplacer: (markdown: string, link: string) => string;
1
+ export declare const linkReplacer: (markdown: string, base: string) => string;
@@ -1 +1 @@
1
- {"version":3,"file":"link-replacer.js","sourceRoot":"","sources":["../../src/link-replacer.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAgB,EAAE,IAAY,EAAE,EAAE;IAC9D,MAAM,KAAK,GAAG,yBAAyB,CAAC;IACxC,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;QAC5D,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,OAAO,KAAK,CAAC;QACd,CAAC;QACD,OAAO,GAAG,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;IACtD,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC"}
1
+ {"version":3,"file":"link-replacer.js","sourceRoot":"","sources":["../../src/link-replacer.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAgB,EAAE,IAAY,EAAE,EAAE;IAC9D,MAAM,KAAK,GAAG,6BAA6B,CAAC;IAC5C,OAAO,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACxD,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;QACd,CAAC;QACD,IAAI,CAAC;YACJ,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;YAC5C,OAAO,GAAG,GAAG,GAAG,WAAW,GAAG,IAAI,EAAE,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC"}
@@ -1,4 +1,10 @@
1
- import { type Browser, type BrowserContext } from "playwright";
1
+ import { type Browser } from "playwright-core";
2
+ export type LoadHtmlOptions = {
3
+ browser?: Browser;
4
+ timeout?: number;
5
+ waitUntil?: "load" | "domcontentloaded" | "networkidle";
6
+ superBypassMode?: boolean;
7
+ };
2
8
  /**
3
9
  * Useful function for load the HTML of a URL using Playwright.
4
10
  * **Not recommended** for use in production environments.
@@ -13,4 +19,4 @@ import { type Browser, type BrowserContext } from "playwright";
13
19
  * console.log(html);
14
20
  * ```
15
21
  */
16
- export declare const loadHtml: (url: string, context?: BrowserContext | Browser) => Promise<string>;
22
+ export declare const loadHtml: (url: string, options?: LoadHtmlOptions) => Promise<string>;
@@ -1 +1 @@
1
- {"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAkC,EAAE,EAAE;IACjF,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAExE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
1
+ {"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../src/loaders/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,QAAQ,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AASlE,MAAM,mBAAmB,GAAG;IAC3B,SAAS,EACR,qHAAqH;IACtH,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACvC,iBAAiB,EAAE,CAAC;IACpB,QAAQ,EAAE,KAAK;IACf,QAAQ,EAAE,KAAK;IACf,iBAAiB,EAAE,IAAI;IACvB,MAAM,EAAE,OAAO;IACf,UAAU,EAAE,kBAAkB;CAC9B,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAE,OAAyB,EAAE,EAAE;IACxE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;IACvE,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAE7G,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE;YAChC,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9E,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACrC,IAAI,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBACxD,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,CAAC;YACD,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IAClC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IAEnB,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;SAAM,CAAC;QACP,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,OAAO,IAAI,CAAC;AACb,CAAC,CAAC"}
@@ -0,0 +1,2 @@
1
+ import { type Handle } from "hast-util-to-mdast";
2
+ export declare const customBrHandler: Handle;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"custom-br-handler.js","sourceRoot":"","sources":["../../../src/mdast-handlers/custom-br-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAElE,MAAM,CAAC,MAAM,eAAe,GAAW,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACtD,OAAO,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AACxC,CAAC,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"mdast-to-markdown.js","sourceRoot":"","sources":["../../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAqC,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEvF,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAY/C;;GAEG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAA2B;IACxE,UAAU,EAAE,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;IAC/C,MAAM,EAAE,GAAG;CACX,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAA4B,EAAE,OAAgC,EAAU,EAAE;IACzG,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB,EAAE,GAAG,EAAE,GAAG,iCAAiC,EAAE,GAAG,OAAO,EAAE,CAAC;IAE/F,IAAI,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAE9D,IAAI,OAAO,EAAE,CAAC;QACb,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}
1
+ {"version":3,"file":"mdast-to-markdown.js","sourceRoot":"","sources":["../../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAqC,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEvF,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAY/C;;GAEG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAA2B;IACxE,UAAU,EAAE,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;IAC/C,MAAM,EAAE,GAAG;CACX,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAA4B,EAAE,OAAgC,EAAU,EAAE;IACzG,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB,EAAE,GAAG,EAAE,GAAG,iCAAiC,EAAE,GAAG,OAAO,EAAE,CAAC;IAE/F,IAAI,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAEvF,IAAI,OAAO,EAAE,CAAC;QACb,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC"}
@@ -0,0 +1,4 @@
1
+ import type { Nodes as Hast } from "hast";
2
+ export declare const getLangFromHast: (node: Hast) => string | undefined;
3
+ export declare const getLangFromStr: (str: string) => string | undefined;
4
+ export declare const getUrlFromHast: (node: Hast) => string | undefined;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hast-utils.js","sourceRoot":"","sources":["../../../src/utils/hast-utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAErD,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAU,EAAE,EAAE;IAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClC,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtD,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAC7B,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO;IACR,CAAC;IACD,MAAM,OAAO,GAAG,IAAe,CAAC;IAChC,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO;IACR,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,QAAQ,EAAE,CAAC;QACd,OAAO,QAAkB,CAAC;IAC3B,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,EAAE;IAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACX,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAU,EAAsB,EAAE;IAChE,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,SAAS,CAAC;IAClB,CAAC;IAED,MAAM,iBAAiB,GAAG,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAClE,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1E,OAAO,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1F,CAAC;IACF,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC,CAAC"}
@@ -0,0 +1,46 @@
1
+ // src/utils/hast-utils.ts
2
+ import { select, selectAll } from "hast-util-select";
3
+ var getLangFromHast = (node) => {
4
+ const html = select("html", node);
5
+ if (html && typeof html.properties.lang === "string") {
6
+ return html.properties.lang;
7
+ }
8
+ if (node.type !== "element") {
9
+ return;
10
+ }
11
+ const element = node;
12
+ if (element.tagName !== "html") {
13
+ return;
14
+ }
15
+ const langAttr = element.properties.lang || element.properties["xml:lang"];
16
+ if (langAttr) {
17
+ return langAttr;
18
+ }
19
+ return void 0;
20
+ };
21
+ var getLangFromStr = (str) => {
22
+ const match = str.match(/lang=["']([^"']+)["']/);
23
+ if (match) {
24
+ return match[1];
25
+ }
26
+ return void 0;
27
+ };
28
+ var getUrlFromHast = (node) => {
29
+ if (node.type !== "element") {
30
+ return void 0;
31
+ }
32
+ const metaTagAttributes = ["og:url", "twitter:url"];
33
+ const metaTags = selectAll("meta", node);
34
+ for (const meta of metaTags) {
35
+ const property = meta.properties.property || meta.properties.name;
36
+ if (typeof property === "string" && metaTagAttributes.includes(property)) {
37
+ return typeof meta.properties.content === "string" ? meta.properties.content : void 0;
38
+ }
39
+ }
40
+ return void 0;
41
+ };
42
+ export {
43
+ getLangFromHast,
44
+ getLangFromStr,
45
+ getUrlFromHast
46
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "webforai",
3
- "version": "1.5.0",
3
+ "version": "1.6.0",
4
4
  "description": "A library that provides a web interface for AI",
5
5
  "author": "inaridiy",
6
6
  "license": "MIT",
@@ -79,17 +79,17 @@
79
79
  }
80
80
  },
81
81
  "peerDependencies": {
82
- "@cloudflare/puppeteer": "^0.0.6",
83
- "playwright": "^1.0.0",
84
- "puppeteer": "^22.10.0"
82
+ "@cloudflare/puppeteer": ">=0.0.6",
83
+ "playwright-core": ">=1.4",
84
+ "puppeteer": ">=22"
85
85
  },
86
86
  "peerDependenciesMeta": {
87
- "playwright": {
88
- "optional": true
89
- },
90
87
  "@cloudflare/puppeteer": {
91
88
  "optional": true
92
89
  },
90
+ "playwright-core": {
91
+ "optional": false
92
+ },
93
93
  "puppeteer": {
94
94
  "optional": true
95
95
  }
@@ -97,36 +97,37 @@
97
97
  "dependencies": {
98
98
  "@clack/prompts": "^0.7.0",
99
99
  "commander": "^12.1.0",
100
- "fastest-levenshtein": "^1.0.16",
101
- "hast-util-from-html": "^2.0.1",
102
- "hast-util-select": "^6.0.1",
103
- "hast-util-to-html": "^9.0.0",
100
+ "hast-util-from-html": "^2.0.3",
101
+ "hast-util-select": "^6.0.2",
102
+ "hast-util-to-html": "^9.0.3",
104
103
  "hast-util-to-mdast": "^10.1.0",
105
104
  "hast-util-to-string": "^3.0.0",
106
105
  "hast-util-to-text": "^4.0.0",
107
- "mathml-to-latex": "^1.3.0",
106
+ "mathml-to-latex": "^1.4.1",
108
107
  "mdast-util-gfm": "^3.0.0",
109
108
  "mdast-util-math": "^3.0.0",
110
109
  "mdast-util-to-markdown": "^2.1.0",
111
110
  "picocolors": "^1.0.1",
112
- "puppeteer": "^22.10.0",
113
111
  "trim-trailing-lines": "^2.1.0",
114
- "unist-util-filter": "^5.0.1"
112
+ "unist-util-filter": "^5.0.1",
113
+ "zx": "^8.1.5"
115
114
  },
116
115
  "devDependencies": {
117
116
  "@cloudflare/puppeteer": "^0.0.6",
118
117
  "@tsconfig/recommended": "^1.0.3",
118
+ "fastest-levenshtein": "^1.0.16",
119
119
  "@types/hast": "^3.0.2",
120
120
  "@types/mdast": "^4.0.2",
121
- "@types/node": "^20.11.0",
121
+ "@types/node": "^20.14.10",
122
122
  "arg": "^5.0.2",
123
123
  "esbuild": "^0.19.11",
124
124
  "glob": "^10.3.10",
125
125
  "ncp": "^2.0.0",
126
126
  "np": "^9.2.0",
127
- "playwright": "^1.40.1",
127
+ "playwright-core": "^1.40.1",
128
+ "puppeteer": "^23.2.2",
128
129
  "rimraf": "^5.0.5",
129
- "tsx": "^4.7.0",
130
+ "tsx": "^4.19.1",
130
131
  "typescript": "^5.4.5"
131
132
  },
132
133
  "scripts": {
package/dist/cjs/bin.js DELETED
@@ -1,254 +0,0 @@
1
- #!/usr/bin/env node
2
- "use strict";
3
- var __create = Object.create;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
6
- var __getOwnPropNames = Object.getOwnPropertyNames;
7
- var __getProtoOf = Object.getPrototypeOf;
8
- var __hasOwnProp = Object.prototype.hasOwnProperty;
9
- var __copyProps = (to, from, except, desc) => {
10
- if (from && typeof from === "object" || typeof from === "function") {
11
- for (let key of __getOwnPropNames(from))
12
- if (!__hasOwnProp.call(to, key) && key !== except)
13
- __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
14
- }
15
- return to;
16
- };
17
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
18
- // If the importer is in node compatibility mode or this is not an ESM
19
- // file that has been converted to a CommonJS file using a Babel-
20
- // compatible transform (i.e. "__esModule" has not been set), then set
21
- // "default" to the CommonJS "module.exports" for node compatibility.
22
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
23
- mod
24
- ));
25
- var import_node_fs = __toESM(require("node:fs"), 1);
26
- var import_node_path = __toESM(require("node:path"), 1);
27
- var import_prompts = require("@clack/prompts");
28
- var import_commander = require("commander");
29
- var import_picocolors = __toESM(require("picocolors"), 1);
30
- var import_package = __toESM(require("../package.json"), 1);
31
- var import_html_to_markdown = require("./html-to-markdown");
32
- var import_playwright = require("./loaders/playwright");
33
- var import_puppeteer = require("./loaders/puppeteer");
34
- var import_bin_utils = require("./utils/bin-utils");
35
- const MODES = ["default", "ai"];
36
- const LOADERS = ["fetch", "playwright", "puppeteer"];
37
- const loadersHint = {
38
- fetch: "Fetch HTML content from the given URL",
39
- playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
40
- puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
41
- };
42
- const aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
43
- const readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
44
- import_commander.program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(import_package.default.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`, MODES[0]).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`, LOADERS[0]).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
45
- process.env.DEBUG = "true";
46
- }).action(
47
- async (source, outputPath, options) => {
48
- let finalSource;
49
- let sourceIsUrl;
50
- let finalOutputPath;
51
- let finalMode;
52
- let finalLoader = "fetch";
53
- const isOutputFile = !options.stdout;
54
- if (process.env.DEBUG) {
55
- console.info("Arguments and options:");
56
- console.table({
57
- source,
58
- outputPath,
59
- ...options
60
- });
61
- console.info("isOutputFile:", isOutputFile);
62
- }
63
- try {
64
- finalSource = await (0, import_prompts.text)({
65
- message: "Enter the URL or html path to be converted to markdown:",
66
- placeholder: "https://example.com",
67
- initialValue: source,
68
- validate: (value) => {
69
- if (value.trim() === "") {
70
- return "Source is required";
71
- }
72
- if (!(0, import_bin_utils.isUrl)(value)) {
73
- if (!import_node_fs.default.existsSync(value)) {
74
- return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
75
- }
76
- if (import_node_fs.default.statSync(value).isDirectory()) {
77
- return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
78
- }
79
- }
80
- }
81
- });
82
- if ((0, import_prompts.isCancel)(finalSource)) {
83
- (0, import_prompts.cancel)("Canceled.");
84
- return;
85
- }
86
- sourceIsUrl = (0, import_bin_utils.isUrl)(finalSource);
87
- if (sourceIsUrl) {
88
- finalLoader = await (0, import_prompts.select)({
89
- message: "Select loader:",
90
- options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" })),
91
- initialValue: options.loader && LOADERS.includes(options.loader) ? options.loader : "fetch"
92
- });
93
- if ((0, import_prompts.isCancel)(finalLoader)) {
94
- (0, import_prompts.cancel)("Canceled.");
95
- return;
96
- }
97
- }
98
- if (isOutputFile) {
99
- finalOutputPath = await (0, import_prompts.text)({
100
- message: "Enter the output file path:",
101
- placeholder: "output.md",
102
- initialValue: outputPath ? outputPath : sourceIsUrl ? `${(0, import_bin_utils.urlToFilename)(finalSource)}.md` : (0, import_bin_utils.changeFileExtension)(finalSource, "md"),
103
- validate: (value) => {
104
- if (value.trim() === "") {
105
- return "Output path is required";
106
- }
107
- if (import_node_fs.default.existsSync(value) && import_node_fs.default.statSync(value).isDirectory()) {
108
- return "No directory can be specified.";
109
- }
110
- if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
111
- return "No directory can be specified.";
112
- }
113
- }
114
- });
115
- if ((0, import_prompts.isCancel)(finalOutputPath)) {
116
- (0, import_prompts.cancel)("Canceled.");
117
- return;
118
- }
119
- if (import_node_fs.default.existsSync(finalOutputPath)) {
120
- const isOutputFileOverwrite = await (0, import_prompts.confirm)({
121
- message: "The file already exists. Overwrite?",
122
- initialValue: false
123
- });
124
- if ((0, import_prompts.isCancel)(isOutputFileOverwrite)) {
125
- (0, import_prompts.cancel)("Canceled.");
126
- return;
127
- }
128
- if (isOutputFileOverwrite === false) {
129
- finalOutputPath = await (0, import_prompts.text)({
130
- message: "Enter the output file path:",
131
- placeholder: "output.md",
132
- initialValue: (0, import_bin_utils.getNextAvailableFilePath)(finalOutputPath),
133
- validate: (value) => {
134
- if (value.trim() === "") {
135
- return "Output path is required";
136
- }
137
- if (import_node_fs.default.existsSync(value)) {
138
- return "The file already exists";
139
- }
140
- if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
141
- return "No directory can be specified.";
142
- }
143
- }
144
- });
145
- if ((0, import_prompts.isCancel)(finalOutputPath)) {
146
- (0, import_prompts.cancel)("Canceled.");
147
- return;
148
- }
149
- }
150
- }
151
- } else {
152
- finalOutputPath = "";
153
- }
154
- finalMode = await (0, import_prompts.select)({
155
- message: "Select processing mode:",
156
- options: MODES.map((mode) => ({ value: mode, label: mode })),
157
- initialValue: options.mode && MODES.includes(options.mode) ? options.mode : "default"
158
- });
159
- if ((0, import_prompts.isCancel)(finalMode)) {
160
- (0, import_prompts.cancel)("Canceled.");
161
- return;
162
- }
163
- } catch (e) {
164
- const error = e;
165
- console.error(import_picocolors.default.red("An error occurred while processing the given value:"), error.message);
166
- if (process.env.DEBUG) {
167
- console.error(error.stack);
168
- }
169
- process.exit(1);
170
- }
171
- if (process.env.DEBUG) {
172
- console.info("Processing with the following options:");
173
- console.table({ finalSource, sourceIsUrl, isOutputFile, finalOutputPath, finalMode, finalLoader });
174
- }
175
- let content;
176
- const loadSpinner = (0, import_prompts.spinner)();
177
- try {
178
- if (sourceIsUrl) {
179
- loadSpinner.start("Downloading content.");
180
- if (finalLoader === "playwright") {
181
- content = await (0, import_playwright.loadHtml)(finalSource);
182
- } else if (finalLoader === "puppeteer") {
183
- content = await (0, import_puppeteer.loadHtml)(finalSource);
184
- } else {
185
- const response = await fetch(finalSource);
186
- content = await response.text();
187
- }
188
- loadSpinner.stop("Content download is complete!");
189
- } else {
190
- loadSpinner.start("Retrieving content.");
191
- if (!import_node_fs.default.existsSync(finalSource)) {
192
- console.error(import_picocolors.default.red("The file does not exist"));
193
- process.exit(1);
194
- }
195
- const stats = import_node_fs.default.statSync(finalSource);
196
- if (!stats.isFile()) {
197
- console.error(import_picocolors.default.red("The source must be a file"));
198
- process.exit(1);
199
- }
200
- content = import_node_fs.default.readFileSync(finalSource, "utf-8");
201
- loadSpinner.stop("Content retrieval is complete!");
202
- }
203
- } catch (e) {
204
- const error = e;
205
- loadSpinner.stop(import_picocolors.default.red("An error occurred while retrieving content"));
206
- console.error(error.message);
207
- if (process.env.DEBUG) {
208
- console.error(error.stack);
209
- }
210
- process.exit(1);
211
- }
212
- if (process.env.DEBUG) {
213
- console.info("Content:");
214
- console.info(content);
215
- }
216
- const convertSpinner = (0, import_prompts.spinner)();
217
- try {
218
- convertSpinner.start("Converting content to markdown.");
219
- const markdown = (0, import_html_to_markdown.htmlToMarkdown)(content, {
220
- ...sourceIsUrl || !!options.baseUrl && options.baseUrl.trim() !== "" ? { baseUrl: options.baseUrl || finalSource } : {},
221
- ...finalMode === "ai" ? aiModeOptions : readabilityModeOptions
222
- });
223
- if (process.env.DEBUG) {
224
- console.info("Markdown:");
225
- console.info(markdown);
226
- }
227
- if (isOutputFile) {
228
- const directory = import_node_path.default.dirname(finalOutputPath);
229
- if (!import_node_fs.default.existsSync(directory)) {
230
- import_node_fs.default.mkdirSync(directory, { recursive: true });
231
- }
232
- import_node_fs.default.writeFileSync(finalOutputPath, markdown);
233
- convertSpinner.stop("Markdown conversion is complete!");
234
- (0, import_prompts.outro)(`Converted successfully to markdown! Output saved to: ${import_node_path.default.resolve(finalOutputPath)} \u{1F389}`);
235
- } else {
236
- convertSpinner.stop("Markdown conversion is complete!");
237
- (0, import_prompts.outro)("Converted successfully to markdown\u{1F389}");
238
- console.log(`
239
-
240
- ${markdown}`);
241
- }
242
- } catch (e) {
243
- const error = e;
244
- convertSpinner.stop(import_picocolors.default.red("An error occurred while converting and saving the markdown"));
245
- console.error(error.message);
246
- if (process.env.DEBUG) {
247
- console.error(error.stack);
248
- }
249
- process.exit(1);
250
- }
251
- }
252
- );
253
- (0, import_prompts.intro)(import_picocolors.default.inverse(`webforai CLI version ${import_package.default.version}`));
254
- import_commander.program.parse();