webforai 0.0.20 → 0.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/cjs/extract-hast.js +1 -0
  2. package/dist/cjs/extract-hast.js.map +7 -0
  3. package/dist/cjs/html-to-markdown.js +1 -0
  4. package/dist/cjs/html-to-markdown.js.map +7 -0
  5. package/dist/cjs/html-to-mdast.js +1 -0
  6. package/dist/cjs/html-to-mdast.js.map +7 -0
  7. package/dist/cjs/index.js +1 -0
  8. package/dist/cjs/index.js.map +7 -0
  9. package/dist/cjs/link-replacer.js +1 -0
  10. package/dist/cjs/link-replacer.js.map +7 -0
  11. package/dist/cjs/loaders/playwright.js +1 -0
  12. package/dist/cjs/loaders/playwright.js.map +7 -0
  13. package/dist/cjs/md-splitter.js +1 -0
  14. package/dist/cjs/md-splitter.js.map +7 -0
  15. package/dist/cjs/mdast-handlers/custom-div-handler.js +1 -0
  16. package/dist/cjs/mdast-handlers/custom-div-handler.js.map +7 -0
  17. package/dist/cjs/mdast-handlers/math-handler.js +1 -0
  18. package/dist/cjs/mdast-handlers/math-handler.js.map +7 -0
  19. package/dist/cjs/mdast-to-markdown.js +1 -0
  20. package/dist/cjs/mdast-to-markdown.js.map +7 -0
  21. package/dist/cjs/utils/common.js +1 -0
  22. package/dist/cjs/utils/common.js.map +7 -0
  23. package/dist/cjs/utils/mdast-utils.js +1 -0
  24. package/dist/cjs/utils/mdast-utils.js.map +7 -0
  25. package/dist/extract-hast.js +1 -0
  26. package/dist/extract-hast.js.map +7 -0
  27. package/dist/html-to-markdown.js +1 -0
  28. package/dist/html-to-markdown.js.map +7 -0
  29. package/dist/html-to-mdast.js +1 -0
  30. package/dist/html-to-mdast.js.map +7 -0
  31. package/dist/index.js +1 -0
  32. package/dist/index.js.map +7 -0
  33. package/dist/link-replacer.js +1 -0
  34. package/dist/link-replacer.js.map +7 -0
  35. package/dist/loaders/playwright.js +1 -0
  36. package/dist/loaders/playwright.js.map +7 -0
  37. package/dist/md-splitter.js +1 -0
  38. package/dist/md-splitter.js.map +7 -0
  39. package/dist/mdast-handlers/custom-div-handler.js +1 -0
  40. package/dist/mdast-handlers/custom-div-handler.js.map +7 -0
  41. package/dist/mdast-handlers/math-handler.js +1 -0
  42. package/dist/mdast-handlers/math-handler.js.map +7 -0
  43. package/dist/mdast-to-markdown.js +1 -0
  44. package/dist/mdast-to-markdown.js.map +7 -0
  45. package/dist/types/loaders/playwright.d.ts +2 -1
  46. package/dist/utils/common.js +1 -0
  47. package/dist/utils/common.js.map +7 -0
  48. package/dist/utils/mdast-utils.js +1 -0
  49. package/dist/utils/mdast-utils.js.map +7 -0
  50. package/package.json +1 -1
@@ -43,3 +43,4 @@ const extractHast = (hast) => {
43
43
  0 && (module.exports = {
44
44
  extractHast
45
45
  });
46
+ //# sourceMappingURL=extract-hast.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/extract-hast.ts"],
4
+ "sourcesContent": ["import type { Root as Hast } from \"hast\";\r\nimport { select, selectAll } from \"hast-util-select\";\r\n\r\nimport { remove } from \"unist-util-remove\";\r\n\r\nexport const extractHast = (hast: Hast) => {\r\n\tconst removeSelectors = [\"header\", \"footer\", \"nav\", \"speechify-ignore\"];\r\n\r\n\tfor (const selector of removeSelectors) {\r\n\t\tconst matchedNodes = selectAll(selector, hast);\r\n\t\tfor (const node of matchedNodes) {\r\n\t\t\tremove(hast, { type: node.type, tagName: node.tagName });\r\n\t\t}\r\n\t}\r\n\r\n\tconst selectors = [\"article\", \"div#article\", \"main\", \"div.container\"];\r\n\tfor (const selector of selectors) {\r\n\t\tconst element = select(selector, hast);\r\n\t\tif (element) return element;\r\n\t}\r\n\r\n\treturn hast;\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,8BAAkC;AAElC,+BAAuB;AAEhB,MAAM,cAAc,CAAC,SAAe;AAC1C,QAAM,kBAAkB,CAAC,UAAU,UAAU,OAAO,kBAAkB;AAEtE,aAAW,YAAY,iBAAiB;AACvC,UAAM,mBAAe,mCAAU,UAAU,IAAI;AAC7C,eAAW,QAAQ,cAAc;AAChC,2CAAO,MAAM,EAAE,MAAM,KAAK,MAAM,SAAS,KAAK,QAAQ,CAAC;AAAA,IACxD;AAAA,EACD;AAEA,QAAM,YAAY,CAAC,WAAW,eAAe,QAAQ,eAAe;AACpE,aAAW,YAAY,WAAW;AACjC,UAAM,cAAU,gCAAO,UAAU,IAAI;AACrC,QAAI;AAAS,aAAO;AAAA,EACrB;AAEA,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -32,3 +32,4 @@ const htmlToMarkdown = (html, options) => {
32
32
  0 && (module.exports = {
33
33
  htmlToMarkdown
34
34
  });
35
+ //# sourceMappingURL=html-to-markdown.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/html-to-markdown.ts"],
4
+ "sourcesContent": ["import type { Root as Hast } from \"hast\";\r\nimport { htmlToMdast } from \"./html-to-mdast\";\r\nimport { mdastToMarkdown } from \"./mdast-to-markdown\";\r\n\r\nexport interface HtmlToMarkdownOptions {\r\n\textractHast?: false | ((hast: Hast) => Hast);\r\n\tsolveLinks?: string;\r\n}\r\n\r\nexport const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {\r\n\tconst mdast = htmlToMdast(html, options);\r\n\tconst markdown = mdastToMarkdown(mdast, options);\r\n\treturn markdown;\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,2BAA4B;AAC5B,+BAAgC;AAOzB,MAAM,iBAAiB,CAAC,MAAc,YAA4C;AACxF,QAAM,YAAQ,kCAAY,MAAM,OAAO;AACvC,QAAM,eAAW,0CAAgB,OAAO,OAAO;AAC/C,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -39,3 +39,4 @@ const htmlToMdast = (html, options) => {
39
39
  0 && (module.exports = {
40
40
  htmlToMdast
41
41
  });
42
+ //# sourceMappingURL=html-to-mdast.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/html-to-mdast.ts"],
4
+ "sourcesContent": ["import type { Root as Hast } from \"hast\";\r\nimport type { Nodes as Mdast } from \"mdast\";\r\n\r\nimport { fromHtml } from \"hast-util-from-html\";\r\nimport { toMdast } from \"hast-util-to-mdast\";\r\n\r\nimport { extractHast } from \"./extract-hast\";\r\nimport { customDivHandler } from \"./mdast-handlers/custom-div-handler\";\r\nimport { mathHandler } from \"./mdast-handlers/math-handler\";\r\n\r\nexport type HtmlToMdastOptions = {\r\n\textractHast?: false | ((hast: Hast) => Hast);\r\n};\r\n\r\nexport const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {\r\n\tconst hast = fromHtml(html, { fragment: true });\r\n\tconst extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);\r\n\tconst mdast = toMdast(extractedHast, {\r\n\t\thandlers: { div: customDivHandler, math: mathHandler, br: () => {} },\r\n\t});\r\n\r\n\treturn mdast;\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,iCAAyB;AACzB,gCAAwB;AAExB,0BAA4B;AAC5B,gCAAiC;AACjC,0BAA4B;AAMrB,MAAM,cAAc,CAAC,MAAc,YAAwC;AACjF,QAAM,WAAO,qCAAS,MAAM,EAAE,UAAU,KAAK,CAAC;AAC9C,QAAM,gBAAgB,SAAS,cAAc,QAAQ,YAAY,IAAI,QAAI,iCAAY,IAAI;AACzF,QAAM,YAAQ,mCAAQ,eAAe;AAAA,IACpC,UAAU,EAAE,KAAK,4CAAkB,MAAM,iCAAa,IAAI,MAAM;AAAA,IAAC,EAAE;AAAA,EACpE,CAAC;AAED,SAAO;AACR;",
6
+ "names": []
7
+ }
package/dist/cjs/index.js CHANGED
@@ -26,3 +26,4 @@ __reExport(src_exports, require("./mdast-to-markdown"), module.exports);
26
26
  ...require("./html-to-mdast"),
27
27
  ...require("./mdast-to-markdown")
28
28
  });
29
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/index.ts"],
4
+ "sourcesContent": ["export * from \"./html-to-markdown\";\r\nexport * from \"./md-splitter\";\r\nexport * from \"./html-to-mdast\";\r\nexport * from \"./mdast-to-markdown\";\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;AAAA;AAAA;AAAA,wBAAc,+BAAd;AACA,wBAAc,0BADd;AAEA,wBAAc,4BAFd;AAGA,wBAAc,gCAHd;",
6
+ "names": []
7
+ }
@@ -33,3 +33,4 @@ const linkReplacer = (markdown, link) => {
33
33
  0 && (module.exports = {
34
34
  linkReplacer
35
35
  });
36
+ //# sourceMappingURL=link-replacer.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/link-replacer.ts"],
4
+ "sourcesContent": ["export const linkReplacer = (markdown: string, link: string) => {\r\n\tconst regex = /(!?\\[.*?\\]\\()(.+?)(\\))/g;\r\n\treturn markdown.replace(regex, (match, pre, thisUrl, post) => {\r\n\t\tif (thisUrl.startsWith(\"http\") || thisUrl.startsWith(\"#\")) return match;\r\n\t\treturn `${pre}${new URL(thisUrl, link).href}${post}`;\r\n\t});\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAO,MAAM,eAAe,CAAC,UAAkB,SAAiB;AAC/D,QAAM,QAAQ;AACd,SAAO,SAAS,QAAQ,OAAO,CAAC,OAAO,KAAK,SAAS,SAAS;AAC7D,QAAI,QAAQ,WAAW,MAAM,KAAK,QAAQ,WAAW,GAAG;AAAG,aAAO;AAClE,WAAO,GAAG,GAAG,GAAG,IAAI,IAAI,SAAS,IAAI,EAAE,IAAI,GAAG,IAAI;AAAA,EACnD,CAAC;AACF;",
6
+ "names": []
7
+ }
@@ -55,3 +55,4 @@ const loadHtml = async (url, context) => {
55
55
  0 && (module.exports = {
56
56
  loadHtml
57
57
  });
58
+ //# sourceMappingURL=playwright.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../../src/loaders/playwright.ts"],
4
+ "sourcesContent": ["import * as playwright from \"playwright\";\r\n\r\nconst sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));\r\n\r\nexport const loadHtml = async (url: string, context?: playwright.BrowserContext) => {\r\n\tlet _browser: playwright.Browser | null = null;\r\n\tlet _context: playwright.BrowserContext;\r\n\tif (!context) {\r\n\t\t_browser = await playwright.chromium.launch();\r\n\t\t_context = await _browser.newContext();\r\n\t} else {\r\n\t\t_context = context;\r\n\t}\r\n\r\n\tconst page = await _context.newPage();\r\n\tawait page.goto(url);\r\n\tawait Promise.race([page.waitForLoadState(\"networkidle\"), sleep(10000)]);\r\n\tconst html = await page.content();\r\n\tawait page.close();\r\n\tif (_browser) await _browser.close();\r\n\treturn html;\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAA4B;AAE5B,MAAM,QAAQ,CAAC,OAAe,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAEvE,MAAM,WAAW,OAAO,KAAa,YAAwC;AACnF,MAAI,WAAsC;AAC1C,MAAI;AACJ,MAAI,CAAC,SAAS;AACb,eAAW,MAAM,WAAW,SAAS,OAAO;AAC5C,eAAW,MAAM,SAAS,WAAW;AAAA,EACtC,OAAO;AACN,eAAW;AAAA,EACZ;AAEA,QAAM,OAAO,MAAM,SAAS,QAAQ;AACpC,QAAM,KAAK,KAAK,GAAG;AACnB,QAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,aAAa,GAAG,MAAM,GAAK,CAAC,CAAC;AACvE,QAAM,OAAO,MAAM,KAAK,QAAQ;AAChC,QAAM,KAAK,MAAM;AACjB,MAAI;AAAU,UAAM,SAAS,MAAM;AACnC,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -52,3 +52,4 @@ const mdastSplitter = async (mdast, checker, options) => {
52
52
  0 && (module.exports = {
53
53
  mdastSplitter
54
54
  });
55
+ //# sourceMappingURL=md-splitter.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/md-splitter.ts"],
4
+ "sourcesContent": ["import type { Nodes as Mdast, RootContent } from \"mdast\";\r\nimport { mdastToMarkdown } from \"./mdast-to-markdown\";\r\nimport { chunk } from \"./utils/common\";\r\nimport { internalType, unwarpRoot, warpRoot } from \"./utils/mdast-utils\";\r\n\r\nconst PRIORITY_SPLITTERS = [\"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\", \"list\", \"table\", \"code\"];\r\ntype SplitterGenerator = Generator<string, void, unknown>;\r\nconst getSplitterGenerator = function* () {\r\n\tfor (const splitter of PRIORITY_SPLITTERS) yield splitter;\r\n};\r\n\r\nconst _mdastSplitter = async (\r\n\tcontents: RootContent[],\r\n\tchecker: (markdown: string) => Promise<boolean>,\r\n\tsplitterGenerator: SplitterGenerator,\r\n): Promise<RootContent[][]> => {\r\n\tconst markdown = mdastToMarkdown(warpRoot(contents));\r\n\tif ((await checker(markdown)) || 3 > contents.length) return [contents];\r\n\tconst splitter = splitterGenerator.next().value;\r\n\r\n\tconst chunked = splitter\r\n\t\t? contents.reduce<RootContent[][]>((acc, content) => {\r\n\t\t\t\tif (internalType(content) === splitter || acc.length === 0) {\r\n\t\t\t\t\tacc.push([content]);\r\n\t\t\t\t\treturn acc;\r\n\t\t\t\t}\r\n\t\t\t\tacc[acc.length - 1].push(content);\r\n\t\t\t\treturn acc;\r\n\t\t }, [])\r\n\t\t: chunk(contents, 3);\r\n\r\n\tconst splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));\r\n\r\n\treturn Promise.all(splitting).then((chunks) => chunks.flat());\r\n};\r\n\r\nexport const mdastSplitter = async (\r\n\tmdast: Mdast,\r\n\tchecker: (markdown: string) => Promise<boolean>,\r\n\toptions?: { signal?: AbortSignal }, //TODO\r\n): Promise<RootContent[][]> => {\r\n\treturn _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,+BAAgC;AAChC,oBAAsB;AACtB,yBAAmD;AAEnD,MAAM,qBAAqB,CAAC,MAAM,MAAM,MAAM,MAAM,MAAM,MAAM,QAAQ,SAAS,MAAM;AAEvF,MAAM,uBAAuB,aAAa;AACzC,aAAW,YAAY;AAAoB,UAAM;AAClD;AAEA,MAAM,iBAAiB,OACtB,UACA,SACA,sBAC8B;AAC9B,QAAM,eAAW,8CAAgB,6BAAS,QAAQ,CAAC;AACnD,MAAK,MAAM,QAAQ,QAAQ,KAAM,IAAI,SAAS;AAAQ,WAAO,CAAC,QAAQ;AACtE,QAAM,WAAW,kBAAkB,KAAK,EAAE;AAE1C,QAAM,UAAU,WACb,SAAS,OAAwB,CAAC,KAAK,YAAY;AACnD,YAAI,iCAAa,OAAO,MAAM,YAAY,IAAI,WAAW,GAAG;AAC3D,UAAI,KAAK,CAAC,OAAO,CAAC;AAClB,aAAO;AAAA,IACR;AACA,QAAI,IAAI,SAAS,CAAC,EAAE,KAAK,OAAO;AAChC,WAAO;AAAA,EACP,GAAG,CAAC,CAAC,QACL,qBAAM,UAAU,CAAC;AAEpB,QAAM,YAAY,QAAQ,IAAI,CAACA,WAAU,eAAeA,QAAO,SAAS,iBAAiB,CAAC;AAE1F,SAAO,QAAQ,IAAI,SAAS,EAAE,KAAK,CAAC,WAAW,OAAO,KAAK,CAAC;AAC7D;AAEO,MAAM,gBAAgB,OAC5B,OACA,SACA,YAC8B;AAC9B,SAAO,mBAAe,+BAAW,KAAK,GAAG,SAAS,qBAAqB,CAAC;AACzE;",
6
+ "names": ["chunk"]
7
+ }
@@ -63,3 +63,4 @@ const customDivHandler = (state, node) => {
63
63
  0 && (module.exports = {
64
64
  customDivHandler
65
65
  });
66
+ //# sourceMappingURL=custom-div-handler.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../../src/mdast-handlers/custom-div-handler.ts"],
4
+ "sourcesContent": ["import { type Handle, defaultHandlers } from \"hast-util-to-mdast\";\r\nimport { toText } from \"hast-util-to-text\";\r\nimport type { Code } from \"mdast\";\r\nimport { trimTrailingLines } from \"trim-trailing-lines\";\r\n\r\nexport const customDivHandler: Handle = (state, node) => {\r\n\tconst classNames = Array.isArray(node.properties.className) ? node.properties.className : [];\r\n\r\n\tlet highlightLanguage: string | undefined;\r\n\tfor (const className of classNames) {\r\n\t\tif (String(className) === \"highlight-text-html-basic\") {\r\n\t\t\thighlightLanguage = \"html\";\r\n\t\t\tbreak;\r\n\t\t} else if (String(className).includes(\"highlight-source-\")) {\r\n\t\t\thighlightLanguage = String(className).slice(\"highlight-source-\".length);\r\n\t\t\tbreak;\r\n\t\t} else if (String(className).includes(\"language-\")) {\r\n\t\t\thighlightLanguage = String(className).slice(\"language-\".length);\r\n\t\t\tbreak;\r\n\t\t} else if (String(className).includes(\"code-block_wrapper\")) {\r\n\t\t\tconst text = toText(node);\r\n\t\t\tif (!text.includes(\".\")) break;\r\n\t\t\tconst lang = toText(node).split(\"\\n\")[0].split(\".\").slice(-1)[0];\r\n\t\t\thighlightLanguage = lang;\r\n\t\t\tbreak;\r\n\t\t}\r\n\t}\r\n\r\n\tif (highlightLanguage !== undefined) {\r\n\t\tconst code = node.children.find((child) => child.type === \"element\" && child.tagName === \"pre\") || node;\r\n\r\n\t\tconst result: Code = {\r\n\t\t\ttype: \"code\",\r\n\t\t\tlang: highlightLanguage,\r\n\t\t\tmeta: null,\r\n\t\t\tvalue: trimTrailingLines(toText(code)).trim(),\r\n\t\t};\r\n\t\tstate.patch(node, result);\r\n\t\treturn result;\r\n\t}\r\n\r\n\treturn defaultHandlers.div(state, node);\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gCAA6C;AAC7C,+BAAuB;AAEvB,iCAAkC;AAE3B,MAAM,mBAA2B,CAAC,OAAO,SAAS;AACxD,QAAM,aAAa,MAAM,QAAQ,KAAK,WAAW,SAAS,IAAI,KAAK,WAAW,YAAY,CAAC;AAE3F,MAAI;AACJ,aAAW,aAAa,YAAY;AACnC,QAAI,OAAO,SAAS,MAAM,6BAA6B;AACtD,0BAAoB;AACpB;AAAA,IACD,WAAW,OAAO,SAAS,EAAE,SAAS,mBAAmB,GAAG;AAC3D,0BAAoB,OAAO,SAAS,EAAE,MAAM,oBAAoB,MAAM;AACtE;AAAA,IACD,WAAW,OAAO,SAAS,EAAE,SAAS,WAAW,GAAG;AACnD,0BAAoB,OAAO,SAAS,EAAE,MAAM,YAAY,MAAM;AAC9D;AAAA,IACD,WAAW,OAAO,SAAS,EAAE,SAAS,oBAAoB,GAAG;AAC5D,YAAM,WAAO,iCAAO,IAAI;AACxB,UAAI,CAAC,KAAK,SAAS,GAAG;AAAG;AACzB,YAAM,WAAO,iCAAO,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,MAAM,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC;AAC/D,0BAAoB;AACpB;AAAA,IACD;AAAA,EACD;AAEA,MAAI,sBAAsB,QAAW;AACpC,UAAM,OAAO,KAAK,SAAS,KAAK,CAAC,UAAU,MAAM,SAAS,aAAa,MAAM,YAAY,KAAK,KAAK;AAEnG,UAAM,SAAe;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAO,kDAAkB,iCAAO,IAAI,CAAC,EAAE,KAAK;AAAA,IAC7C;AACA,UAAM,MAAM,MAAM,MAAM;AACxB,WAAO;AAAA,EACR;AAEA,SAAO,0CAAgB,IAAI,OAAO,IAAI;AACvC;",
6
+ "names": []
7
+ }
@@ -34,3 +34,4 @@ const mathHandler = (state, node) => {
34
34
  0 && (module.exports = {
35
35
  mathHandler
36
36
  });
37
+ //# sourceMappingURL=math-handler.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../../src/mdast-handlers/math-handler.ts"],
4
+ "sourcesContent": ["import { toHtml } from \"hast-util-to-html\";\r\nimport type { Handle } from \"hast-util-to-mdast\";\r\nimport { MathMLToLaTeX } from \"mathml-to-latex\";\r\nimport type { InlineMath, Math as MDMath } from \"mdast-util-math\";\r\n\r\nexport const mathHandler: Handle = (state, node) => {\r\n\tconst mathMl = toHtml(node);\r\n\tconst latex = MathMLToLaTeX.convert(mathMl);\r\n\tconst result: InlineMath | MDMath = { type: \"inlineMath\", value: latex };\r\n\tstate.patch(node, result);\r\n\treturn result;\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,+BAAuB;AAEvB,6BAA8B;AAGvB,MAAM,cAAsB,CAAC,OAAO,SAAS;AACnD,QAAM,aAAS,iCAAO,IAAI;AAC1B,QAAM,QAAQ,qCAAc,QAAQ,MAAM;AAC1C,QAAM,SAA8B,EAAE,MAAM,cAAc,OAAO,MAAM;AACvE,QAAM,MAAM,MAAM,MAAM;AACxB,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -38,3 +38,4 @@ const mdastToMarkdown = (mdast, options) => {
38
38
  0 && (module.exports = {
39
39
  mdastToMarkdown
40
40
  });
41
+ //# sourceMappingURL=mdast-to-markdown.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/mdast-to-markdown.ts"],
4
+ "sourcesContent": ["import type { Nodes as Mdast, RootContent } from \"mdast\";\r\n\r\nimport { gfmToMarkdown } from \"mdast-util-gfm\";\r\nimport { mathToMarkdown } from \"mdast-util-math\";\r\nimport { toMarkdown } from \"mdast-util-to-markdown\";\r\n\r\nimport { linkReplacer } from \"./link-replacer\";\r\nimport { warpRoot } from \"./utils/mdast-utils\";\r\n\r\nexport const mdastToMarkdown = (mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string => {\r\n\tlet markdown = toMarkdown(warpRoot(mdast), {\r\n\t\textensions: [gfmToMarkdown(), mathToMarkdown()],\r\n\t});\r\n\r\n\tif (options?.solveLinks) markdown = linkReplacer(markdown, options.solveLinks);\r\n\r\n\treturn markdown;\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAEA,4BAA8B;AAC9B,6BAA+B;AAC/B,oCAA2B;AAE3B,2BAA6B;AAC7B,yBAAyB;AAElB,MAAM,kBAAkB,CAAC,OAA8B,YAA8C;AAC3G,MAAI,eAAW,8CAAW,6BAAS,KAAK,GAAG;AAAA,IAC1C,YAAY,KAAC,qCAAc,OAAG,uCAAe,CAAC;AAAA,EAC/C,CAAC;AAED,MAAI,SAAS;AAAY,mBAAW,mCAAa,UAAU,QAAQ,UAAU;AAE7E,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -34,3 +34,4 @@ const chunk = (array, size) => {
34
34
  0 && (module.exports = {
35
35
  chunk
36
36
  });
37
+ //# sourceMappingURL=common.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../../src/utils/common.ts"],
4
+ "sourcesContent": ["export const chunk = <T>(array: T[], size: number): T[][] => {\r\n\treturn array.reduce<T[][]>((acc, _, index) => {\r\n\t\tif (index % size === 0) {\r\n\t\t\tacc.push(array.slice(index, index + size));\r\n\t\t\treturn acc;\r\n\t\t}\r\n\t\treturn acc;\r\n\t}, []);\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAO,MAAM,QAAQ,CAAI,OAAY,SAAwB;AAC5D,SAAO,MAAM,OAAc,CAAC,KAAK,GAAG,UAAU;AAC7C,QAAI,QAAQ,SAAS,GAAG;AACvB,UAAI,KAAK,MAAM,MAAM,OAAO,QAAQ,IAAI,CAAC;AACzC,aAAO;AAAA,IACR;AACA,WAAO;AAAA,EACR,GAAG,CAAC,CAAC;AACN;",
6
+ "names": []
7
+ }
@@ -44,3 +44,4 @@ const internalType = (content) => {
44
44
  unwarpRoot,
45
45
  warpRoot
46
46
  });
47
+ //# sourceMappingURL=mdast-utils.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../../src/utils/mdast-utils.ts"],
4
+ "sourcesContent": ["import type { Nodes as Mdast, RootContent } from \"mdast\";\r\n\r\nexport const unwarpRoot = (mdast: Mdast): RootContent[] => {\r\n\tif (mdast.type === \"root\") return mdast.children;\r\n\treturn [mdast];\r\n};\r\n\r\nexport const warpRoot = (mdast: RootContent[] | Mdast): Mdast => {\r\n\tif (Array.isArray(mdast)) return { type: \"root\", children: mdast };\r\n\treturn mdast;\r\n};\r\n\r\nexport const internalType = (content: RootContent): string => {\r\n\tif (content.type === \"heading\") return `h${content.depth}`;\r\n\treturn content.type;\r\n};\r\n"],
5
+ "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAEO,MAAM,aAAa,CAAC,UAAgC;AAC1D,MAAI,MAAM,SAAS;AAAQ,WAAO,MAAM;AACxC,SAAO,CAAC,KAAK;AACd;AAEO,MAAM,WAAW,CAAC,UAAwC;AAChE,MAAI,MAAM,QAAQ,KAAK;AAAG,WAAO,EAAE,MAAM,QAAQ,UAAU,MAAM;AACjE,SAAO;AACR;AAEO,MAAM,eAAe,CAAC,YAAiC;AAC7D,MAAI,QAAQ,SAAS;AAAW,WAAO,IAAI,QAAQ,KAAK;AACxD,SAAO,QAAQ;AAChB;",
6
+ "names": []
7
+ }
@@ -20,3 +20,4 @@ var extractHast = (hast) => {
20
20
  export {
21
21
  extractHast
22
22
  };
23
+ //# sourceMappingURL=extract-hast.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/extract-hast.ts"],
4
+ "sourcesContent": ["import type { Root as Hast } from \"hast\";\r\nimport { select, selectAll } from \"hast-util-select\";\r\n\r\nimport { remove } from \"unist-util-remove\";\r\n\r\nexport const extractHast = (hast: Hast) => {\r\n\tconst removeSelectors = [\"header\", \"footer\", \"nav\", \"speechify-ignore\"];\r\n\r\n\tfor (const selector of removeSelectors) {\r\n\t\tconst matchedNodes = selectAll(selector, hast);\r\n\t\tfor (const node of matchedNodes) {\r\n\t\t\tremove(hast, { type: node.type, tagName: node.tagName });\r\n\t\t}\r\n\t}\r\n\r\n\tconst selectors = [\"article\", \"div#article\", \"main\", \"div.container\"];\r\n\tfor (const selector of selectors) {\r\n\t\tconst element = select(selector, hast);\r\n\t\tif (element) return element;\r\n\t}\r\n\r\n\treturn hast;\r\n};\r\n"],
5
+ "mappings": ";AACA,SAAS,QAAQ,iBAAiB;AAElC,SAAS,cAAc;AAEhB,IAAM,cAAc,CAAC,SAAe;AAC1C,QAAM,kBAAkB,CAAC,UAAU,UAAU,OAAO,kBAAkB;AAEtE,aAAW,YAAY,iBAAiB;AACvC,UAAM,eAAe,UAAU,UAAU,IAAI;AAC7C,eAAW,QAAQ,cAAc;AAChC,aAAO,MAAM,EAAE,MAAM,KAAK,MAAM,SAAS,KAAK,QAAQ,CAAC;AAAA,IACxD;AAAA,EACD;AAEA,QAAM,YAAY,CAAC,WAAW,eAAe,QAAQ,eAAe;AACpE,aAAW,YAAY,WAAW;AACjC,UAAM,UAAU,OAAO,UAAU,IAAI;AACrC,QAAI;AAAS,aAAO;AAAA,EACrB;AAEA,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -9,3 +9,4 @@ var htmlToMarkdown = (html, options) => {
9
9
  export {
10
10
  htmlToMarkdown
11
11
  };
12
+ //# sourceMappingURL=html-to-markdown.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/html-to-markdown.ts"],
4
+ "sourcesContent": ["import type { Root as Hast } from \"hast\";\r\nimport { htmlToMdast } from \"./html-to-mdast\";\r\nimport { mdastToMarkdown } from \"./mdast-to-markdown\";\r\n\r\nexport interface HtmlToMarkdownOptions {\r\n\textractHast?: false | ((hast: Hast) => Hast);\r\n\tsolveLinks?: string;\r\n}\r\n\r\nexport const htmlToMarkdown = (html: string, options?: HtmlToMarkdownOptions): string => {\r\n\tconst mdast = htmlToMdast(html, options);\r\n\tconst markdown = mdastToMarkdown(mdast, options);\r\n\treturn markdown;\r\n};\r\n"],
5
+ "mappings": ";AACA,SAAS,mBAAmB;AAC5B,SAAS,uBAAuB;AAOzB,IAAM,iBAAiB,CAAC,MAAc,YAA4C;AACxF,QAAM,QAAQ,YAAY,MAAM,OAAO;AACvC,QAAM,WAAW,gBAAgB,OAAO,OAAO;AAC/C,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -16,3 +16,4 @@ var htmlToMdast = (html, options) => {
16
16
  export {
17
17
  htmlToMdast
18
18
  };
19
+ //# sourceMappingURL=html-to-mdast.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/html-to-mdast.ts"],
4
+ "sourcesContent": ["import type { Root as Hast } from \"hast\";\r\nimport type { Nodes as Mdast } from \"mdast\";\r\n\r\nimport { fromHtml } from \"hast-util-from-html\";\r\nimport { toMdast } from \"hast-util-to-mdast\";\r\n\r\nimport { extractHast } from \"./extract-hast\";\r\nimport { customDivHandler } from \"./mdast-handlers/custom-div-handler\";\r\nimport { mathHandler } from \"./mdast-handlers/math-handler\";\r\n\r\nexport type HtmlToMdastOptions = {\r\n\textractHast?: false | ((hast: Hast) => Hast);\r\n};\r\n\r\nexport const htmlToMdast = (html: string, options?: HtmlToMdastOptions): Mdast => {\r\n\tconst hast = fromHtml(html, { fragment: true });\r\n\tconst extractedHast = options?.extractHast ? options.extractHast(hast) : extractHast(hast);\r\n\tconst mdast = toMdast(extractedHast, {\r\n\t\thandlers: { div: customDivHandler, math: mathHandler, br: () => {} },\r\n\t});\r\n\r\n\treturn mdast;\r\n};\r\n"],
5
+ "mappings": ";AAGA,SAAS,gBAAgB;AACzB,SAAS,eAAe;AAExB,SAAS,mBAAmB;AAC5B,SAAS,wBAAwB;AACjC,SAAS,mBAAmB;AAMrB,IAAM,cAAc,CAAC,MAAc,YAAwC;AACjF,QAAM,OAAO,SAAS,MAAM,EAAE,UAAU,KAAK,CAAC;AAC9C,QAAM,gBAAgB,SAAS,cAAc,QAAQ,YAAY,IAAI,IAAI,YAAY,IAAI;AACzF,QAAM,QAAQ,QAAQ,eAAe;AAAA,IACpC,UAAU,EAAE,KAAK,kBAAkB,MAAM,aAAa,IAAI,MAAM;AAAA,IAAC,EAAE;AAAA,EACpE,CAAC;AAED,SAAO;AACR;",
6
+ "names": []
7
+ }
package/dist/index.js CHANGED
@@ -3,3 +3,4 @@ export * from "./html-to-markdown.js";
3
3
  export * from "./md-splitter.js";
4
4
  export * from "./html-to-mdast.js";
5
5
  export * from "./mdast-to-markdown.js";
6
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/index.ts"],
4
+ "sourcesContent": ["export * from \"./html-to-markdown\";\r\nexport * from \"./md-splitter\";\r\nexport * from \"./html-to-mdast\";\r\nexport * from \"./mdast-to-markdown\";\r\n"],
5
+ "mappings": ";AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;",
6
+ "names": []
7
+ }
@@ -10,3 +10,4 @@ var linkReplacer = (markdown, link) => {
10
10
  export {
11
11
  linkReplacer
12
12
  };
13
+ //# sourceMappingURL=link-replacer.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/link-replacer.ts"],
4
+ "sourcesContent": ["export const linkReplacer = (markdown: string, link: string) => {\r\n\tconst regex = /(!?\\[.*?\\]\\()(.+?)(\\))/g;\r\n\treturn markdown.replace(regex, (match, pre, thisUrl, post) => {\r\n\t\tif (thisUrl.startsWith(\"http\") || thisUrl.startsWith(\"#\")) return match;\r\n\t\treturn `${pre}${new URL(thisUrl, link).href}${post}`;\r\n\t});\r\n};\r\n"],
5
+ "mappings": ";AAAO,IAAM,eAAe,CAAC,UAAkB,SAAiB;AAC/D,QAAM,QAAQ;AACd,SAAO,SAAS,QAAQ,OAAO,CAAC,OAAO,KAAK,SAAS,SAAS;AAC7D,QAAI,QAAQ,WAAW,MAAM,KAAK,QAAQ,WAAW,GAAG;AAAG,aAAO;AAClE,WAAO,GAAG,GAAG,GAAG,IAAI,IAAI,SAAS,IAAI,EAAE,IAAI,GAAG,IAAI;AAAA,EACnD,CAAC;AACF;",
6
+ "names": []
7
+ }
@@ -22,3 +22,4 @@ var loadHtml = async (url, context) => {
22
22
  export {
23
23
  loadHtml
24
24
  };
25
+ //# sourceMappingURL=playwright.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/loaders/playwright.ts"],
4
+ "sourcesContent": ["import * as playwright from \"playwright\";\r\n\r\nconst sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));\r\n\r\nexport const loadHtml = async (url: string, context?: playwright.BrowserContext) => {\r\n\tlet _browser: playwright.Browser | null = null;\r\n\tlet _context: playwright.BrowserContext;\r\n\tif (!context) {\r\n\t\t_browser = await playwright.chromium.launch();\r\n\t\t_context = await _browser.newContext();\r\n\t} else {\r\n\t\t_context = context;\r\n\t}\r\n\r\n\tconst page = await _context.newPage();\r\n\tawait page.goto(url);\r\n\tawait Promise.race([page.waitForLoadState(\"networkidle\"), sleep(10000)]);\r\n\tconst html = await page.content();\r\n\tawait page.close();\r\n\tif (_browser) await _browser.close();\r\n\treturn html;\r\n};\r\n"],
5
+ "mappings": ";AAAA,YAAY,gBAAgB;AAE5B,IAAM,QAAQ,CAAC,OAAe,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AAEvE,IAAM,WAAW,OAAO,KAAa,YAAwC;AACnF,MAAI,WAAsC;AAC1C,MAAI;AACJ,MAAI,CAAC,SAAS;AACb,eAAW,MAAiB,oBAAS,OAAO;AAC5C,eAAW,MAAM,SAAS,WAAW;AAAA,EACtC,OAAO;AACN,eAAW;AAAA,EACZ;AAEA,QAAM,OAAO,MAAM,SAAS,QAAQ;AACpC,QAAM,KAAK,KAAK,GAAG;AACnB,QAAM,QAAQ,KAAK,CAAC,KAAK,iBAAiB,aAAa,GAAG,MAAM,GAAK,CAAC,CAAC;AACvE,QAAM,OAAO,MAAM,KAAK,QAAQ;AAChC,QAAM,KAAK,MAAM;AACjB,MAAI;AAAU,UAAM,SAAS,MAAM;AACnC,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -29,3 +29,4 @@ var mdastSplitter = async (mdast, checker, options) => {
29
29
  export {
30
30
  mdastSplitter
31
31
  };
32
+ //# sourceMappingURL=md-splitter.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/md-splitter.ts"],
4
+ "sourcesContent": ["import type { Nodes as Mdast, RootContent } from \"mdast\";\r\nimport { mdastToMarkdown } from \"./mdast-to-markdown\";\r\nimport { chunk } from \"./utils/common\";\r\nimport { internalType, unwarpRoot, warpRoot } from \"./utils/mdast-utils\";\r\n\r\nconst PRIORITY_SPLITTERS = [\"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\", \"list\", \"table\", \"code\"];\r\ntype SplitterGenerator = Generator<string, void, unknown>;\r\nconst getSplitterGenerator = function* () {\r\n\tfor (const splitter of PRIORITY_SPLITTERS) yield splitter;\r\n};\r\n\r\nconst _mdastSplitter = async (\r\n\tcontents: RootContent[],\r\n\tchecker: (markdown: string) => Promise<boolean>,\r\n\tsplitterGenerator: SplitterGenerator,\r\n): Promise<RootContent[][]> => {\r\n\tconst markdown = mdastToMarkdown(warpRoot(contents));\r\n\tif ((await checker(markdown)) || 3 > contents.length) return [contents];\r\n\tconst splitter = splitterGenerator.next().value;\r\n\r\n\tconst chunked = splitter\r\n\t\t? contents.reduce<RootContent[][]>((acc, content) => {\r\n\t\t\t\tif (internalType(content) === splitter || acc.length === 0) {\r\n\t\t\t\t\tacc.push([content]);\r\n\t\t\t\t\treturn acc;\r\n\t\t\t\t}\r\n\t\t\t\tacc[acc.length - 1].push(content);\r\n\t\t\t\treturn acc;\r\n\t\t }, [])\r\n\t\t: chunk(contents, 3);\r\n\r\n\tconst splitting = chunked.map((chunk) => _mdastSplitter(chunk, checker, splitterGenerator));\r\n\r\n\treturn Promise.all(splitting).then((chunks) => chunks.flat());\r\n};\r\n\r\nexport const mdastSplitter = async (\r\n\tmdast: Mdast,\r\n\tchecker: (markdown: string) => Promise<boolean>,\r\n\toptions?: { signal?: AbortSignal }, //TODO\r\n): Promise<RootContent[][]> => {\r\n\treturn _mdastSplitter(unwarpRoot(mdast), checker, getSplitterGenerator());\r\n};\r\n"],
5
+ "mappings": ";AACA,SAAS,uBAAuB;AAChC,SAAS,aAAa;AACtB,SAAS,cAAc,YAAY,gBAAgB;AAEnD,IAAM,qBAAqB,CAAC,MAAM,MAAM,MAAM,MAAM,MAAM,MAAM,QAAQ,SAAS,MAAM;AAEvF,IAAM,uBAAuB,aAAa;AACzC,aAAW,YAAY;AAAoB,UAAM;AAClD;AAEA,IAAM,iBAAiB,OACtB,UACA,SACA,sBAC8B;AAC9B,QAAM,WAAW,gBAAgB,SAAS,QAAQ,CAAC;AACnD,MAAK,MAAM,QAAQ,QAAQ,KAAM,IAAI,SAAS;AAAQ,WAAO,CAAC,QAAQ;AACtE,QAAM,WAAW,kBAAkB,KAAK,EAAE;AAE1C,QAAM,UAAU,WACb,SAAS,OAAwB,CAAC,KAAK,YAAY;AACnD,QAAI,aAAa,OAAO,MAAM,YAAY,IAAI,WAAW,GAAG;AAC3D,UAAI,KAAK,CAAC,OAAO,CAAC;AAClB,aAAO;AAAA,IACR;AACA,QAAI,IAAI,SAAS,CAAC,EAAE,KAAK,OAAO;AAChC,WAAO;AAAA,EACP,GAAG,CAAC,CAAC,IACL,MAAM,UAAU,CAAC;AAEpB,QAAM,YAAY,QAAQ,IAAI,CAACA,WAAU,eAAeA,QAAO,SAAS,iBAAiB,CAAC;AAE1F,SAAO,QAAQ,IAAI,SAAS,EAAE,KAAK,CAAC,WAAW,OAAO,KAAK,CAAC;AAC7D;AAEO,IAAM,gBAAgB,OAC5B,OACA,SACA,YAC8B;AAC9B,SAAO,eAAe,WAAW,KAAK,GAAG,SAAS,qBAAqB,CAAC;AACzE;",
6
+ "names": ["chunk"]
7
+ }
@@ -40,3 +40,4 @@ var customDivHandler = (state, node) => {
40
40
  export {
41
41
  customDivHandler
42
42
  };
43
+ //# sourceMappingURL=custom-div-handler.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/mdast-handlers/custom-div-handler.ts"],
4
+ "sourcesContent": ["import { type Handle, defaultHandlers } from \"hast-util-to-mdast\";\r\nimport { toText } from \"hast-util-to-text\";\r\nimport type { Code } from \"mdast\";\r\nimport { trimTrailingLines } from \"trim-trailing-lines\";\r\n\r\nexport const customDivHandler: Handle = (state, node) => {\r\n\tconst classNames = Array.isArray(node.properties.className) ? node.properties.className : [];\r\n\r\n\tlet highlightLanguage: string | undefined;\r\n\tfor (const className of classNames) {\r\n\t\tif (String(className) === \"highlight-text-html-basic\") {\r\n\t\t\thighlightLanguage = \"html\";\r\n\t\t\tbreak;\r\n\t\t} else if (String(className).includes(\"highlight-source-\")) {\r\n\t\t\thighlightLanguage = String(className).slice(\"highlight-source-\".length);\r\n\t\t\tbreak;\r\n\t\t} else if (String(className).includes(\"language-\")) {\r\n\t\t\thighlightLanguage = String(className).slice(\"language-\".length);\r\n\t\t\tbreak;\r\n\t\t} else if (String(className).includes(\"code-block_wrapper\")) {\r\n\t\t\tconst text = toText(node);\r\n\t\t\tif (!text.includes(\".\")) break;\r\n\t\t\tconst lang = toText(node).split(\"\\n\")[0].split(\".\").slice(-1)[0];\r\n\t\t\thighlightLanguage = lang;\r\n\t\t\tbreak;\r\n\t\t}\r\n\t}\r\n\r\n\tif (highlightLanguage !== undefined) {\r\n\t\tconst code = node.children.find((child) => child.type === \"element\" && child.tagName === \"pre\") || node;\r\n\r\n\t\tconst result: Code = {\r\n\t\t\ttype: \"code\",\r\n\t\t\tlang: highlightLanguage,\r\n\t\t\tmeta: null,\r\n\t\t\tvalue: trimTrailingLines(toText(code)).trim(),\r\n\t\t};\r\n\t\tstate.patch(node, result);\r\n\t\treturn result;\r\n\t}\r\n\r\n\treturn defaultHandlers.div(state, node);\r\n};\r\n"],
5
+ "mappings": ";AAAA,SAAsB,uBAAuB;AAC7C,SAAS,cAAc;AAEvB,SAAS,yBAAyB;AAE3B,IAAM,mBAA2B,CAAC,OAAO,SAAS;AACxD,QAAM,aAAa,MAAM,QAAQ,KAAK,WAAW,SAAS,IAAI,KAAK,WAAW,YAAY,CAAC;AAE3F,MAAI;AACJ,aAAW,aAAa,YAAY;AACnC,QAAI,OAAO,SAAS,MAAM,6BAA6B;AACtD,0BAAoB;AACpB;AAAA,IACD,WAAW,OAAO,SAAS,EAAE,SAAS,mBAAmB,GAAG;AAC3D,0BAAoB,OAAO,SAAS,EAAE,MAAM,oBAAoB,MAAM;AACtE;AAAA,IACD,WAAW,OAAO,SAAS,EAAE,SAAS,WAAW,GAAG;AACnD,0BAAoB,OAAO,SAAS,EAAE,MAAM,YAAY,MAAM;AAC9D;AAAA,IACD,WAAW,OAAO,SAAS,EAAE,SAAS,oBAAoB,GAAG;AAC5D,YAAM,OAAO,OAAO,IAAI;AACxB,UAAI,CAAC,KAAK,SAAS,GAAG;AAAG;AACzB,YAAM,OAAO,OAAO,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,MAAM,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC;AAC/D,0BAAoB;AACpB;AAAA,IACD;AAAA,EACD;AAEA,MAAI,sBAAsB,QAAW;AACpC,UAAM,OAAO,KAAK,SAAS,KAAK,CAAC,UAAU,MAAM,SAAS,aAAa,MAAM,YAAY,KAAK,KAAK;AAEnG,UAAM,SAAe;AAAA,MACpB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,MAAM;AAAA,MACN,OAAO,kBAAkB,OAAO,IAAI,CAAC,EAAE,KAAK;AAAA,IAC7C;AACA,UAAM,MAAM,MAAM,MAAM;AACxB,WAAO;AAAA,EACR;AAEA,SAAO,gBAAgB,IAAI,OAAO,IAAI;AACvC;",
6
+ "names": []
7
+ }
@@ -11,3 +11,4 @@ var mathHandler = (state, node) => {
11
11
  export {
12
12
  mathHandler
13
13
  };
14
+ //# sourceMappingURL=math-handler.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/mdast-handlers/math-handler.ts"],
4
+ "sourcesContent": ["import { toHtml } from \"hast-util-to-html\";\r\nimport type { Handle } from \"hast-util-to-mdast\";\r\nimport { MathMLToLaTeX } from \"mathml-to-latex\";\r\nimport type { InlineMath, Math as MDMath } from \"mdast-util-math\";\r\n\r\nexport const mathHandler: Handle = (state, node) => {\r\n\tconst mathMl = toHtml(node);\r\n\tconst latex = MathMLToLaTeX.convert(mathMl);\r\n\tconst result: InlineMath | MDMath = { type: \"inlineMath\", value: latex };\r\n\tstate.patch(node, result);\r\n\treturn result;\r\n};\r\n"],
5
+ "mappings": ";AAAA,SAAS,cAAc;AAEvB,SAAS,qBAAqB;AAGvB,IAAM,cAAsB,CAAC,OAAO,SAAS;AACnD,QAAM,SAAS,OAAO,IAAI;AAC1B,QAAM,QAAQ,cAAc,QAAQ,MAAM;AAC1C,QAAM,SAA8B,EAAE,MAAM,cAAc,OAAO,MAAM;AACvE,QAAM,MAAM,MAAM,MAAM;AACxB,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -15,3 +15,4 @@ var mdastToMarkdown = (mdast, options) => {
15
15
  export {
16
16
  mdastToMarkdown
17
17
  };
18
+ //# sourceMappingURL=mdast-to-markdown.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/mdast-to-markdown.ts"],
4
+ "sourcesContent": ["import type { Nodes as Mdast, RootContent } from \"mdast\";\r\n\r\nimport { gfmToMarkdown } from \"mdast-util-gfm\";\r\nimport { mathToMarkdown } from \"mdast-util-math\";\r\nimport { toMarkdown } from \"mdast-util-to-markdown\";\r\n\r\nimport { linkReplacer } from \"./link-replacer\";\r\nimport { warpRoot } from \"./utils/mdast-utils\";\r\n\r\nexport const mdastToMarkdown = (mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string => {\r\n\tlet markdown = toMarkdown(warpRoot(mdast), {\r\n\t\textensions: [gfmToMarkdown(), mathToMarkdown()],\r\n\t});\r\n\r\n\tif (options?.solveLinks) markdown = linkReplacer(markdown, options.solveLinks);\r\n\r\n\treturn markdown;\r\n};\r\n"],
5
+ "mappings": ";AAEA,SAAS,qBAAqB;AAC9B,SAAS,sBAAsB;AAC/B,SAAS,kBAAkB;AAE3B,SAAS,oBAAoB;AAC7B,SAAS,gBAAgB;AAElB,IAAM,kBAAkB,CAAC,OAA8B,YAA8C;AAC3G,MAAI,WAAW,WAAW,SAAS,KAAK,GAAG;AAAA,IAC1C,YAAY,CAAC,cAAc,GAAG,eAAe,CAAC;AAAA,EAC/C,CAAC;AAED,MAAI,SAAS;AAAY,eAAW,aAAa,UAAU,QAAQ,UAAU;AAE7E,SAAO;AACR;",
6
+ "names": []
7
+ }
@@ -1 +1,2 @@
1
- export declare const loadHtml: (url: string, context?: any) => Promise<any>;
1
+ import * as playwright from "playwright";
2
+ export declare const loadHtml: (url: string, context?: playwright.BrowserContext) => Promise<string>;
@@ -11,3 +11,4 @@ var chunk = (array, size) => {
11
11
  export {
12
12
  chunk
13
13
  };
14
+ //# sourceMappingURL=common.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/utils/common.ts"],
4
+ "sourcesContent": ["export const chunk = <T>(array: T[], size: number): T[][] => {\r\n\treturn array.reduce<T[][]>((acc, _, index) => {\r\n\t\tif (index % size === 0) {\r\n\t\t\tacc.push(array.slice(index, index + size));\r\n\t\t\treturn acc;\r\n\t\t}\r\n\t\treturn acc;\r\n\t}, []);\r\n};\r\n"],
5
+ "mappings": ";AAAO,IAAM,QAAQ,CAAI,OAAY,SAAwB;AAC5D,SAAO,MAAM,OAAc,CAAC,KAAK,GAAG,UAAU;AAC7C,QAAI,QAAQ,SAAS,GAAG;AACvB,UAAI,KAAK,MAAM,MAAM,OAAO,QAAQ,IAAI,CAAC;AACzC,aAAO;AAAA,IACR;AACA,WAAO;AAAA,EACR,GAAG,CAAC,CAAC;AACN;",
6
+ "names": []
7
+ }
@@ -19,3 +19,4 @@ export {
19
19
  unwarpRoot,
20
20
  warpRoot
21
21
  };
22
+ //# sourceMappingURL=mdast-utils.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/utils/mdast-utils.ts"],
4
+ "sourcesContent": ["import type { Nodes as Mdast, RootContent } from \"mdast\";\r\n\r\nexport const unwarpRoot = (mdast: Mdast): RootContent[] => {\r\n\tif (mdast.type === \"root\") return mdast.children;\r\n\treturn [mdast];\r\n};\r\n\r\nexport const warpRoot = (mdast: RootContent[] | Mdast): Mdast => {\r\n\tif (Array.isArray(mdast)) return { type: \"root\", children: mdast };\r\n\treturn mdast;\r\n};\r\n\r\nexport const internalType = (content: RootContent): string => {\r\n\tif (content.type === \"heading\") return `h${content.depth}`;\r\n\treturn content.type;\r\n};\r\n"],
5
+ "mappings": ";AAEO,IAAM,aAAa,CAAC,UAAgC;AAC1D,MAAI,MAAM,SAAS;AAAQ,WAAO,MAAM;AACxC,SAAO,CAAC,KAAK;AACd;AAEO,IAAM,WAAW,CAAC,UAAwC;AAChE,MAAI,MAAM,QAAQ,KAAK;AAAG,WAAO,EAAE,MAAM,QAAQ,UAAU,MAAM;AACjE,SAAO;AACR;AAEO,IAAM,eAAe,CAAC,YAAiC;AAC7D,MAAI,QAAQ,SAAS;AAAW,WAAO,IAAI,QAAQ,KAAK;AACxD,SAAO,QAAQ;AAChB;",
6
+ "names": []
7
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "webforai",
3
- "version": "0.0.20",
3
+ "version": "0.0.21",
4
4
  "description": "A library that provides a web interface for AI",
5
5
  "author": "inaridiy",
6
6
  "license": "MIT",