webforai 0.0.6 → 0.0.8

package/README

@@ -0,0 +1,61 @@
+# Web for AI
+
+A library that provides a web interface for AI
+
+## Features
+
+- Ultra-precise HTML2Markdown conversion
+- Ultra-precise Markdown segmentation based on AST
+- Ultra-precise HTML retrieval functions using headless browsers
+- The core functionality is edge-native (runs on Cloudflare Workers!!!)
+
+## Demo
+
+There is a demo API for Html2Markdown deployed on CloudflareWorker. Please access the following link
+
+- [https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai](https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai)
+- [https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E](https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E)
+
+## Quick Start
+
+Just install and execute scripts
+
+```bash
+pnpm i webforai playwright
+```
+
+```js
+import { promises as fs } from "fs";
+import { htmlToMarkdown, htmlToMdast } from "webforai";
+import { loadHtml } from "webforai/loaders/playwright";
+
+const url = "https://www.npmjs.com/package/webforai";
+const html = await loadHtml(url);
+
+const markdown = htmlToMarkdown(html, { solveLinks: url });
+
+await fs.writeFile("output.md", markdown);
+```
+
+other examples are in [examples](./examples/src/index.ts)
+
+## Usage
+
+### Main Functions
+
+**`htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string`**
+Convert HTML to Markdown. By default, unnecessary HTML is excluded and processed.
+If `solveLinks` is specified, the relative links in the Mdast will be resolved.
+This function just calls htmlToMdast and mdastToMarkdown in that order internally.
+
+**`htmlToMdast(html: string, options?: HtmlToMdastOptions): Mdast`**
+This project uses Hast and Mdast as defined by syntax-tree internally.
+This function converts HTML to Mdast, an intermediate representation, which is required when using `mdastSplitter`, etc.
+
+**`mdastToMarkdown(mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string`**
+Convert Mdast to Markdown. If `solveLinks` is specified, the relative links in the Mdast will be resolved.
+
+### Loader Functions
+
+**`loadHtml(url: string, options?: LoadHtmlOptions): Promise<string>`**
+Load HTML from the specified URL. This function uses Playwright internally.

package/dist/mdast-to-markdown.d.ts

@@ -1,5 +1,5 @@
-import type { Nodes as Mdast } from "mdast";
-export declare const mdastToMarkdown: (mdast: Mdast, options?: {
+import type { Nodes as Mdast, RootContent } from "mdast";
+export declare const mdastToMarkdown: (mdast: Mdast | RootContent[], options?: {
     solveLinks?: string;
 }) => string;
 //# sourceMappingURL=mdast-to-markdown.d.ts.map

package/dist/mdast-to-markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mdast-to-markdown.d.ts","sourceRoot":"","sources":["../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,MAAM,OAAO,CAAC;AAQ5C,eAAO,MAAM,eAAe,UAAW,KAAK,YAAY;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,KAAG,MAQjF,CAAC"}
+{"version":3,"file":"mdast-to-markdown.d.ts","sourceRoot":"","sources":["../src/mdast-to-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AASzD,eAAO,MAAM,eAAe,UAAW,KAAK,GAAG,WAAW,EAAE,YAAY;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,KAAG,MAQjG,CAAC"}

package/dist/mdast-to-markdown.js

@@ -5,8 +5,9 @@ const mdast_util_gfm_1 = require("mdast-util-gfm");
 const mdast_util_math_1 = require("mdast-util-math");
 const mdast_util_to_markdown_1 = require("mdast-util-to-markdown");
 const link_replacer_1 = require("./link-replacer");
+const mdast_utils_1 = require("./utils/mdast-utils");
 const mdastToMarkdown = (mdast, options) => {
-    let markdown = (0, mdast_util_to_markdown_1.toMarkdown)(mdast, {
+    let markdown = (0, mdast_util_to_markdown_1.toMarkdown)((0, mdast_utils_1.warpRoot)(mdast), {
         extensions: [(0, mdast_util_gfm_1.gfmToMarkdown)(), (0, mdast_util_math_1.mathToMarkdown)()],
     });
     if (options?.solveLinks)
@@ -14,4 +15,4 @@ const mdastToMarkdown = (mdast, options) => {
     return markdown;
 };
 exports.mdastToMarkdown = mdastToMarkdown;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdG8tbWFya2Rvd24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWRhc3QtdG8tbWFya2Rvd24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBRUEsbURBQStDO0FBQy9DLHFEQUFpRDtBQUNqRCxtRUFBb0Q7QUFFcEQsbURBQStDO0FBRXhDLE1BQU0sZUFBZSxHQUFHLENBQUMsS0FBWSxFQUFFLE9BQWlDLEVBQVUsRUFBRTtJQUMxRixJQUFJLFFBQVEsR0FBRyxJQUFBLG1DQUFVLEVBQUMsS0FBSyxFQUFFO1FBQ2hDLFVBQVUsRUFBRSxDQUFDLElBQUEsOEJBQWEsR0FBRSxFQUFFLElBQUEsZ0NBQWMsR0FBRSxDQUFDO0tBQy9DLENBQUMsQ0FBQztJQUVILElBQUksT0FBTyxFQUFFLFVBQVU7UUFBRSxRQUFRLEdBQUcsSUFBQSw0QkFBWSxFQUFDLFFBQVEsRUFBRSxPQUFPLENBQUMsVUFBVSxDQUFDLENBQUM7SUFFL0UsT0FBTyxRQUFRLENBQUM7QUFDakIsQ0FBQyxDQUFDO0FBUlcsUUFBQSxlQUFlLG1CQVExQiJ9
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWRhc3QtdG8tbWFya2Rvd24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbWRhc3QtdG8tbWFya2Rvd24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBRUEsbURBQStDO0FBQy9DLHFEQUFpRDtBQUNqRCxtRUFBb0Q7QUFFcEQsbURBQStDO0FBQy9DLHFEQUErQztBQUV4QyxNQUFNLGVBQWUsR0FBRyxDQUFDLEtBQTRCLEVBQUUsT0FBaUMsRUFBVSxFQUFFO0lBQzFHLElBQUksUUFBUSxHQUFHLElBQUEsbUNBQVUsRUFBQyxJQUFBLHNCQUFRLEVBQUMsS0FBSyxDQUFDLEVBQUU7UUFDMUMsVUFBVSxFQUFFLENBQUMsSUFBQSw4QkFBYSxHQUFFLEVBQUUsSUFBQSxnQ0FBYyxHQUFFLENBQUM7S0FDL0MsQ0FBQyxDQUFDO0lBRUgsSUFBSSxPQUFPLEVBQUUsVUFBVTtRQUFFLFFBQVEsR0FBRyxJQUFBLDRCQUFZLEVBQUMsUUFBUSxFQUFFLE9BQU8sQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUUvRSxPQUFPLFFBQVEsQ0FBQztBQUNqQixDQUFDLENBQUM7QUFSVyxRQUFBLGVBQWUsbUJBUTFCIn0=

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webforai",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "description": "",
   "repository": {
     "type": "git",

package/src/mdast-to-markdown.ts

@@ -1,13 +1,14 @@
-import type { Nodes as Mdast } from "mdast";
+import type { Nodes as Mdast, RootContent } from "mdast";
 
 import { gfmToMarkdown } from "mdast-util-gfm";
 import { mathToMarkdown } from "mdast-util-math";
 import { toMarkdown } from "mdast-util-to-markdown";
 
 import { linkReplacer } from "./link-replacer";
+import { warpRoot } from "./utils/mdast-utils";
 
-export const mdastToMarkdown = (mdast: Mdast, options?: { solveLinks?: string }): string => {
-	let markdown = toMarkdown(mdast, {
+export const mdastToMarkdown = (mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string => {
+	let markdown = toMarkdown(warpRoot(mdast), {
 		extensions: [gfmToMarkdown(), mathToMarkdown()],
 	});