webforai 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -0
- package/package.json +1 -1
package/README.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Web for AI
|
|
2
|
+
|
|
3
|
+
A library that provides a web interface for AI
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Ultra-precise HTML2Markdown conversion
|
|
8
|
+
- Ultra-precise Markdown segmentation based on AST
|
|
9
|
+
- Ultra-precise HTML retrieval functions using headless browsers
|
|
10
|
+
- The core functionality is edge-native (runs on Cloudflare Workers!!!)
|
|
11
|
+
|
|
12
|
+
## Demo
|
|
13
|
+
|
|
14
|
+
There is a demo API for Html2Markdown deployed on CloudflareWorker. Please access the following link
|
|
15
|
+
|
|
16
|
+
- [https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai](https://site-reader-api.inaridiy.workers.dev/?url=https://www.npmjs.com/package/webforai)
|
|
17
|
+
- [https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E](https://site-reader-api.inaridiy.workers.dev/?html=%3Ch1%3EHello%20World%3C/h1%3E)
|
|
18
|
+
|
|
19
|
+
## Quick Start
|
|
20
|
+
|
|
21
|
+
Just install and execute scripts
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pnpm i webforai playwright
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```js
|
|
28
|
+
import { promises as fs } from "fs";
|
|
29
|
+
import { htmlToMarkdown, htmlToMdast } from "webforai";
|
|
30
|
+
import { loadHtml } from "webforai/loaders/playwright";
|
|
31
|
+
|
|
32
|
+
const url = "https://www.npmjs.com/package/webforai";
|
|
33
|
+
const html = await loadHtml(url);
|
|
34
|
+
|
|
35
|
+
const markdown = htmlToMarkdown(html, { solveLinks: url });
|
|
36
|
+
|
|
37
|
+
await fs.writeFile("output.md", markdown);
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
other examples are in [examples](./examples/src/index.ts)
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
### Main Functions
|
|
45
|
+
|
|
46
|
+
**`htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions): string`**
|
|
47
|
+
Convert HTML to Markdown. By default, unnecessary HTML is excluded and processed.
|
|
48
|
+
If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
49
|
+
This function just calls htmlToMdast and mdastToMarkdown in that order internally.
|
|
50
|
+
|
|
51
|
+
**`htmlToMdast(html: string, options?: HtmlToMdastOptions): Mdast`**
|
|
52
|
+
This project uses Hast and Mdast as defined by syntax-tree internally.
|
|
53
|
+
This function converts HTML to Mdast, an intermediate representation, which is required when using `mdastSplitter`, etc.
|
|
54
|
+
|
|
55
|
+
**`mdastToMarkdown(mdast: Mdast | RootContent[], options?: { solveLinks?: string }): string`**
|
|
56
|
+
Convert Mdast to Markdown. If `solveLinks` is specified, the relative links in the Mdast will be resolved.
|
|
57
|
+
|
|
58
|
+
### Loader Functions
|
|
59
|
+
|
|
60
|
+
**`loadHtml(url: string, options?: LoadHtmlOptions): Promise<string>`**
|
|
61
|
+
Load HTML from the specified URL. This function uses Playwright internally.
|