webforai 2.1.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -4
- package/dist/adapters/index.js +58 -0
- package/dist/adapters/registry.js +149 -0
- package/dist/adapters/sites/declarative.js +178 -0
- package/dist/adapters/sites/hackernews.js +94 -0
- package/dist/adapters/sites/youtube.js +165 -0
- package/dist/adapters/types.js +0 -0
- package/dist/bin.js +4142 -549
- package/dist/cjs/adapters/index.js +86 -0
- package/dist/cjs/adapters/registry.js +178 -0
- package/dist/cjs/adapters/sites/declarative.js +202 -0
- package/dist/cjs/adapters/sites/hackernews.js +117 -0
- package/dist/cjs/adapters/sites/youtube.js +191 -0
- package/dist/cjs/adapters/types.js +16 -0
- package/dist/cjs/constants.js +2 -2
- package/dist/cjs/detect-client-shell.js +78 -0
- package/dist/cjs/extract-meta-refresh.js +71 -0
- package/dist/cjs/extractors/index.js +5 -0
- package/dist/cjs/extractors/lib/constants.js +186 -0
- package/dist/cjs/extractors/lib/sanitize.js +429 -0
- package/dist/cjs/extractors/lib/score.js +264 -0
- package/dist/cjs/extractors/lib/terminators.js +131 -0
- package/dist/cjs/extractors/lib/ui-chrome.js +119 -0
- package/dist/cjs/extractors/pipeExtractors.js +12 -9
- package/dist/cjs/extractors/presets/auto.js +44 -0
- package/dist/cjs/extractors/presets/minimal-filter.js +27 -48
- package/dist/cjs/extractors/presets/takumi.js +84 -122
- package/dist/cjs/html-to-markdown.js +44 -7
- package/dist/cjs/html-to-mdast.js +10 -1
- package/dist/cjs/index.js +43 -2
- package/dist/cjs/link-replacer.js +19 -13
- package/dist/cjs/loaders/cf-puppeteer.js +14 -7
- package/dist/cjs/loaders/fetch.js +14 -2
- package/dist/cjs/loaders/playwright.js +32 -30
- package/dist/cjs/loaders/puppeteer.js +14 -7
- package/dist/cjs/md-splitter.js +24 -11
- package/dist/cjs/mdast-handlers/custom-a-handler.js +25 -9
- package/dist/cjs/mdast-handlers/custom-code-handler.js +18 -7
- package/dist/cjs/mdast-handlers/custom-div-handler.js +12 -33
- package/dist/cjs/mdast-handlers/custom-img-handler.js +23 -1
- package/dist/cjs/mdast-handlers/custom-table-handler.js +16 -0
- package/dist/cjs/mdast-handlers/definition-list-handler.js +110 -0
- package/dist/cjs/mdast-handlers/math-handler.js +12 -3
- package/dist/cjs/mdast-handlers/sup-sub-handler.js +39 -0
- package/dist/cjs/mdast-to-markdown.js +2 -5
- package/dist/cjs/metadata/index.js +223 -0
- package/dist/cjs/normalize/index.js +177 -0
- package/dist/cjs/platform/client.js +204 -0
- package/dist/cjs/platform/error.js +36 -0
- package/dist/cjs/platform/fetch.js +37 -0
- package/dist/cjs/platform/index.js +44 -0
- package/dist/cjs/platform/types.js +40 -0
- package/dist/cjs/utils/hast-fast.js +284 -0
- package/dist/constants.js +2 -2
- package/dist/detect-client-shell.js +53 -0
- package/dist/extract-meta-refresh.js +47 -0
- package/dist/extractors/index.js +3 -0
- package/dist/extractors/lib/constants.js +150 -0
- package/dist/extractors/lib/sanitize.js +419 -0
- package/dist/extractors/lib/score.js +242 -0
- package/dist/extractors/lib/terminators.js +107 -0
- package/dist/extractors/lib/ui-chrome.js +96 -0
- package/dist/extractors/pipeExtractors.js +12 -9
- package/dist/extractors/presets/auto.js +20 -0
- package/dist/extractors/presets/minimal-filter.js +35 -47
- package/dist/extractors/presets/takumi.js +91 -122
- package/dist/html-to-markdown.js +42 -6
- package/dist/html-to-mdast.js +10 -1
- package/dist/index.js +45 -2
- package/dist/link-replacer.js +19 -13
- package/dist/loaders/cf-puppeteer.js +15 -8
- package/dist/loaders/fetch.js +14 -2
- package/dist/loaders/playwright.js +32 -30
- package/dist/loaders/puppeteer.js +15 -8
- package/dist/md-splitter.js +24 -11
- package/dist/mdast-handlers/custom-a-handler.js +25 -9
- package/dist/mdast-handlers/custom-code-handler.js +17 -7
- package/dist/mdast-handlers/custom-div-handler.js +12 -33
- package/dist/mdast-handlers/custom-img-handler.js +23 -1
- package/dist/mdast-handlers/custom-table-handler.js +16 -0
- package/dist/mdast-handlers/definition-list-handler.js +87 -0
- package/dist/mdast-handlers/math-handler.js +12 -3
- package/dist/mdast-handlers/sup-sub-handler.js +16 -0
- package/dist/mdast-to-markdown.js +2 -5
- package/dist/metadata/index.js +198 -0
- package/dist/normalize/index.js +159 -0
- package/dist/platform/client.js +182 -0
- package/dist/platform/error.js +13 -0
- package/dist/platform/fetch.js +14 -0
- package/dist/platform/index.js +23 -0
- package/dist/platform/types.js +13 -0
- package/dist/types/adapters/index.d.ts +24 -0
- package/dist/types/adapters/index.js.map +1 -0
- package/dist/types/adapters/registry.d.ts +35 -0
- package/dist/types/adapters/registry.js.map +1 -0
- package/dist/types/adapters/sites/declarative.d.ts +13 -0
- package/dist/types/adapters/sites/declarative.js.map +1 -0
- package/dist/types/adapters/sites/hackernews.d.ts +13 -0
- package/dist/types/adapters/sites/hackernews.js.map +1 -0
- package/dist/types/adapters/sites/youtube.d.ts +31 -0
- package/dist/types/adapters/sites/youtube.js.map +1 -0
- package/dist/types/adapters/types.d.ts +99 -0
- package/dist/types/adapters/types.js.map +1 -0
- package/dist/types/cli/bin.js.map +1 -1
- package/dist/types/cli/commands/jobs/files.d.ts +28 -0
- package/dist/types/cli/commands/jobs/files.js.map +1 -0
- package/dist/types/cli/commands/jobs/llms-txt.d.ts +33 -0
- package/dist/types/cli/commands/jobs/llms-txt.js.map +1 -0
- package/dist/types/cli/commands/jobs/options.d.ts +58 -0
- package/dist/types/cli/commands/jobs/options.js.map +1 -0
- package/dist/types/cli/commands/jobs/run.d.ts +44 -0
- package/dist/types/cli/commands/jobs/run.js.map +1 -0
- package/dist/types/cli/commands/skill/content.d.ts +9 -0
- package/dist/types/cli/commands/skill/content.js.map +1 -0
- package/dist/types/cli/commands/skill/index.d.ts +19 -0
- package/dist/types/cli/commands/skill/index.js.map +1 -0
- package/dist/types/cli/commands/webforai/interactive.d.ts +7 -0
- package/dist/types/cli/commands/webforai/interactive.js.map +1 -0
- package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -1
- package/dist/types/cli/commands/webforai/options.d.ts +62 -0
- package/dist/types/cli/commands/webforai/options.js.map +1 -0
- package/dist/types/cli/commands/webforai/run.d.ts +10 -0
- package/dist/types/cli/commands/webforai/run.js.map +1 -0
- package/dist/types/cli/constants.d.ts +14 -4
- package/dist/types/cli/constants.js.map +1 -1
- package/dist/types/cli/helpers/selectExtractMode.js.map +1 -1
- package/dist/types/cli/helpers/selectLoader.js.map +1 -1
- package/dist/types/cli/utils.d.ts +8 -1
- package/dist/types/cli/utils.js.map +1 -1
- package/dist/types/constants.d.ts +7 -1
- package/dist/types/constants.js.map +1 -1
- package/dist/types/detect-client-shell.d.ts +24 -0
- package/dist/types/detect-client-shell.js.map +1 -0
- package/dist/types/extract-meta-refresh.d.ts +29 -0
- package/dist/types/extract-meta-refresh.js.map +1 -0
- package/dist/types/extractors/index.d.ts +1 -0
- package/dist/types/extractors/index.js.map +1 -1
- package/dist/types/extractors/lib/constants.d.ts +81 -0
- package/dist/types/extractors/lib/constants.js.map +1 -0
- package/dist/types/extractors/lib/sanitize.d.ts +66 -0
- package/dist/types/extractors/lib/sanitize.js.map +1 -0
- package/dist/types/extractors/lib/score.d.ts +85 -0
- package/dist/types/extractors/lib/score.js.map +1 -0
- package/dist/types/extractors/lib/terminators.d.ts +26 -0
- package/dist/types/extractors/lib/terminators.js.map +1 -0
- package/dist/types/extractors/lib/ui-chrome.d.ts +18 -0
- package/dist/types/extractors/lib/ui-chrome.js.map +1 -0
- package/dist/types/extractors/pipeExtractors.d.ts +5 -0
- package/dist/types/extractors/pipeExtractors.js.map +1 -1
- package/dist/types/extractors/presets/auto.d.ts +30 -0
- package/dist/types/extractors/presets/auto.js.map +1 -0
- package/dist/types/extractors/presets/minimal-filter.d.ts +10 -2
- package/dist/types/extractors/presets/minimal-filter.js.map +1 -1
- package/dist/types/extractors/presets/takumi.d.ts +19 -3
- package/dist/types/extractors/presets/takumi.js.map +1 -1
- package/dist/types/extractors/types.d.ts +8 -0
- package/dist/types/html-to-markdown.d.ts +38 -0
- package/dist/types/html-to-markdown.js.map +1 -1
- package/dist/types/html-to-mdast.d.ts +11 -0
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/index.d.ts +8 -3
- package/dist/types/index.js.map +1 -1
- package/dist/types/link-replacer.d.ts +3 -1
- package/dist/types/link-replacer.js.map +1 -1
- package/dist/types/loaders/cf-puppeteer.js.map +1 -1
- package/dist/types/loaders/fetch.d.ts +4 -0
- package/dist/types/loaders/fetch.js.map +1 -1
- package/dist/types/loaders/playwright.js.map +1 -1
- package/dist/types/loaders/puppeteer.js.map +1 -1
- package/dist/types/md-splitter.d.ts +1 -1
- package/dist/types/md-splitter.js.map +1 -1
- package/dist/types/mdast-handlers/custom-a-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-code-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/custom-code-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-div-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-img-handler.js.map +1 -1
- package/dist/types/mdast-handlers/custom-table-handler.js.map +1 -1
- package/dist/types/mdast-handlers/definition-list-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/definition-list-handler.js.map +1 -0
- package/dist/types/mdast-handlers/math-handler.js.map +1 -1
- package/dist/types/mdast-handlers/sup-sub-handler.d.ts +10 -0
- package/dist/types/mdast-handlers/sup-sub-handler.js.map +1 -0
- package/dist/types/mdast-to-markdown.js.map +1 -1
- package/dist/types/metadata/index.d.ts +47 -0
- package/dist/types/metadata/index.js.map +1 -0
- package/dist/types/normalize/index.d.ts +53 -0
- package/dist/types/normalize/index.js.map +1 -0
- package/dist/types/platform/client.d.ts +55 -0
- package/dist/types/platform/client.js.map +1 -0
- package/dist/types/platform/error.d.ts +22 -0
- package/dist/types/platform/error.js.map +1 -0
- package/dist/types/platform/fetch.d.ts +36 -0
- package/dist/types/platform/fetch.js.map +1 -0
- package/dist/types/platform/index.d.ts +4 -0
- package/dist/types/platform/index.js.map +1 -0
- package/dist/types/platform/types.d.ts +164 -0
- package/dist/types/platform/types.js.map +1 -0
- package/dist/types/utils/hast-fast.d.ts +101 -0
- package/dist/types/utils/hast-fast.js.map +1 -0
- package/dist/utils/hast-fast.js +246 -0
- package/package.json +25 -7
- package/dist/types/cli/commands/webforai/index.d.ts +0 -6
- package/dist/types/cli/commands/webforai/index.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,23 +1,34 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webforai",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "4.0.0",
|
|
4
|
+
"description": "Convert web pages and local HTML to clean, LLM-ready Markdown: library, CLI (npx webforai) and hosted-API client.",
|
|
5
5
|
"author": "inaridiy",
|
|
6
6
|
"license": "Apache-2.0",
|
|
7
7
|
"keywords": [
|
|
8
|
-
"
|
|
8
|
+
"html-to-markdown",
|
|
9
|
+
"markdown",
|
|
10
|
+
"llm",
|
|
9
11
|
"ai",
|
|
12
|
+
"scraper",
|
|
13
|
+
"crawler",
|
|
14
|
+
"readability",
|
|
15
|
+
"web-scraping",
|
|
16
|
+
"cli",
|
|
17
|
+
"rag",
|
|
10
18
|
"html",
|
|
11
19
|
"html2md",
|
|
12
|
-
"markdown",
|
|
13
20
|
"mdast",
|
|
14
21
|
"hast"
|
|
15
22
|
],
|
|
16
23
|
"repository": {
|
|
17
24
|
"type": "git",
|
|
18
|
-
"url": "https://github.com/inaridiy/webforai.git"
|
|
25
|
+
"url": "https://github.com/inaridiy/webforai.git",
|
|
26
|
+
"directory": "packages/webforai"
|
|
19
27
|
},
|
|
20
28
|
"homepage": "https://webforai.dev/",
|
|
29
|
+
"bugs": {
|
|
30
|
+
"url": "https://github.com/inaridiy/webforai/issues"
|
|
31
|
+
},
|
|
21
32
|
"files": [
|
|
22
33
|
"dist",
|
|
23
34
|
"!dist/types/**/*.js"
|
|
@@ -38,6 +49,11 @@
|
|
|
38
49
|
"import": "./dist/index.js",
|
|
39
50
|
"require": "./dist/cjs/index.js"
|
|
40
51
|
},
|
|
52
|
+
"./platform": {
|
|
53
|
+
"types": "./dist/types/platform/index.d.ts",
|
|
54
|
+
"import": "./dist/platform/index.js",
|
|
55
|
+
"require": "./dist/cjs/platform/index.js"
|
|
56
|
+
},
|
|
41
57
|
"./loaders/playwright": {
|
|
42
58
|
"types": "./dist/types/loaders/playwright.d.ts",
|
|
43
59
|
"import": "./dist/loaders/playwright.js",
|
|
@@ -75,6 +91,9 @@
|
|
|
75
91
|
],
|
|
76
92
|
"loaders/puppeteer": [
|
|
77
93
|
"./dist/types/loaders/puppeteer.d.ts"
|
|
94
|
+
],
|
|
95
|
+
"platform": [
|
|
96
|
+
"./dist/types/platform/index.d.ts"
|
|
78
97
|
]
|
|
79
98
|
}
|
|
80
99
|
},
|
|
@@ -110,8 +129,7 @@
|
|
|
110
129
|
"mdast-util-to-markdown": "^2.1.0",
|
|
111
130
|
"picocolors": "^1.0.1",
|
|
112
131
|
"trim-trailing-lines": "^2.1.0",
|
|
113
|
-
"unist-util-filter": "^5.0.1"
|
|
114
|
-
"zx": "^8.1.5"
|
|
132
|
+
"unist-util-filter": "^5.0.1"
|
|
115
133
|
},
|
|
116
134
|
"devDependencies": {
|
|
117
135
|
"@cloudflare/puppeteer": "^0.0.6",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/cli/commands/webforai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,WAAW,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAE3F,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EACnC,WAAmB,EACnB,OAA6E,EAC5E,EAAE;IACH,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,wBAAwB,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAExE,MAAM,UAAU,GAAG,WAAW,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACtF,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;IAE/C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;IACzE,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAEvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;IACzD,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAE3C,IAAI,IAAY,CAAC;IACjB,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;IACpB,IAAI,CAAC;QACJ,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC9B,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACpE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE;QACrC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QACnD,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAC3D,CAAC,CAAC;IACH,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IAEnD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3C,MAAM,iBAAiB,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAEtF,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEzC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC"}
|