webforai 1.5.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +1032 -216
- package/dist/cjs/cli/bin.js +34 -0
- package/dist/cjs/cli/commands/handlers/index.js +84 -0
- package/dist/cjs/cli/commands/handlers/loadHtml.js +127 -0
- package/dist/cjs/cli/constants.js +34 -0
- package/dist/cjs/cli/helpers/assertContinue.js +34 -0
- package/dist/cjs/cli/helpers/inputOutputPath.js +89 -0
- package/dist/cjs/cli/helpers/inputSourcePath.js +64 -0
- package/dist/cjs/cli/helpers/selectExtractMode.js +39 -0
- package/dist/cjs/cli/helpers/selectLoader.js +44 -0
- package/dist/cjs/{utils/bin-utils.js → cli/utils.js} +31 -33
- package/dist/cjs/html-to-markdown.test.js +6 -34
- package/dist/cli/bin.js +12 -0
- package/dist/cli/commands/handlers/index.js +51 -0
- package/dist/cli/commands/handlers/loadHtml.js +94 -0
- package/dist/cli/constants.js +9 -0
- package/dist/cli/helpers/assertContinue.js +11 -0
- package/dist/cli/helpers/inputOutputPath.js +56 -0
- package/dist/cli/helpers/inputSourcePath.js +31 -0
- package/dist/cli/helpers/selectExtractMode.js +16 -0
- package/dist/cli/helpers/selectLoader.js +21 -0
- package/dist/{utils/bin-utils.js → cli/utils.js} +28 -30
- package/dist/html-to-markdown.test.js +6 -12
- package/dist/types/cli/bin.js.map +1 -0
- package/dist/types/cli/commands/handlers/index.d.ts +13 -0
- package/dist/types/cli/commands/handlers/index.js.map +1 -0
- package/dist/types/cli/commands/handlers/loadHtml.d.ts +2 -0
- package/dist/types/cli/commands/handlers/loadHtml.js.map +1 -0
- package/dist/types/cli/constants.d.ts +5 -0
- package/dist/types/cli/constants.js.map +1 -0
- package/dist/types/cli/helpers/assertContinue.d.ts +1 -0
- package/dist/types/cli/helpers/assertContinue.js.map +1 -0
- package/dist/types/cli/helpers/inputOutputPath.d.ts +1 -0
- package/dist/types/cli/helpers/inputOutputPath.js.map +1 -0
- package/dist/types/cli/helpers/inputSourcePath.d.ts +1 -0
- package/dist/types/cli/helpers/inputSourcePath.js.map +1 -0
- package/dist/types/cli/helpers/selectExtractMode.d.ts +1 -0
- package/dist/types/cli/helpers/selectExtractMode.js.map +1 -0
- package/dist/types/cli/helpers/selectLoader.d.ts +1 -0
- package/dist/types/cli/helpers/selectLoader.js.map +1 -0
- package/dist/types/{utils/bin-utils.d.ts → cli/utils.d.ts} +3 -3
- package/dist/types/cli/utils.js.map +1 -0
- package/package.json +7 -17
- package/dist/cjs/bin.js +0 -254
- package/dist/types/bin.js.map +0 -1
- package/dist/types/utils/bin-utils.js.map +0 -1
- /package/dist/types/{bin.d.ts → cli/bin.d.ts} +0 -0
|
@@ -1,26 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __copyProps = (to, from, except, desc) => {
|
|
9
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
10
|
-
for (let key of __getOwnPropNames(from))
|
|
11
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
12
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
13
|
-
}
|
|
14
|
-
return to;
|
|
15
|
-
};
|
|
16
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
17
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
18
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
19
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
20
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
21
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
22
|
-
mod
|
|
23
|
-
));
|
|
24
2
|
var import_fastest_levenshtein = require("fastest-levenshtein");
|
|
25
3
|
var import_vitest = require("vitest");
|
|
26
4
|
var import_html_to_markdown = require("./html-to-markdown");
|
|
@@ -129,18 +107,12 @@ Cell 1 Cell 2`;
|
|
|
129
107
|
});
|
|
130
108
|
});
|
|
131
109
|
(0, import_vitest.describe)("htmlToMarkdown E2E", () => {
|
|
132
|
-
(0, import_vitest.it)("
|
|
133
|
-
const
|
|
134
|
-
const
|
|
135
|
-
const original = await import("../README.md?raw");
|
|
136
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown, original.default);
|
|
137
|
-
(0, import_vitest.expect)(d).lte(200);
|
|
138
|
-
});
|
|
139
|
-
(0, import_vitest.it)("should convert GitHub README to Markdown ", async () => {
|
|
110
|
+
(0, import_vitest.it)("Converting for good", async () => {
|
|
111
|
+
const html1 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
|
|
112
|
+
const markdown1 = (0, import_html_to_markdown.htmlToMarkdown)(html1, { linkAsText: true, hideImage: true });
|
|
140
113
|
const html2 = await (0, import_fetch.loadHtml)("https://github.com/inaridiy/webforai");
|
|
141
|
-
const
|
|
142
|
-
const
|
|
143
|
-
|
|
144
|
-
(0, import_vitest.expect)(d).lte(200);
|
|
114
|
+
const markdown2 = (0, import_html_to_markdown.htmlToMarkdown)(html2, { linkAsText: true, hideImage: true });
|
|
115
|
+
const d = (0, import_fastest_levenshtein.distance)(markdown1, markdown2);
|
|
116
|
+
(0, import_vitest.expect)(d).lte(2500);
|
|
145
117
|
});
|
|
146
118
|
});
|
package/dist/cli/bin.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// src/cli/bin.ts
|
|
2
|
+
import { intro } from "@clack/prompts";
|
|
3
|
+
import { program } from "commander";
|
|
4
|
+
import pc from "picocolors";
|
|
5
|
+
import packageInfo from "../../package.json";
|
|
6
|
+
import { webforaiCommand } from "./commands/handlers/index.js";
|
|
7
|
+
import { LOADERS, MODES } from "./constants.js";
|
|
8
|
+
program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version").argument("[source]", "URL or path to process").argument("[outputPath]", "Path to output file or directory").option("--mode <mode>", `Processing mode (${MODES.join(", ")})`).option("--loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("--baseUrl <baseUrl>", "Base URL for relative links").option("-o, --stdout", "output to stdout").option("-d, --debug", "output extra debugging information").on("option:debug", () => {
|
|
9
|
+
process.env.DEBUG = "true";
|
|
10
|
+
}).action(webforaiCommand);
|
|
11
|
+
intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
|
|
12
|
+
program.parse();
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// src/cli/commands/handlers/index.ts
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { outro } from "@clack/prompts";
|
|
5
|
+
import { htmlToMarkdown } from "../../../html-to-markdown.js";
|
|
6
|
+
import { inputOutputPath } from "../../helpers/inputOutputPath.js";
|
|
7
|
+
import { inputSourcePath } from "../../helpers/inputSourcePath.js";
|
|
8
|
+
import { selectExtractMode } from "../../helpers/selectExtractMode.js";
|
|
9
|
+
import { selectLoader } from "../../helpers/selectLoader.js";
|
|
10
|
+
import { isUrl } from "../../utils.js";
|
|
11
|
+
import { loadHtml } from "./loadHtml.js";
|
|
12
|
+
var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
|
|
13
|
+
var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
|
|
14
|
+
var webforaiCommand = async (initialPath, initialOutputPath, options) => {
|
|
15
|
+
const sourcePath = initialPath ?? await inputSourcePath();
|
|
16
|
+
if (options.debug) {
|
|
17
|
+
console.debug(`sourcePath: ${sourcePath}`);
|
|
18
|
+
}
|
|
19
|
+
const loader = isUrl(sourcePath) ? { type: "web", loader: options.loader ?? await selectLoader() } : { type: "local", loader: null };
|
|
20
|
+
if (options.debug) {
|
|
21
|
+
console.debug(`loader: ${loader}`);
|
|
22
|
+
}
|
|
23
|
+
const outputPath = initialOutputPath ?? await inputOutputPath(sourcePath);
|
|
24
|
+
if (options.debug) {
|
|
25
|
+
console.debug(`outputPath: ${outputPath}`);
|
|
26
|
+
}
|
|
27
|
+
const mode = options.mode ?? await selectExtractMode();
|
|
28
|
+
if (options.debug) {
|
|
29
|
+
console.debug(`mode: ${mode}`);
|
|
30
|
+
}
|
|
31
|
+
const content = await loadHtml(sourcePath, loader);
|
|
32
|
+
if (options.debug) {
|
|
33
|
+
console.debug(`content: ${content}`);
|
|
34
|
+
}
|
|
35
|
+
const markdown = htmlToMarkdown(content, {
|
|
36
|
+
baseUrl: isUrl(sourcePath) ? sourcePath : options.baseUrl,
|
|
37
|
+
...mode === "ai" ? aiModeOptions : readabilityModeOptions
|
|
38
|
+
});
|
|
39
|
+
if (options.debug) {
|
|
40
|
+
console.debug(`markdown: ${markdown}`);
|
|
41
|
+
}
|
|
42
|
+
const directory = path.dirname(outputPath);
|
|
43
|
+
if (!fs.existsSync(directory)) {
|
|
44
|
+
fs.mkdirSync(directory, { recursive: true });
|
|
45
|
+
}
|
|
46
|
+
fs.writeFileSync(outputPath, markdown);
|
|
47
|
+
outro("Markdown conversion is complete!");
|
|
48
|
+
};
|
|
49
|
+
export {
|
|
50
|
+
webforaiCommand
|
|
51
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// src/cli/commands/handlers/loadHtml.ts
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import { cancel, spinner as clackSpinner, confirm, spinner } from "@clack/prompts";
|
|
4
|
+
import pc from "picocolors";
|
|
5
|
+
import { $ } from "zx";
|
|
6
|
+
var loadHtml = async (sourcePath, loader) => {
|
|
7
|
+
const loadSpinner = clackSpinner();
|
|
8
|
+
try {
|
|
9
|
+
switch (loader.type) {
|
|
10
|
+
case "local":
|
|
11
|
+
return await loadLocalHtml(sourcePath, loadSpinner);
|
|
12
|
+
case "web":
|
|
13
|
+
switch (loader.loader) {
|
|
14
|
+
case "fetch":
|
|
15
|
+
return await loadFetchHtml(sourcePath, loadSpinner);
|
|
16
|
+
case "playwright":
|
|
17
|
+
return await loadPlaywrightHtml(sourcePath, loadSpinner);
|
|
18
|
+
default:
|
|
19
|
+
throw new Error("Invalid loader");
|
|
20
|
+
}
|
|
21
|
+
default:
|
|
22
|
+
throw new Error("Invalid loader type");
|
|
23
|
+
}
|
|
24
|
+
} catch (error) {
|
|
25
|
+
loadSpinner.stop(pc.red("Content loading failed!"));
|
|
26
|
+
console.error(error);
|
|
27
|
+
process.exit(1);
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
var loadLocalHtml = async (sourcePath, loadSpinner) => {
|
|
31
|
+
loadSpinner.start("Retrieving content.");
|
|
32
|
+
try {
|
|
33
|
+
await fs.access(sourcePath);
|
|
34
|
+
} catch {
|
|
35
|
+
throw new Error("The file does not exist");
|
|
36
|
+
}
|
|
37
|
+
const stats = await fs.stat(sourcePath);
|
|
38
|
+
if (!stats.isFile()) {
|
|
39
|
+
throw new Error("The source must be a file");
|
|
40
|
+
}
|
|
41
|
+
const content = await fs.readFile(sourcePath, "utf-8");
|
|
42
|
+
loadSpinner.stop("Content retrieval is complete!");
|
|
43
|
+
return content;
|
|
44
|
+
};
|
|
45
|
+
var loadFetchHtml = async (sourcePath, loadSpinner) => {
|
|
46
|
+
loadSpinner.start("Downloading content.");
|
|
47
|
+
const { loadHtml: loadHtml2 } = await import("../../../loaders/fetch.js");
|
|
48
|
+
const content = await loadHtml2(sourcePath);
|
|
49
|
+
loadSpinner.stop("Content download is complete!");
|
|
50
|
+
return content;
|
|
51
|
+
};
|
|
52
|
+
var loadPlaywrightHtml = async (sourcePath, loadSpinner) => {
|
|
53
|
+
const launchSpinner = clackSpinner();
|
|
54
|
+
launchSpinner.start("Launching playwright.");
|
|
55
|
+
const playwright = await import("playwright");
|
|
56
|
+
let browser;
|
|
57
|
+
try {
|
|
58
|
+
browser = await playwright.chromium.launch();
|
|
59
|
+
launchSpinner.stop("Playwright launched!");
|
|
60
|
+
} catch (error) {
|
|
61
|
+
await handlePlaywrightLaunchError(error);
|
|
62
|
+
browser = await playwright.chromium.launch();
|
|
63
|
+
}
|
|
64
|
+
loadSpinner.start("Loading content.");
|
|
65
|
+
const { loadHtml: loadHtml2 } = await import("../../../loaders/playwright.js");
|
|
66
|
+
const content = await loadHtml2(sourcePath, browser);
|
|
67
|
+
loadSpinner.stop("Content download is complete!");
|
|
68
|
+
await browser.close();
|
|
69
|
+
return content;
|
|
70
|
+
};
|
|
71
|
+
var handlePlaywrightLaunchError = async (error) => {
|
|
72
|
+
if (!error.message.includes("npx playwright install")) {
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
const isInstall = await confirm({
|
|
76
|
+
message: "To continue, you will need to download a browser. Do you wish to continue?"
|
|
77
|
+
});
|
|
78
|
+
if (!isInstall) {
|
|
79
|
+
cancel("End of process.");
|
|
80
|
+
process.exit(1);
|
|
81
|
+
}
|
|
82
|
+
const installDepsSpinner = spinner();
|
|
83
|
+
installDepsSpinner.start("Installing browser dependencies.");
|
|
84
|
+
try {
|
|
85
|
+
await $`npx playwright install`;
|
|
86
|
+
installDepsSpinner.stop("Browser dependencies installed!");
|
|
87
|
+
} catch (error2) {
|
|
88
|
+
installDepsSpinner.stop(pc.red("Browser dependencies failed to install!"));
|
|
89
|
+
throw error2;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
export {
|
|
93
|
+
loadHtml
|
|
94
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// src/cli/helpers/assertContinue.ts
|
|
2
|
+
import { cancel, isCancel } from "@clack/prompts";
|
|
3
|
+
function assertContinue(message, cancelMessage = "Canceled.") {
|
|
4
|
+
if (isCancel(message)) {
|
|
5
|
+
cancel(cancelMessage);
|
|
6
|
+
process.exit(1);
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export {
|
|
10
|
+
assertContinue
|
|
11
|
+
};
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// src/cli/helpers/inputOutputPath.ts
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import { confirm, text } from "@clack/prompts";
|
|
4
|
+
import { getNextAvailableFilePath, sourcePathToOutputPath } from "../utils.js";
|
|
5
|
+
import { assertContinue } from "./assertContinue.js";
|
|
6
|
+
var inputOutputPath = async (sourcePath) => {
|
|
7
|
+
const outputPath = await text({
|
|
8
|
+
message: "Enter the output file path:",
|
|
9
|
+
placeholder: "output.md",
|
|
10
|
+
initialValue: sourcePathToOutputPath(sourcePath),
|
|
11
|
+
validate: (value) => {
|
|
12
|
+
if (value.trim() === "") {
|
|
13
|
+
return "Output path is required";
|
|
14
|
+
}
|
|
15
|
+
if (fs.existsSync(value) && fs.statSync(value).isDirectory()) {
|
|
16
|
+
return "No directory can be specified.";
|
|
17
|
+
}
|
|
18
|
+
if (!fs.existsSync(value) && value.endsWith("/")) {
|
|
19
|
+
return "No directory can be specified.";
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
assertContinue(outputPath);
|
|
24
|
+
if (!fs.existsSync(outputPath)) {
|
|
25
|
+
return outputPath;
|
|
26
|
+
}
|
|
27
|
+
const isOutputFileOverwrite = await confirm({
|
|
28
|
+
message: "The file already exists. Overwrite?",
|
|
29
|
+
initialValue: false
|
|
30
|
+
});
|
|
31
|
+
assertContinue(isOutputFileOverwrite);
|
|
32
|
+
if (isOutputFileOverwrite) {
|
|
33
|
+
return outputPath;
|
|
34
|
+
}
|
|
35
|
+
const escapedOutputPath = await text({
|
|
36
|
+
message: "Enter the output file path:",
|
|
37
|
+
placeholder: "output.md",
|
|
38
|
+
initialValue: getNextAvailableFilePath(outputPath),
|
|
39
|
+
validate: (value) => {
|
|
40
|
+
if (value.trim() === "") {
|
|
41
|
+
return "Output path is required";
|
|
42
|
+
}
|
|
43
|
+
if (fs.existsSync(value)) {
|
|
44
|
+
return "The file already exists";
|
|
45
|
+
}
|
|
46
|
+
if (!fs.existsSync(value) && value.endsWith("/")) {
|
|
47
|
+
return "No directory can be specified.";
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
assertContinue(escapedOutputPath);
|
|
52
|
+
return escapedOutputPath;
|
|
53
|
+
};
|
|
54
|
+
export {
|
|
55
|
+
inputOutputPath
|
|
56
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// src/cli/helpers/inputSourcePath.ts
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import { text } from "@clack/prompts";
|
|
4
|
+
import { DEFAULT_PATH } from "../constants.js";
|
|
5
|
+
import { isUrl } from "../utils.js";
|
|
6
|
+
import { assertContinue } from "./assertContinue.js";
|
|
7
|
+
var inputSourcePath = async () => {
|
|
8
|
+
const result = await text({
|
|
9
|
+
message: "Enter the URL or html path to be converted to markdown:",
|
|
10
|
+
placeholder: DEFAULT_PATH,
|
|
11
|
+
initialValue: "",
|
|
12
|
+
validate: (value) => {
|
|
13
|
+
if (value.trim() === "") {
|
|
14
|
+
return "Source is required";
|
|
15
|
+
}
|
|
16
|
+
if (!isUrl(value)) {
|
|
17
|
+
if (!fs.existsSync(value)) {
|
|
18
|
+
return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
|
|
19
|
+
}
|
|
20
|
+
if (fs.statSync(value).isDirectory()) {
|
|
21
|
+
return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
assertContinue(result);
|
|
27
|
+
return result;
|
|
28
|
+
};
|
|
29
|
+
export {
|
|
30
|
+
inputSourcePath
|
|
31
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// src/cli/helpers/selectExtractMode.ts
|
|
2
|
+
import { select } from "@clack/prompts";
|
|
3
|
+
import { MODES } from "../constants.js";
|
|
4
|
+
import { assertContinue } from "./assertContinue.js";
|
|
5
|
+
var selectExtractMode = async () => {
|
|
6
|
+
const result = await select({
|
|
7
|
+
message: "Select processing mode:",
|
|
8
|
+
options: MODES.map((mode) => ({ value: mode, label: mode })),
|
|
9
|
+
initialValue: MODES[0]
|
|
10
|
+
});
|
|
11
|
+
assertContinue(result);
|
|
12
|
+
return result;
|
|
13
|
+
};
|
|
14
|
+
export {
|
|
15
|
+
selectExtractMode
|
|
16
|
+
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// src/cli/helpers/selectLoader.ts
|
|
2
|
+
import { select } from "@clack/prompts";
|
|
3
|
+
import { LOADERS } from "../constants.js";
|
|
4
|
+
import { assertContinue } from "./assertContinue.js";
|
|
5
|
+
var loadersHint = {
|
|
6
|
+
fetch: "Fetch HTML content from the given URL",
|
|
7
|
+
playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
|
|
8
|
+
puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
|
|
9
|
+
};
|
|
10
|
+
var selectLoader = async () => {
|
|
11
|
+
const result = await select({
|
|
12
|
+
message: "Select loader:",
|
|
13
|
+
initialValue: "fetch",
|
|
14
|
+
options: LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
|
|
15
|
+
});
|
|
16
|
+
assertContinue(result);
|
|
17
|
+
return result;
|
|
18
|
+
};
|
|
19
|
+
export {
|
|
20
|
+
selectLoader
|
|
21
|
+
};
|
|
@@ -1,39 +1,14 @@
|
|
|
1
|
-
// src/
|
|
1
|
+
// src/cli/utils.ts
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
-
|
|
5
|
-
function logError(error) {
|
|
6
|
-
log.error("An error occurred during processing");
|
|
7
|
-
console.error(error.message);
|
|
8
|
-
}
|
|
9
|
-
function isUrl(str) {
|
|
4
|
+
var isUrl = (maybeUrl) => {
|
|
10
5
|
try {
|
|
11
|
-
new URL(
|
|
6
|
+
new URL(maybeUrl);
|
|
12
7
|
return true;
|
|
13
8
|
} catch {
|
|
14
9
|
return false;
|
|
15
10
|
}
|
|
16
|
-
}
|
|
17
|
-
function getNextAvailableFilePath(filePath) {
|
|
18
|
-
const parsedPath = path.parse(filePath);
|
|
19
|
-
const directory = parsedPath.dir;
|
|
20
|
-
const fullName = parsedPath.base;
|
|
21
|
-
const [firstPart, ...restParts] = fullName.split(".");
|
|
22
|
-
const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
|
|
23
|
-
const baseName = firstPart.replace(/_\d+$/, "");
|
|
24
|
-
let counter = 1;
|
|
25
|
-
let nextFilePath = filePath;
|
|
26
|
-
while (fs.existsSync(nextFilePath)) {
|
|
27
|
-
const match = firstPart.match(/_(\d+)$/);
|
|
28
|
-
if (match) {
|
|
29
|
-
counter = Number.parseInt(match[1], 10) + 1;
|
|
30
|
-
}
|
|
31
|
-
const newName = `${baseName}_${counter}${restName}`;
|
|
32
|
-
nextFilePath = path.join(directory, newName);
|
|
33
|
-
counter++;
|
|
34
|
-
}
|
|
35
|
-
return nextFilePath;
|
|
36
|
-
}
|
|
11
|
+
};
|
|
37
12
|
function changeFileExtension(filePath, newExtension) {
|
|
38
13
|
const parsedPath = filePath.split("/");
|
|
39
14
|
const fileName = parsedPath[parsedPath.length - 1];
|
|
@@ -78,10 +53,33 @@ function urlToFilename(url) {
|
|
|
78
53
|
return "output";
|
|
79
54
|
}
|
|
80
55
|
}
|
|
56
|
+
var sourcePathToOutputPath = (sourcePath) => {
|
|
57
|
+
return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
|
|
58
|
+
};
|
|
59
|
+
function getNextAvailableFilePath(filePath) {
|
|
60
|
+
const parsedPath = path.parse(filePath);
|
|
61
|
+
const directory = parsedPath.dir;
|
|
62
|
+
const fullName = parsedPath.base;
|
|
63
|
+
const [firstPart, ...restParts] = fullName.split(".");
|
|
64
|
+
const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
|
|
65
|
+
const baseName = firstPart.replace(/_\d+$/, "");
|
|
66
|
+
let counter = 1;
|
|
67
|
+
let nextFilePath = filePath;
|
|
68
|
+
while (fs.existsSync(nextFilePath)) {
|
|
69
|
+
const match = firstPart.match(/_(\d+)$/);
|
|
70
|
+
if (match) {
|
|
71
|
+
counter = Number.parseInt(match[1], 10) + 1;
|
|
72
|
+
}
|
|
73
|
+
const newName = `${baseName}_${counter}${restName}`;
|
|
74
|
+
nextFilePath = path.join(directory, newName);
|
|
75
|
+
counter++;
|
|
76
|
+
}
|
|
77
|
+
return nextFilePath;
|
|
78
|
+
}
|
|
81
79
|
export {
|
|
82
80
|
changeFileExtension,
|
|
83
81
|
getNextAvailableFilePath,
|
|
84
82
|
isUrl,
|
|
85
|
-
|
|
83
|
+
sourcePathToOutputPath,
|
|
86
84
|
urlToFilename
|
|
87
85
|
};
|
|
@@ -107,18 +107,12 @@ describe("htmlToMarkdown", () => {
|
|
|
107
107
|
});
|
|
108
108
|
});
|
|
109
109
|
describe("htmlToMarkdown E2E", () => {
|
|
110
|
-
it("
|
|
111
|
-
const
|
|
112
|
-
const
|
|
113
|
-
const original = await import("../README.md?raw");
|
|
114
|
-
const d = distance(markdown, original.default);
|
|
115
|
-
expect(d).lte(200);
|
|
116
|
-
});
|
|
117
|
-
it("should convert GitHub README to Markdown ", async () => {
|
|
110
|
+
it("Converting for good", async () => {
|
|
111
|
+
const html1 = await loadHtml("https://www.npmjs.com/package/webforai");
|
|
112
|
+
const markdown1 = htmlToMarkdown(html1, { linkAsText: true, hideImage: true });
|
|
118
113
|
const html2 = await loadHtml("https://github.com/inaridiy/webforai");
|
|
119
|
-
const
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
expect(d).lte(200);
|
|
114
|
+
const markdown2 = htmlToMarkdown(html2, { linkAsText: true, hideImage: true });
|
|
115
|
+
const d = distance(markdown1, markdown2);
|
|
116
|
+
expect(d).lte(2500);
|
|
123
117
|
});
|
|
124
118
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bin.js","sourceRoot":"","sources":["../../../src/cli/bin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5B,OAAO,WAAW,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO;KACL,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,wDAAwD,CAAC;KACrE,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,eAAe,EAAE,4BAA4B,CAAC;KAC3E,QAAQ,CAAC,UAAU,EAAE,wBAAwB,CAAC;KAC9C,QAAQ,CAAC,cAAc,EAAE,kCAAkC,CAAC;KAC5D,MAAM,CAAC,eAAe,EAAE,oBAAoB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;KAChE,MAAM,CAAC,mBAAmB,EAAE,kBAAkB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;KACpE,MAAM,CAAC,qBAAqB,EAAE,6BAA6B,CAAC;KAC5D,MAAM,CAAC,cAAc,EAAE,kBAAkB,CAAC;KAC1C,MAAM,CAAC,aAAa,EAAE,oCAAoC,CAAC;KAC3D,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;IACxB,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC;AAC5B,CAAC,CAAC;KACD,MAAM,CAAC,eAAe,CAAC,CAAC;AAE1B,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,wBAAwB,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAEjE,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export type LoaderOptions = {
|
|
2
|
+
type: "web";
|
|
3
|
+
loader: "playwright" | "pw" | "fetch" | string;
|
|
4
|
+
} | {
|
|
5
|
+
type: "local";
|
|
6
|
+
loader: null;
|
|
7
|
+
};
|
|
8
|
+
export declare const webforaiCommand: (initialPath: string, initialOutputPath: string, options: {
|
|
9
|
+
mode?: string;
|
|
10
|
+
loader: string;
|
|
11
|
+
debug?: boolean;
|
|
12
|
+
baseUrl?: string;
|
|
13
|
+
}) => Promise<void>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,aAAa,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC/E,MAAM,sBAAsB,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAM3F,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EACnC,WAAmB,EACnB,iBAAyB,EACzB,OAA6E,EAC5E,EAAE;IACH,MAAM,UAAU,GAAG,WAAW,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,MAAM,GAAkB,KAAK,CAAC,UAAU,CAAC;QAC9C,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,YAAY,EAAE,CAAC,EAAE;QACnE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,UAAU,GAAG,iBAAiB,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,EAAE;QACxC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO;QACzD,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAC3D,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAE3C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAEvC,KAAK,CAAC,kCAAkC,CAAC,CAAC;AAC3C,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loadHtml.js","sourceRoot":"","sources":["../../../../../src/cli/commands/handlers/loadHtml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACnF,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,OAAO,EAAE,CAAC,EAAE,MAAM,IAAI,CAAC;AAKvB,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,UAAkB,EAAE,MAAqB,EAAmB,EAAE;IAC5F,MAAM,WAAW,GAAG,YAAY,EAAE,CAAC;IAEnC,IAAI,CAAC;QACJ,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,OAAO;gBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;YACrD,KAAK,KAAK;gBACT,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;oBACvB,KAAK,OAAO;wBACX,OAAO,MAAM,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACrD,KAAK,YAAY;wBAChB,OAAO,MAAM,kBAAkB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBAC1D;wBACC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBACpC,CAAC;YACF;gBACC,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,IAAI,CAAC;QACJ,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACvD,WAAW,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACnD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAC7F,WAAW,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC3C,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAAE,UAAkB,EAAE,WAAwB,EAAmB,EAAE;IAClG,MAAM,aAAa,GAAG,YAAY,EAAE,CAAC;IACrC,aAAa,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAE7C,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9C,IAAI,OAA0B,CAAC;IAE/B,IAAI,CAAC;QACJ,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC7C,aAAa,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,2BAA2B,CAAC,KAAc,CAAC,CAAC;QAClD,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC9C,CAAC;IAED,WAAW,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAElD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAEtB,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,KAAK,EAAE,KAAY,EAAiB,EAAE;IACzE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACvD,MAAM,KAAK,CAAC;IACb,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC;QAC/B,OAAO,EAAE,4EAA4E;KACrF,CAAC,CAAC;IACH,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,kBAAkB,GAAG,OAAO,EAAE,CAAC;IACrC,kBAAkB,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7D,IAAI,CAAC;QACJ,MAAM,CAAC,CAAA,wBAAwB,CAAC;QAChC,kBAAkB,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../src/cli/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,qBAAqB,CAAC;AAElD,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,YAAY,CAAU,CAAC;AAGxD,MAAM,CAAC,MAAM,KAAK,GAAa,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function assertContinue<T>(message: T | symbol, cancelMessage?: string): asserts message is T;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertContinue.js","sourceRoot":"","sources":["../../../../src/cli/helpers/assertContinue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAElD,MAAM,UAAU,cAAc,CAAI,OAAmB,EAAE,aAAa,GAAG,WAAW;IACjF,IAAI,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,aAAa,CAAC,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const inputOutputPath: (sourcePath: string) => Promise<string>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inputOutputPath.js","sourceRoot":"","sources":["../../../../src/cli/helpers/inputOutputPath.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAC5E,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,UAAkB,EAAE,EAAE;IAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC;QAC7B,OAAO,EAAE,6BAA6B;QACtC,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,sBAAsB,CAAC,UAAU,CAAC;QAChD,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;YAC3B,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACzB,OAAO,yBAAyB,CAAC;YAClC,CAAC;YACD,IAAI,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC9D,OAAO,gCAAgC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClD,OAAO,gCAAgC,CAAC;YACzC,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,cAAc,CAAC,UAAU,CAAC,CAAC;IAE3B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAChC,OAAO,UAAU,CAAC;IACnB,CAAC;IAED,MAAM,qBAAqB,GAAG,MAAM,OAAO,CAAC;QAC3C,OAAO,EAAE,qCAAqC;QAC9C,YAAY,EAAE,KAAK;KACnB,CAAC,CAAC;IACH,cAAc,CAAC,qBAAqB,CAAC,CAAC;IAEtC,IAAI,qBAAqB,EAAE,CAAC;QAC3B,OAAO,UAAU,CAAC;IACnB,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC;QACpC,OAAO,EAAE,6BAA6B;QACtC,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,wBAAwB,CAAC,UAAU,CAAC;QAClD,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;YAC3B,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACzB,OAAO,yBAAyB,CAAC;YAClC,CAAC;YACD,IAAI,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,OAAO,yBAAyB,CAAC;YAClC,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClD,OAAO,gCAAgC,CAAC;YACzC,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IAEH,cAAc,CAAC,iBAAiB,CAAC,CAAC;IAElC,OAAO,iBAAiB,CAAC;AAC1B,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const inputSourcePath: () => Promise<string>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inputSourcePath.js","sourceRoot":"","sources":["../../../../src/cli/helpers/inputSourcePath.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;IACzC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC;QACzB,OAAO,EAAE,yDAAyD;QAClE,WAAW,EAAE,YAAY;QACzB,YAAY,EAAE,EAAE;QAChB,QAAQ,EAAE,CAAC,KAAa,EAAE,EAAE;YAC3B,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACzB,OAAO,oBAAoB,CAAC;YAC7B,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC3B,OAAO,uIAAuI,CAAC;gBAChJ,CAAC;gBACD,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;oBACtC,OAAO,6HAA6H,CAAC;gBACtI,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,cAAc,CAAC,MAAM,CAAC,CAAC;IAEvB,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const selectExtractMode: () => Promise<string>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"selectExtractMode.js","sourceRoot":"","sources":["../../../../src/cli/helpers/selectExtractMode.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,IAAI,EAAE;IAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;QAC3B,OAAO,EAAE,yBAAyB;QAClC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC;KACtB,CAAC,CAAC;IAEH,cAAc,CAAC,MAAM,CAAC,CAAC;IAEvB,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const selectLoader: () => Promise<string>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"selectLoader.js","sourceRoot":"","sources":["../../../../src/cli/helpers/selectLoader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,GAAG;IACnB,KAAK,EAAE,uCAAuC;IAC9C,UAAU,EAAE,kGAAkG;IAC9G,SAAS,EAAE,gGAAgG;CAClG,CAAC;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;IACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;QAC3B,OAAO,EAAE,gBAAgB;QACzB,YAAY,EAAE,OAAO;QACrB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;KAC7F,CAAC,CAAC;IACH,cAAc,CAAC,MAAM,CAAC,CAAC;IAEvB,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export declare
|
|
2
|
-
export declare function isUrl(str: string): boolean;
|
|
3
|
-
export declare function getNextAvailableFilePath(filePath: string): string;
|
|
1
|
+
export declare const isUrl: (maybeUrl: string) => boolean;
|
|
4
2
|
export declare function changeFileExtension(filePath: string, newExtension: string): string;
|
|
5
3
|
export declare function urlToFilename(url: string): string;
|
|
4
|
+
export declare const sourcePathToOutputPath: (sourcePath: string) => string;
|
|
5
|
+
export declare function getNextAvailableFilePath(filePath: string): string;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/cli/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,QAAgB,EAAE,EAAE;IACzC,IAAI,CAAC;QACJ,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,UAAU,mBAAmB,CAAC,QAAgB,EAAE,YAAoB;IACzE,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEnD,MAAM,qBAAqB,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC;IAE/F,IAAI,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,QAAQ,GAAG,qBAAqB,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxF,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC1D,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAClF,MAAM,WAAW,GAAG,GAAG,QAAQ,GAAG,qBAAqB,EAAE,CAAC;IAE1D,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACxC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5B,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ;aACjC,KAAK,CAAC,GAAG,CAAC;aACV,OAAO,EAAE;aACT,MAAM,CAAC,CAAC,GAAa,EAAE,IAAY,EAAE,KAAa,EAAE,EAAE;YACtD,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACrB,OAAO,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACpB,OAAO,GAAG,CAAC;YACZ,CAAC;YACD,oEAAoE;YACpE,OAAO,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC;QACvB,CAAC,EAAE,EAAE,CAAC,CAAC;QACR,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEvF,IAAI,QAAQ,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpE,QAAQ,GAAG,QAAQ;aACjB,WAAW,EAAE;YACd,yEAAyE;aACxE,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;aACrC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAE1B,OAAO,QAAQ,IAAI,QAAQ,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;AACF,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,UAAkB,EAAE,EAAE;IAC5D,OAAO,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;AACtG,CAAC,CAAC;AAEF,MAAM,UAAU,wBAAwB,CAAC,QAAgB;IACxD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC;IACjC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEvE,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEhD,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,QAAQ,CAAC;IAE5B,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,KAAK,EAAE,CAAC;YACX,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,OAAO,GAAG,QAAQ,EAAE,CAAC;QACpD,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7C,OAAO,EAAE,CAAC;IACX,CAAC;IAED,OAAO,YAAY,CAAC;AACrB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "webforai",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.1",
|
|
4
4
|
"description": "A library that provides a web interface for AI",
|
|
5
5
|
"author": "inaridiy",
|
|
6
6
|
"license": "MIT",
|
|
@@ -79,20 +79,9 @@
|
|
|
79
79
|
}
|
|
80
80
|
},
|
|
81
81
|
"peerDependencies": {
|
|
82
|
-
"@cloudflare/puppeteer": "
|
|
83
|
-
"playwright": "
|
|
84
|
-
"puppeteer": "
|
|
85
|
-
},
|
|
86
|
-
"peerDependenciesMeta": {
|
|
87
|
-
"playwright": {
|
|
88
|
-
"optional": true
|
|
89
|
-
},
|
|
90
|
-
"@cloudflare/puppeteer": {
|
|
91
|
-
"optional": true
|
|
92
|
-
},
|
|
93
|
-
"puppeteer": {
|
|
94
|
-
"optional": true
|
|
95
|
-
}
|
|
82
|
+
"@cloudflare/puppeteer": ">=0.0.6",
|
|
83
|
+
"playwright": ">=1.4",
|
|
84
|
+
"puppeteer": ">=22"
|
|
96
85
|
},
|
|
97
86
|
"dependencies": {
|
|
98
87
|
"@clack/prompts": "^0.7.0",
|
|
@@ -109,9 +98,9 @@
|
|
|
109
98
|
"mdast-util-math": "^3.0.0",
|
|
110
99
|
"mdast-util-to-markdown": "^2.1.0",
|
|
111
100
|
"picocolors": "^1.0.1",
|
|
112
|
-
"puppeteer": "^22.10.0",
|
|
113
101
|
"trim-trailing-lines": "^2.1.0",
|
|
114
|
-
"unist-util-filter": "^5.0.1"
|
|
102
|
+
"unist-util-filter": "^5.0.1",
|
|
103
|
+
"zx": "^8.1.5"
|
|
115
104
|
},
|
|
116
105
|
"devDependencies": {
|
|
117
106
|
"@cloudflare/puppeteer": "^0.0.6",
|
|
@@ -125,6 +114,7 @@
|
|
|
125
114
|
"ncp": "^2.0.0",
|
|
126
115
|
"np": "^9.2.0",
|
|
127
116
|
"playwright": "^1.40.1",
|
|
117
|
+
"puppeteer": "^23.2.2",
|
|
128
118
|
"rimraf": "^5.0.5",
|
|
129
119
|
"tsx": "^4.7.0",
|
|
130
120
|
"typescript": "^5.4.5"
|