webforai 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +93 -52
- package/dist/types/cli/bin.js.map +1 -1
- package/dist/types/cli/commands/webforai/index.js.map +1 -1
- package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -1
- package/dist/types/cli/utils.d.ts +0 -27
- package/dist/types/cli/utils.js.map +1 -1
- package/package.json +3 -2
- package/dist/cjs/cli/bin.js +0 -33
- package/dist/cjs/cli/commands/webforai/index.js +0 -86
- package/dist/cjs/cli/commands/webforai/loadHtml.js +0 -93
- package/dist/cjs/cli/constants.js +0 -34
- package/dist/cjs/cli/helpers/assertContinue.js +0 -34
- package/dist/cjs/cli/helpers/inputOutputPath.js +0 -89
- package/dist/cjs/cli/helpers/inputSourcePath.js +0 -64
- package/dist/cjs/cli/helpers/selectExtractMode.js +0 -39
- package/dist/cjs/cli/helpers/selectLoader.js +0 -44
- package/dist/cjs/cli/utils.js +0 -89
- package/dist/cli/bin.js +0 -11
- package/dist/cli/commands/webforai/index.js +0 -53
- package/dist/cli/commands/webforai/loadHtml.js +0 -60
- package/dist/cli/constants.js +0 -9
- package/dist/cli/helpers/assertContinue.js +0 -11
- package/dist/cli/helpers/inputOutputPath.js +0 -56
- package/dist/cli/helpers/inputSourcePath.js +0 -31
- package/dist/cli/helpers/selectExtractMode.js +0 -16
- package/dist/cli/helpers/selectLoader.js +0 -21
- package/dist/cli/utils.js +0 -52
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var constants_exports = {};
|
|
20
|
-
__export(constants_exports, {
|
|
21
|
-
DEFAULT_PATH: () => DEFAULT_PATH,
|
|
22
|
-
LOADERS: () => LOADERS,
|
|
23
|
-
MODES: () => MODES
|
|
24
|
-
});
|
|
25
|
-
module.exports = __toCommonJS(constants_exports);
|
|
26
|
-
const DEFAULT_PATH = "https://example.com";
|
|
27
|
-
const LOADERS = ["fetch", "playwright"];
|
|
28
|
-
const MODES = ["default", "ai"];
|
|
29
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
30
|
-
0 && (module.exports = {
|
|
31
|
-
DEFAULT_PATH,
|
|
32
|
-
LOADERS,
|
|
33
|
-
MODES
|
|
34
|
-
});
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var assertContinue_exports = {};
|
|
20
|
-
__export(assertContinue_exports, {
|
|
21
|
-
assertContinue: () => assertContinue
|
|
22
|
-
});
|
|
23
|
-
module.exports = __toCommonJS(assertContinue_exports);
|
|
24
|
-
var import_prompts = require("@clack/prompts");
|
|
25
|
-
function assertContinue(message, cancelMessage = "Canceled.") {
|
|
26
|
-
if ((0, import_prompts.isCancel)(message)) {
|
|
27
|
-
(0, import_prompts.cancel)(cancelMessage);
|
|
28
|
-
process.exit(1);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
32
|
-
0 && (module.exports = {
|
|
33
|
-
assertContinue
|
|
34
|
-
});
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var inputOutputPath_exports = {};
|
|
30
|
-
__export(inputOutputPath_exports, {
|
|
31
|
-
inputOutputPath: () => inputOutputPath
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(inputOutputPath_exports);
|
|
34
|
-
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
35
|
-
var import_prompts = require("@clack/prompts");
|
|
36
|
-
var import_utils = require("../utils");
|
|
37
|
-
var import_assertContinue = require("./assertContinue");
|
|
38
|
-
const inputOutputPath = async (sourcePath) => {
|
|
39
|
-
const outputPath = await (0, import_prompts.text)({
|
|
40
|
-
message: "Enter the output file path:",
|
|
41
|
-
placeholder: "output.md",
|
|
42
|
-
initialValue: (0, import_utils.sourcePathToOutputPath)(sourcePath),
|
|
43
|
-
validate: (value) => {
|
|
44
|
-
if (value.trim() === "") {
|
|
45
|
-
return "Output path is required";
|
|
46
|
-
}
|
|
47
|
-
if (import_node_fs.default.existsSync(value) && import_node_fs.default.statSync(value).isDirectory()) {
|
|
48
|
-
return "No directory can be specified.";
|
|
49
|
-
}
|
|
50
|
-
if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
|
|
51
|
-
return "No directory can be specified.";
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
|
-
(0, import_assertContinue.assertContinue)(outputPath);
|
|
56
|
-
if (!import_node_fs.default.existsSync(outputPath)) {
|
|
57
|
-
return outputPath;
|
|
58
|
-
}
|
|
59
|
-
const isOutputFileOverwrite = await (0, import_prompts.confirm)({
|
|
60
|
-
message: "The file already exists. Overwrite?",
|
|
61
|
-
initialValue: false
|
|
62
|
-
});
|
|
63
|
-
(0, import_assertContinue.assertContinue)(isOutputFileOverwrite);
|
|
64
|
-
if (isOutputFileOverwrite) {
|
|
65
|
-
return outputPath;
|
|
66
|
-
}
|
|
67
|
-
const escapedOutputPath = await (0, import_prompts.text)({
|
|
68
|
-
message: "Enter the output file path:",
|
|
69
|
-
placeholder: "output.md",
|
|
70
|
-
initialValue: (0, import_utils.getNextAvailableFilePath)(outputPath),
|
|
71
|
-
validate: (value) => {
|
|
72
|
-
if (value.trim() === "") {
|
|
73
|
-
return "Output path is required";
|
|
74
|
-
}
|
|
75
|
-
if (import_node_fs.default.existsSync(value)) {
|
|
76
|
-
return "The file already exists";
|
|
77
|
-
}
|
|
78
|
-
if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
|
|
79
|
-
return "No directory can be specified.";
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
});
|
|
83
|
-
(0, import_assertContinue.assertContinue)(escapedOutputPath);
|
|
84
|
-
return escapedOutputPath;
|
|
85
|
-
};
|
|
86
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
87
|
-
0 && (module.exports = {
|
|
88
|
-
inputOutputPath
|
|
89
|
-
});
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var inputSourcePath_exports = {};
|
|
30
|
-
__export(inputSourcePath_exports, {
|
|
31
|
-
inputSourcePath: () => inputSourcePath
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(inputSourcePath_exports);
|
|
34
|
-
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
35
|
-
var import_prompts = require("@clack/prompts");
|
|
36
|
-
var import_constants = require("../constants");
|
|
37
|
-
var import_utils = require("../utils");
|
|
38
|
-
var import_assertContinue = require("./assertContinue");
|
|
39
|
-
const inputSourcePath = async () => {
|
|
40
|
-
const result = await (0, import_prompts.text)({
|
|
41
|
-
message: "Enter the URL or html path to be converted to markdown:",
|
|
42
|
-
placeholder: import_constants.DEFAULT_PATH,
|
|
43
|
-
initialValue: "",
|
|
44
|
-
validate: (value) => {
|
|
45
|
-
if (value.trim() === "") {
|
|
46
|
-
return "Source is required";
|
|
47
|
-
}
|
|
48
|
-
if (!(0, import_utils.isUrl)(value)) {
|
|
49
|
-
if (!import_node_fs.default.existsSync(value)) {
|
|
50
|
-
return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
|
|
51
|
-
}
|
|
52
|
-
if (import_node_fs.default.statSync(value).isDirectory()) {
|
|
53
|
-
return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
});
|
|
58
|
-
(0, import_assertContinue.assertContinue)(result);
|
|
59
|
-
return result;
|
|
60
|
-
};
|
|
61
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
62
|
-
0 && (module.exports = {
|
|
63
|
-
inputSourcePath
|
|
64
|
-
});
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var selectExtractMode_exports = {};
|
|
20
|
-
__export(selectExtractMode_exports, {
|
|
21
|
-
selectExtractMode: () => selectExtractMode
|
|
22
|
-
});
|
|
23
|
-
module.exports = __toCommonJS(selectExtractMode_exports);
|
|
24
|
-
var import_prompts = require("@clack/prompts");
|
|
25
|
-
var import_constants = require("../constants");
|
|
26
|
-
var import_assertContinue = require("./assertContinue");
|
|
27
|
-
const selectExtractMode = async () => {
|
|
28
|
-
const result = await (0, import_prompts.select)({
|
|
29
|
-
message: "Select processing mode:",
|
|
30
|
-
options: import_constants.MODES.map((mode) => ({ value: mode, label: mode })),
|
|
31
|
-
initialValue: import_constants.MODES[0]
|
|
32
|
-
});
|
|
33
|
-
(0, import_assertContinue.assertContinue)(result);
|
|
34
|
-
return result;
|
|
35
|
-
};
|
|
36
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
37
|
-
0 && (module.exports = {
|
|
38
|
-
selectExtractMode
|
|
39
|
-
});
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var selectLoader_exports = {};
|
|
20
|
-
__export(selectLoader_exports, {
|
|
21
|
-
selectLoader: () => selectLoader
|
|
22
|
-
});
|
|
23
|
-
module.exports = __toCommonJS(selectLoader_exports);
|
|
24
|
-
var import_prompts = require("@clack/prompts");
|
|
25
|
-
var import_constants = require("../constants");
|
|
26
|
-
var import_assertContinue = require("./assertContinue");
|
|
27
|
-
const loadersHint = {
|
|
28
|
-
fetch: "Fetch HTML content from the given URL",
|
|
29
|
-
playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
|
|
30
|
-
puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
|
|
31
|
-
};
|
|
32
|
-
const selectLoader = async () => {
|
|
33
|
-
const result = await (0, import_prompts.select)({
|
|
34
|
-
message: "Select loader:",
|
|
35
|
-
initialValue: "fetch",
|
|
36
|
-
options: import_constants.LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
|
|
37
|
-
});
|
|
38
|
-
(0, import_assertContinue.assertContinue)(result);
|
|
39
|
-
return result;
|
|
40
|
-
};
|
|
41
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
42
|
-
0 && (module.exports = {
|
|
43
|
-
selectLoader
|
|
44
|
-
});
|
package/dist/cjs/cli/utils.js
DELETED
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var utils_exports = {};
|
|
30
|
-
__export(utils_exports, {
|
|
31
|
-
changeFileExtension: () => changeFileExtension,
|
|
32
|
-
getNextAvailableFilePath: () => getNextAvailableFilePath,
|
|
33
|
-
isUrl: () => isUrl,
|
|
34
|
-
sourcePathToOutputPath: () => sourcePathToOutputPath,
|
|
35
|
-
urlToFilename: () => urlToFilename
|
|
36
|
-
});
|
|
37
|
-
module.exports = __toCommonJS(utils_exports);
|
|
38
|
-
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
39
|
-
var import_node_path = __toESM(require("node:path"), 1);
|
|
40
|
-
const isUrl = (maybeUrl) => {
|
|
41
|
-
try {
|
|
42
|
-
new URL(maybeUrl);
|
|
43
|
-
return true;
|
|
44
|
-
} catch {
|
|
45
|
-
return false;
|
|
46
|
-
}
|
|
47
|
-
};
|
|
48
|
-
function changeFileExtension(filePath, newExtension) {
|
|
49
|
-
const parsed = import_node_path.default.parse(filePath);
|
|
50
|
-
const ext = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
|
|
51
|
-
return import_node_path.default.format({ ...parsed, ext });
|
|
52
|
-
}
|
|
53
|
-
function urlToFilename(url) {
|
|
54
|
-
try {
|
|
55
|
-
const urlObj = new URL(url);
|
|
56
|
-
const hostname = urlObj.hostname.startsWith("www.") ? urlObj.hostname.slice(4) : urlObj.hostname;
|
|
57
|
-
const domainParts = hostname.split(".").slice(-2).join("-");
|
|
58
|
-
const pathParts = urlObj.pathname.split("/").filter(Boolean);
|
|
59
|
-
const relevantPathParts = pathParts.slice(-2).map(decodeURIComponent).join("-");
|
|
60
|
-
let filename = [domainParts, relevantPathParts].filter(Boolean).join("-");
|
|
61
|
-
filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
|
|
62
|
-
return filename || "output";
|
|
63
|
-
} catch {
|
|
64
|
-
return "output";
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
const sourcePathToOutputPath = (sourcePath) => isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
|
|
68
|
-
function getNextAvailableFilePath(filePath) {
|
|
69
|
-
const parsed = import_node_path.default.parse(filePath);
|
|
70
|
-
const dir = parsed.dir;
|
|
71
|
-
const ext = parsed.ext;
|
|
72
|
-
const baseName = parsed.name.replace(/_\d+$/, "");
|
|
73
|
-
const match = parsed.name.match(/_(\d+)$/);
|
|
74
|
-
let counter = match ? Number(match[1]) + 1 : 1;
|
|
75
|
-
let newPath = import_node_path.default.join(dir, `${baseName}_${counter}${ext}`);
|
|
76
|
-
while (import_node_fs.default.existsSync(newPath)) {
|
|
77
|
-
counter++;
|
|
78
|
-
newPath = import_node_path.default.join(dir, `${baseName}_${counter}${ext}`);
|
|
79
|
-
}
|
|
80
|
-
return newPath;
|
|
81
|
-
}
|
|
82
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
83
|
-
0 && (module.exports = {
|
|
84
|
-
changeFileExtension,
|
|
85
|
-
getNextAvailableFilePath,
|
|
86
|
-
isUrl,
|
|
87
|
-
sourcePathToOutputPath,
|
|
88
|
-
urlToFilename
|
|
89
|
-
});
|
package/dist/cli/bin.js
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
// src/cli/bin.ts
|
|
2
|
-
import { intro } from "@clack/prompts";
|
|
3
|
-
import { program } from "commander";
|
|
4
|
-
import pc from "picocolors";
|
|
5
|
-
import packageInfo from "../../package.json";
|
|
6
|
-
import { webforaiCommand } from "./commands/webforai/index.js";
|
|
7
|
-
import { LOADERS, MODES } from "./constants.js";
|
|
8
|
-
program.name("webforai").description("CLI tool for ultra-precise HTML to Markdown conversion").version(packageInfo.version, "-v, --version", "output the current version");
|
|
9
|
-
program.argument("[source]", "URL or path to process").option("-o, --output <output>", "Path to output file or directory").option("-m, --mode <mode>", `Processing mode (${MODES.join(", ")})`).option("-l, --loader <loader>", `Loader to use (${LOADERS.join(", ")})`).option("-d, --debug", "output extra debugging information").action(webforaiCommand);
|
|
10
|
-
intro(pc.inverse(`webforai CLI version ${packageInfo.version}`));
|
|
11
|
-
program.parse();
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
// src/cli/commands/webforai/index.ts
|
|
2
|
-
import fs from "node:fs/promises";
|
|
3
|
-
import path from "node:path";
|
|
4
|
-
import { intro, outro, spinner } from "@clack/prompts";
|
|
5
|
-
import pc from "picocolors";
|
|
6
|
-
import packageInfo from "../../../../package.json";
|
|
7
|
-
import { htmlToMarkdown } from "../../../html-to-markdown.js";
|
|
8
|
-
import { inputOutputPath } from "../../helpers/inputOutputPath.js";
|
|
9
|
-
import { inputSourcePath } from "../../helpers/inputSourcePath.js";
|
|
10
|
-
import { selectExtractMode } from "../../helpers/selectExtractMode.js";
|
|
11
|
-
import { selectLoader } from "../../helpers/selectLoader.js";
|
|
12
|
-
import { isUrl } from "../../utils.js";
|
|
13
|
-
import { loadHtml } from "./loadHtml.js";
|
|
14
|
-
var aiModeOptions = { linkAsText: true, tableAsText: true, hideImage: true };
|
|
15
|
-
var readabilityModeOptions = { linkAsText: false, tableAsText: false, hideImage: false };
|
|
16
|
-
var webforaiCommand = async (initialPath, options) => {
|
|
17
|
-
intro(pc.bold(pc.green(`webforai CLI version ${packageInfo.version}`)));
|
|
18
|
-
const sourcePath = initialPath ?? await inputSourcePath();
|
|
19
|
-
options.debug && console.debug(`sourcePath: ${sourcePath}`);
|
|
20
|
-
const loader = isUrl(sourcePath) ? options.loader ?? await selectLoader() : "local";
|
|
21
|
-
options.debug && console.debug(`loader: ${loader}`);
|
|
22
|
-
const outputPath = options.output ?? await inputOutputPath(sourcePath);
|
|
23
|
-
options.debug && console.debug(`outputPath: ${outputPath}`);
|
|
24
|
-
const mode = options.mode ?? await selectExtractMode();
|
|
25
|
-
options.debug && console.debug(`mode: ${mode}`);
|
|
26
|
-
let html;
|
|
27
|
-
const s = spinner();
|
|
28
|
-
try {
|
|
29
|
-
s.start("Loading content...");
|
|
30
|
-
html = await loadHtml(sourcePath, loader, { debug: options.debug });
|
|
31
|
-
s.stop(pc.green("Content loaded!"));
|
|
32
|
-
} catch (error) {
|
|
33
|
-
s.stop(pc.red("Content loading failed!"));
|
|
34
|
-
console.error(error);
|
|
35
|
-
process.exit(1);
|
|
36
|
-
}
|
|
37
|
-
options.debug && console.debug(`html: ${html}`);
|
|
38
|
-
const markdown = htmlToMarkdown(html, {
|
|
39
|
-
baseUrl: isUrl(sourcePath) ? sourcePath : void 0,
|
|
40
|
-
...mode === "ai" ? aiModeOptions : readabilityModeOptions
|
|
41
|
-
});
|
|
42
|
-
options.debug && console.debug(`markdown: ${markdown}`);
|
|
43
|
-
const directory = path.dirname(outputPath);
|
|
44
|
-
const isDirectoryExists = await fs.stat(directory).then((stat) => stat.isDirectory());
|
|
45
|
-
if (!isDirectoryExists) {
|
|
46
|
-
await fs.mkdir(directory, { recursive: true });
|
|
47
|
-
}
|
|
48
|
-
await fs.writeFile(outputPath, markdown);
|
|
49
|
-
outro(pc.green(`${pc.bold("Done!")} Markdown saved to ${outputPath}`));
|
|
50
|
-
};
|
|
51
|
-
export {
|
|
52
|
-
webforaiCommand
|
|
53
|
-
};
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
// src/cli/commands/webforai/loadHtml.ts
|
|
2
|
-
import fs from "node:fs/promises";
|
|
3
|
-
import pc from "picocolors";
|
|
4
|
-
import { chromium } from "playwright-core";
|
|
5
|
-
import playwrightPackageJson from "playwright-core/package.json";
|
|
6
|
-
import { loadHtml as loadHtmlByFetch } from "../../../loaders/fetch.js";
|
|
7
|
-
import { loadHtml as loadHtmlByPlaywright } from "../../../loaders/playwright.js";
|
|
8
|
-
var checkPlaywrightAvailable = async () => {
|
|
9
|
-
const path = chromium.executablePath();
|
|
10
|
-
const isAvailable = await fs.access(path).catch(() => false);
|
|
11
|
-
return !!isAvailable;
|
|
12
|
-
};
|
|
13
|
-
var loadHtml = async (sourcePath, loader, options) => {
|
|
14
|
-
if (loader === "local") {
|
|
15
|
-
options.debug && console.debug(`Loading HTML from local file: ${sourcePath}`);
|
|
16
|
-
const content = await fs.readFile(sourcePath, "utf-8");
|
|
17
|
-
options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
|
|
18
|
-
return content;
|
|
19
|
-
}
|
|
20
|
-
if (loader === "fetch") {
|
|
21
|
-
options.debug && console.debug(`Loading HTML from URL: ${sourcePath}`);
|
|
22
|
-
const content = await loadHtmlByFetch(sourcePath);
|
|
23
|
-
options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
|
|
24
|
-
return content;
|
|
25
|
-
}
|
|
26
|
-
if (loader === "playwright") {
|
|
27
|
-
options.debug && console.debug(`Loading HTML from playwright: ${sourcePath}`);
|
|
28
|
-
const isPlaywrightAvailable = await checkPlaywrightAvailable();
|
|
29
|
-
options.debug && console.debug(`Playwright available: ${isPlaywrightAvailable}`);
|
|
30
|
-
if (!isPlaywrightAvailable) {
|
|
31
|
-
const message = [
|
|
32
|
-
"To use playwright loader, you need to install playwright.",
|
|
33
|
-
"You can install it by running:",
|
|
34
|
-
"",
|
|
35
|
-
` ${pc.bold(`npx playwright@${playwrightPackageJson.version} install chromium`)} `,
|
|
36
|
-
"",
|
|
37
|
-
"Hint1: If you get warning like this:",
|
|
38
|
-
"WARNING: It looks like you are running 'npx playwright install' without first installing your project's dependencies. ",
|
|
39
|
-
`${pc.bold("Ignore this warning.")}`,
|
|
40
|
-
"",
|
|
41
|
-
"Hint2: If you get message like this:",
|
|
42
|
-
"Host system is missing dependencies to run browsers.",
|
|
43
|
-
"You should install dependencies by running:",
|
|
44
|
-
"",
|
|
45
|
-
` ${pc.bold(`sudo npx playwright@${playwrightPackageJson.version} install-deps`)} `,
|
|
46
|
-
"",
|
|
47
|
-
pc.gray("Note: Good luck with that.")
|
|
48
|
-
];
|
|
49
|
-
console.info(message.join("\n"));
|
|
50
|
-
throw new Error("Playwright is not available");
|
|
51
|
-
}
|
|
52
|
-
const content = await loadHtmlByPlaywright(sourcePath);
|
|
53
|
-
options.debug && console.debug(`HTML loaded: ${content.slice(0, 100)}`);
|
|
54
|
-
return content;
|
|
55
|
-
}
|
|
56
|
-
throw new Error(`Unsupported loader: ${loader}`);
|
|
57
|
-
};
|
|
58
|
-
export {
|
|
59
|
-
loadHtml
|
|
60
|
-
};
|
package/dist/cli/constants.js
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
// src/cli/helpers/assertContinue.ts
|
|
2
|
-
import { cancel, isCancel } from "@clack/prompts";
|
|
3
|
-
function assertContinue(message, cancelMessage = "Canceled.") {
|
|
4
|
-
if (isCancel(message)) {
|
|
5
|
-
cancel(cancelMessage);
|
|
6
|
-
process.exit(1);
|
|
7
|
-
}
|
|
8
|
-
}
|
|
9
|
-
export {
|
|
10
|
-
assertContinue
|
|
11
|
-
};
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
// src/cli/helpers/inputOutputPath.ts
|
|
2
|
-
import fs from "node:fs";
|
|
3
|
-
import { confirm, text } from "@clack/prompts";
|
|
4
|
-
import { getNextAvailableFilePath, sourcePathToOutputPath } from "../utils.js";
|
|
5
|
-
import { assertContinue } from "./assertContinue.js";
|
|
6
|
-
var inputOutputPath = async (sourcePath) => {
|
|
7
|
-
const outputPath = await text({
|
|
8
|
-
message: "Enter the output file path:",
|
|
9
|
-
placeholder: "output.md",
|
|
10
|
-
initialValue: sourcePathToOutputPath(sourcePath),
|
|
11
|
-
validate: (value) => {
|
|
12
|
-
if (value.trim() === "") {
|
|
13
|
-
return "Output path is required";
|
|
14
|
-
}
|
|
15
|
-
if (fs.existsSync(value) && fs.statSync(value).isDirectory()) {
|
|
16
|
-
return "No directory can be specified.";
|
|
17
|
-
}
|
|
18
|
-
if (!fs.existsSync(value) && value.endsWith("/")) {
|
|
19
|
-
return "No directory can be specified.";
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
});
|
|
23
|
-
assertContinue(outputPath);
|
|
24
|
-
if (!fs.existsSync(outputPath)) {
|
|
25
|
-
return outputPath;
|
|
26
|
-
}
|
|
27
|
-
const isOutputFileOverwrite = await confirm({
|
|
28
|
-
message: "The file already exists. Overwrite?",
|
|
29
|
-
initialValue: false
|
|
30
|
-
});
|
|
31
|
-
assertContinue(isOutputFileOverwrite);
|
|
32
|
-
if (isOutputFileOverwrite) {
|
|
33
|
-
return outputPath;
|
|
34
|
-
}
|
|
35
|
-
const escapedOutputPath = await text({
|
|
36
|
-
message: "Enter the output file path:",
|
|
37
|
-
placeholder: "output.md",
|
|
38
|
-
initialValue: getNextAvailableFilePath(outputPath),
|
|
39
|
-
validate: (value) => {
|
|
40
|
-
if (value.trim() === "") {
|
|
41
|
-
return "Output path is required";
|
|
42
|
-
}
|
|
43
|
-
if (fs.existsSync(value)) {
|
|
44
|
-
return "The file already exists";
|
|
45
|
-
}
|
|
46
|
-
if (!fs.existsSync(value) && value.endsWith("/")) {
|
|
47
|
-
return "No directory can be specified.";
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
});
|
|
51
|
-
assertContinue(escapedOutputPath);
|
|
52
|
-
return escapedOutputPath;
|
|
53
|
-
};
|
|
54
|
-
export {
|
|
55
|
-
inputOutputPath
|
|
56
|
-
};
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
// src/cli/helpers/inputSourcePath.ts
|
|
2
|
-
import fs from "node:fs";
|
|
3
|
-
import { text } from "@clack/prompts";
|
|
4
|
-
import { DEFAULT_PATH } from "../constants.js";
|
|
5
|
-
import { isUrl } from "../utils.js";
|
|
6
|
-
import { assertContinue } from "./assertContinue.js";
|
|
7
|
-
var inputSourcePath = async () => {
|
|
8
|
-
const result = await text({
|
|
9
|
-
message: "Enter the URL or html path to be converted to markdown:",
|
|
10
|
-
placeholder: DEFAULT_PATH,
|
|
11
|
-
initialValue: "",
|
|
12
|
-
validate: (value) => {
|
|
13
|
-
if (value.trim() === "") {
|
|
14
|
-
return "Source is required";
|
|
15
|
-
}
|
|
16
|
-
if (!isUrl(value)) {
|
|
17
|
-
if (!fs.existsSync(value)) {
|
|
18
|
-
return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
|
|
19
|
-
}
|
|
20
|
-
if (fs.statSync(value).isDirectory()) {
|
|
21
|
-
return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
});
|
|
26
|
-
assertContinue(result);
|
|
27
|
-
return result;
|
|
28
|
-
};
|
|
29
|
-
export {
|
|
30
|
-
inputSourcePath
|
|
31
|
-
};
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
// src/cli/helpers/selectExtractMode.ts
|
|
2
|
-
import { select } from "@clack/prompts";
|
|
3
|
-
import { MODES } from "../constants.js";
|
|
4
|
-
import { assertContinue } from "./assertContinue.js";
|
|
5
|
-
var selectExtractMode = async () => {
|
|
6
|
-
const result = await select({
|
|
7
|
-
message: "Select processing mode:",
|
|
8
|
-
options: MODES.map((mode) => ({ value: mode, label: mode })),
|
|
9
|
-
initialValue: MODES[0]
|
|
10
|
-
});
|
|
11
|
-
assertContinue(result);
|
|
12
|
-
return result;
|
|
13
|
-
};
|
|
14
|
-
export {
|
|
15
|
-
selectExtractMode
|
|
16
|
-
};
|