webforai 1.5.1 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +258 -278
- package/dist/cjs/extract-hast/index.js +4 -3
- package/dist/cjs/extract-hast/readability.js +28 -61
- package/dist/cjs/extract-hast/strongly.js +2 -1
- package/dist/cjs/extract-hast/utils.js +0 -15
- package/dist/cjs/html-to-mdast.js +6 -6
- package/dist/cjs/link-replacer.js +10 -5
- package/dist/cjs/loaders/playwright.js +36 -7
- package/dist/cjs/{cli/constants.js → mdast-handlers/custom-br-handler.js} +9 -12
- package/dist/cjs/mdast-to-markdown.js +1 -1
- package/dist/cjs/utils/hast-utils.js +71 -0
- package/dist/extract-hast/index.js +4 -3
- package/dist/extract-hast/readability.js +29 -62
- package/dist/extract-hast/strongly.js +2 -1
- package/dist/extract-hast/utils.js +0 -14
- package/dist/html-to-mdast.js +6 -6
- package/dist/link-replacer.js +10 -5
- package/dist/loaders/playwright.js +36 -7
- package/dist/mdast-handlers/custom-br-handler.js +8 -0
- package/dist/mdast-to-markdown.js +1 -1
- package/dist/types/cli/bin.js.map +1 -1
- package/dist/types/cli/commands/webforai/index.d.ts +6 -0
- package/dist/types/cli/commands/webforai/index.js.map +1 -0
- package/dist/types/cli/commands/webforai/loadHtml.d.ts +3 -0
- package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -0
- package/dist/types/extract-hast/index.d.ts +9 -4
- package/dist/types/extract-hast/index.js.map +1 -1
- package/dist/types/extract-hast/readability.d.ts +2 -1
- package/dist/types/extract-hast/readability.js.map +1 -1
- package/dist/types/extract-hast/strongly.d.ts +2 -1
- package/dist/types/extract-hast/strongly.js.map +1 -1
- package/dist/types/extract-hast/utils.d.ts +0 -1
- package/dist/types/extract-hast/utils.js.map +1 -1
- package/dist/types/html-to-mdast.d.ts +4 -0
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/link-replacer.d.ts +1 -1
- package/dist/types/link-replacer.js.map +1 -1
- package/dist/types/loaders/playwright.d.ts +8 -2
- package/dist/types/loaders/playwright.js.map +1 -1
- package/dist/types/mdast-handlers/custom-br-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/custom-br-handler.js.map +1 -0
- package/dist/types/mdast-to-markdown.js.map +1 -1
- package/dist/types/utils/hast-utils.d.ts +4 -0
- package/dist/types/utils/hast-utils.js.map +1 -0
- package/dist/utils/hast-utils.js +46 -0
- package/package.json +22 -10
- package/dist/cjs/cli/bin.js +0 -34
- package/dist/cjs/cli/commands/handlers/index.js +0 -84
- package/dist/cjs/cli/commands/handlers/loadHtml.js +0 -127
- package/dist/cjs/cli/helpers/assertContinue.js +0 -34
- package/dist/cjs/cli/helpers/inputOutputPath.js +0 -89
- package/dist/cjs/cli/helpers/inputSourcePath.js +0 -64
- package/dist/cjs/cli/helpers/selectExtractMode.js +0 -39
- package/dist/cjs/cli/helpers/selectLoader.js +0 -44
- package/dist/cjs/cli/utils.js +0 -122
- package/dist/cjs/html-to-markdown.test.js +0 -118
- package/dist/cjs/link-replacer.test.js +0 -47
- package/dist/cjs/loaders/fetch.test.js +0 -9
- package/dist/cjs/loaders/playwright.test.js +0 -17
- package/dist/cjs/loaders/puppeteer.test.js +0 -17
- package/dist/cli/bin.js +0 -12
- package/dist/cli/commands/handlers/index.js +0 -51
- package/dist/cli/commands/handlers/loadHtml.js +0 -94
- package/dist/cli/constants.js +0 -9
- package/dist/cli/helpers/assertContinue.js +0 -11
- package/dist/cli/helpers/inputOutputPath.js +0 -56
- package/dist/cli/helpers/inputSourcePath.js +0 -31
- package/dist/cli/helpers/selectExtractMode.js +0 -16
- package/dist/cli/helpers/selectLoader.js +0 -21
- package/dist/cli/utils.js +0 -85
- package/dist/html-to-markdown.test.js +0 -118
- package/dist/link-replacer.test.js +0 -47
- package/dist/loaders/fetch.test.js +0 -9
- package/dist/loaders/playwright.test.js +0 -17
- package/dist/loaders/puppeteer.test.js +0 -17
- package/dist/types/cli/commands/handlers/index.d.ts +0 -13
- package/dist/types/cli/commands/handlers/index.js.map +0 -1
- package/dist/types/cli/commands/handlers/loadHtml.d.ts +0 -2
- package/dist/types/cli/commands/handlers/loadHtml.js.map +0 -1
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var loadHtml_exports = {};
|
|
30
|
-
__export(loadHtml_exports, {
|
|
31
|
-
loadHtml: () => loadHtml
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(loadHtml_exports);
|
|
34
|
-
var import_promises = __toESM(require("node:fs/promises"), 1);
|
|
35
|
-
var import_prompts = require("@clack/prompts");
|
|
36
|
-
var import_picocolors = __toESM(require("picocolors"), 1);
|
|
37
|
-
var import_zx = require("zx");
|
|
38
|
-
const loadHtml = async (sourcePath, loader) => {
|
|
39
|
-
const loadSpinner = (0, import_prompts.spinner)();
|
|
40
|
-
try {
|
|
41
|
-
switch (loader.type) {
|
|
42
|
-
case "local":
|
|
43
|
-
return await loadLocalHtml(sourcePath, loadSpinner);
|
|
44
|
-
case "web":
|
|
45
|
-
switch (loader.loader) {
|
|
46
|
-
case "fetch":
|
|
47
|
-
return await loadFetchHtml(sourcePath, loadSpinner);
|
|
48
|
-
case "playwright":
|
|
49
|
-
return await loadPlaywrightHtml(sourcePath, loadSpinner);
|
|
50
|
-
default:
|
|
51
|
-
throw new Error("Invalid loader");
|
|
52
|
-
}
|
|
53
|
-
default:
|
|
54
|
-
throw new Error("Invalid loader type");
|
|
55
|
-
}
|
|
56
|
-
} catch (error) {
|
|
57
|
-
loadSpinner.stop(import_picocolors.default.red("Content loading failed!"));
|
|
58
|
-
console.error(error);
|
|
59
|
-
process.exit(1);
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
const loadLocalHtml = async (sourcePath, loadSpinner) => {
|
|
63
|
-
loadSpinner.start("Retrieving content.");
|
|
64
|
-
try {
|
|
65
|
-
await import_promises.default.access(sourcePath);
|
|
66
|
-
} catch {
|
|
67
|
-
throw new Error("The file does not exist");
|
|
68
|
-
}
|
|
69
|
-
const stats = await import_promises.default.stat(sourcePath);
|
|
70
|
-
if (!stats.isFile()) {
|
|
71
|
-
throw new Error("The source must be a file");
|
|
72
|
-
}
|
|
73
|
-
const content = await import_promises.default.readFile(sourcePath, "utf-8");
|
|
74
|
-
loadSpinner.stop("Content retrieval is complete!");
|
|
75
|
-
return content;
|
|
76
|
-
};
|
|
77
|
-
const loadFetchHtml = async (sourcePath, loadSpinner) => {
|
|
78
|
-
loadSpinner.start("Downloading content.");
|
|
79
|
-
const { loadHtml: loadHtml2 } = await import("../../../loaders/fetch");
|
|
80
|
-
const content = await loadHtml2(sourcePath);
|
|
81
|
-
loadSpinner.stop("Content download is complete!");
|
|
82
|
-
return content;
|
|
83
|
-
};
|
|
84
|
-
const loadPlaywrightHtml = async (sourcePath, loadSpinner) => {
|
|
85
|
-
const launchSpinner = (0, import_prompts.spinner)();
|
|
86
|
-
launchSpinner.start("Launching playwright.");
|
|
87
|
-
const playwright = await import("playwright");
|
|
88
|
-
let browser;
|
|
89
|
-
try {
|
|
90
|
-
browser = await playwright.chromium.launch();
|
|
91
|
-
launchSpinner.stop("Playwright launched!");
|
|
92
|
-
} catch (error) {
|
|
93
|
-
await handlePlaywrightLaunchError(error);
|
|
94
|
-
browser = await playwright.chromium.launch();
|
|
95
|
-
}
|
|
96
|
-
loadSpinner.start("Loading content.");
|
|
97
|
-
const { loadHtml: loadHtml2 } = await import("../../../loaders/playwright");
|
|
98
|
-
const content = await loadHtml2(sourcePath, browser);
|
|
99
|
-
loadSpinner.stop("Content download is complete!");
|
|
100
|
-
await browser.close();
|
|
101
|
-
return content;
|
|
102
|
-
};
|
|
103
|
-
const handlePlaywrightLaunchError = async (error) => {
|
|
104
|
-
if (!error.message.includes("npx playwright install")) {
|
|
105
|
-
throw error;
|
|
106
|
-
}
|
|
107
|
-
const isInstall = await (0, import_prompts.confirm)({
|
|
108
|
-
message: "To continue, you will need to download a browser. Do you wish to continue?"
|
|
109
|
-
});
|
|
110
|
-
if (!isInstall) {
|
|
111
|
-
(0, import_prompts.cancel)("End of process.");
|
|
112
|
-
process.exit(1);
|
|
113
|
-
}
|
|
114
|
-
const installDepsSpinner = (0, import_prompts.spinner)();
|
|
115
|
-
installDepsSpinner.start("Installing browser dependencies.");
|
|
116
|
-
try {
|
|
117
|
-
await import_zx.$`npx playwright install`;
|
|
118
|
-
installDepsSpinner.stop("Browser dependencies installed!");
|
|
119
|
-
} catch (error2) {
|
|
120
|
-
installDepsSpinner.stop(import_picocolors.default.red("Browser dependencies failed to install!"));
|
|
121
|
-
throw error2;
|
|
122
|
-
}
|
|
123
|
-
};
|
|
124
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
125
|
-
0 && (module.exports = {
|
|
126
|
-
loadHtml
|
|
127
|
-
});
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var assertContinue_exports = {};
|
|
20
|
-
__export(assertContinue_exports, {
|
|
21
|
-
assertContinue: () => assertContinue
|
|
22
|
-
});
|
|
23
|
-
module.exports = __toCommonJS(assertContinue_exports);
|
|
24
|
-
var import_prompts = require("@clack/prompts");
|
|
25
|
-
function assertContinue(message, cancelMessage = "Canceled.") {
|
|
26
|
-
if ((0, import_prompts.isCancel)(message)) {
|
|
27
|
-
(0, import_prompts.cancel)(cancelMessage);
|
|
28
|
-
process.exit(1);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
32
|
-
0 && (module.exports = {
|
|
33
|
-
assertContinue
|
|
34
|
-
});
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var inputOutputPath_exports = {};
|
|
30
|
-
__export(inputOutputPath_exports, {
|
|
31
|
-
inputOutputPath: () => inputOutputPath
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(inputOutputPath_exports);
|
|
34
|
-
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
35
|
-
var import_prompts = require("@clack/prompts");
|
|
36
|
-
var import_utils = require("../utils");
|
|
37
|
-
var import_assertContinue = require("./assertContinue");
|
|
38
|
-
const inputOutputPath = async (sourcePath) => {
|
|
39
|
-
const outputPath = await (0, import_prompts.text)({
|
|
40
|
-
message: "Enter the output file path:",
|
|
41
|
-
placeholder: "output.md",
|
|
42
|
-
initialValue: (0, import_utils.sourcePathToOutputPath)(sourcePath),
|
|
43
|
-
validate: (value) => {
|
|
44
|
-
if (value.trim() === "") {
|
|
45
|
-
return "Output path is required";
|
|
46
|
-
}
|
|
47
|
-
if (import_node_fs.default.existsSync(value) && import_node_fs.default.statSync(value).isDirectory()) {
|
|
48
|
-
return "No directory can be specified.";
|
|
49
|
-
}
|
|
50
|
-
if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
|
|
51
|
-
return "No directory can be specified.";
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
|
-
(0, import_assertContinue.assertContinue)(outputPath);
|
|
56
|
-
if (!import_node_fs.default.existsSync(outputPath)) {
|
|
57
|
-
return outputPath;
|
|
58
|
-
}
|
|
59
|
-
const isOutputFileOverwrite = await (0, import_prompts.confirm)({
|
|
60
|
-
message: "The file already exists. Overwrite?",
|
|
61
|
-
initialValue: false
|
|
62
|
-
});
|
|
63
|
-
(0, import_assertContinue.assertContinue)(isOutputFileOverwrite);
|
|
64
|
-
if (isOutputFileOverwrite) {
|
|
65
|
-
return outputPath;
|
|
66
|
-
}
|
|
67
|
-
const escapedOutputPath = await (0, import_prompts.text)({
|
|
68
|
-
message: "Enter the output file path:",
|
|
69
|
-
placeholder: "output.md",
|
|
70
|
-
initialValue: (0, import_utils.getNextAvailableFilePath)(outputPath),
|
|
71
|
-
validate: (value) => {
|
|
72
|
-
if (value.trim() === "") {
|
|
73
|
-
return "Output path is required";
|
|
74
|
-
}
|
|
75
|
-
if (import_node_fs.default.existsSync(value)) {
|
|
76
|
-
return "The file already exists";
|
|
77
|
-
}
|
|
78
|
-
if (!import_node_fs.default.existsSync(value) && value.endsWith("/")) {
|
|
79
|
-
return "No directory can be specified.";
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
});
|
|
83
|
-
(0, import_assertContinue.assertContinue)(escapedOutputPath);
|
|
84
|
-
return escapedOutputPath;
|
|
85
|
-
};
|
|
86
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
87
|
-
0 && (module.exports = {
|
|
88
|
-
inputOutputPath
|
|
89
|
-
});
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var inputSourcePath_exports = {};
|
|
30
|
-
__export(inputSourcePath_exports, {
|
|
31
|
-
inputSourcePath: () => inputSourcePath
|
|
32
|
-
});
|
|
33
|
-
module.exports = __toCommonJS(inputSourcePath_exports);
|
|
34
|
-
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
35
|
-
var import_prompts = require("@clack/prompts");
|
|
36
|
-
var import_constants = require("../constants");
|
|
37
|
-
var import_utils = require("../utils");
|
|
38
|
-
var import_assertContinue = require("./assertContinue");
|
|
39
|
-
const inputSourcePath = async () => {
|
|
40
|
-
const result = await (0, import_prompts.text)({
|
|
41
|
-
message: "Enter the URL or html path to be converted to markdown:",
|
|
42
|
-
placeholder: import_constants.DEFAULT_PATH,
|
|
43
|
-
initialValue: "",
|
|
44
|
-
validate: (value) => {
|
|
45
|
-
if (value.trim() === "") {
|
|
46
|
-
return "Source is required";
|
|
47
|
-
}
|
|
48
|
-
if (!(0, import_utils.isUrl)(value)) {
|
|
49
|
-
if (!import_node_fs.default.existsSync(value)) {
|
|
50
|
-
return "It appears that you are specifying a local file, but the file cannot be found. hint: when specifying a url, start with http or https.";
|
|
51
|
-
}
|
|
52
|
-
if (import_node_fs.default.statSync(value).isDirectory()) {
|
|
53
|
-
return "You are specifying a local file, but you cannot specify a directory. hint: when specifying a url, start with http or https.";
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
});
|
|
58
|
-
(0, import_assertContinue.assertContinue)(result);
|
|
59
|
-
return result;
|
|
60
|
-
};
|
|
61
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
62
|
-
0 && (module.exports = {
|
|
63
|
-
inputSourcePath
|
|
64
|
-
});
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var selectExtractMode_exports = {};
|
|
20
|
-
__export(selectExtractMode_exports, {
|
|
21
|
-
selectExtractMode: () => selectExtractMode
|
|
22
|
-
});
|
|
23
|
-
module.exports = __toCommonJS(selectExtractMode_exports);
|
|
24
|
-
var import_prompts = require("@clack/prompts");
|
|
25
|
-
var import_constants = require("../constants");
|
|
26
|
-
var import_assertContinue = require("./assertContinue");
|
|
27
|
-
const selectExtractMode = async () => {
|
|
28
|
-
const result = await (0, import_prompts.select)({
|
|
29
|
-
message: "Select processing mode:",
|
|
30
|
-
options: import_constants.MODES.map((mode) => ({ value: mode, label: mode })),
|
|
31
|
-
initialValue: import_constants.MODES[0]
|
|
32
|
-
});
|
|
33
|
-
(0, import_assertContinue.assertContinue)(result);
|
|
34
|
-
return result;
|
|
35
|
-
};
|
|
36
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
37
|
-
0 && (module.exports = {
|
|
38
|
-
selectExtractMode
|
|
39
|
-
});
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __defProp = Object.defineProperty;
|
|
3
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
-
var __export = (target, all) => {
|
|
7
|
-
for (var name in all)
|
|
8
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
-
};
|
|
10
|
-
var __copyProps = (to, from, except, desc) => {
|
|
11
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
-
for (let key of __getOwnPropNames(from))
|
|
13
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
-
}
|
|
16
|
-
return to;
|
|
17
|
-
};
|
|
18
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
-
var selectLoader_exports = {};
|
|
20
|
-
__export(selectLoader_exports, {
|
|
21
|
-
selectLoader: () => selectLoader
|
|
22
|
-
});
|
|
23
|
-
module.exports = __toCommonJS(selectLoader_exports);
|
|
24
|
-
var import_prompts = require("@clack/prompts");
|
|
25
|
-
var import_constants = require("../constants");
|
|
26
|
-
var import_assertContinue = require("./assertContinue");
|
|
27
|
-
const loadersHint = {
|
|
28
|
-
fetch: "Fetch HTML content from the given URL",
|
|
29
|
-
playwright: "Retrieve HTML content after rendering using Playwright; Playwright must be installed in advance.",
|
|
30
|
-
puppeteer: "Retrieve HTML content after rendering using Puppeteer; Puppeteer must be installed in advance."
|
|
31
|
-
};
|
|
32
|
-
const selectLoader = async () => {
|
|
33
|
-
const result = await (0, import_prompts.select)({
|
|
34
|
-
message: "Select loader:",
|
|
35
|
-
initialValue: "fetch",
|
|
36
|
-
options: import_constants.LOADERS.map((mode) => ({ value: mode, label: mode, hint: loadersHint[mode] || "" }))
|
|
37
|
-
});
|
|
38
|
-
(0, import_assertContinue.assertContinue)(result);
|
|
39
|
-
return result;
|
|
40
|
-
};
|
|
41
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
42
|
-
0 && (module.exports = {
|
|
43
|
-
selectLoader
|
|
44
|
-
});
|
package/dist/cjs/cli/utils.js
DELETED
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var utils_exports = {};
|
|
30
|
-
__export(utils_exports, {
|
|
31
|
-
changeFileExtension: () => changeFileExtension,
|
|
32
|
-
getNextAvailableFilePath: () => getNextAvailableFilePath,
|
|
33
|
-
isUrl: () => isUrl,
|
|
34
|
-
sourcePathToOutputPath: () => sourcePathToOutputPath,
|
|
35
|
-
urlToFilename: () => urlToFilename
|
|
36
|
-
});
|
|
37
|
-
module.exports = __toCommonJS(utils_exports);
|
|
38
|
-
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
39
|
-
var import_node_path = __toESM(require("node:path"), 1);
|
|
40
|
-
const isUrl = (maybeUrl) => {
|
|
41
|
-
try {
|
|
42
|
-
new URL(maybeUrl);
|
|
43
|
-
return true;
|
|
44
|
-
} catch {
|
|
45
|
-
return false;
|
|
46
|
-
}
|
|
47
|
-
};
|
|
48
|
-
function changeFileExtension(filePath, newExtension) {
|
|
49
|
-
const parsedPath = filePath.split("/");
|
|
50
|
-
const fileName = parsedPath[parsedPath.length - 1];
|
|
51
|
-
const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
|
|
52
|
-
if (fileName.startsWith(".")) {
|
|
53
|
-
const parts = fileName.split(".");
|
|
54
|
-
if (parts.length === 2) {
|
|
55
|
-
return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
|
|
56
|
-
}
|
|
57
|
-
parts[parts.length - 1] = newExtension.replace(/^\./, "");
|
|
58
|
-
return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
|
|
59
|
-
}
|
|
60
|
-
const lastDotIndex = fileName.lastIndexOf(".");
|
|
61
|
-
const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
|
|
62
|
-
const newFileName = `${baseName}${formattedNewExtension}`;
|
|
63
|
-
parsedPath[parsedPath.length - 1] = newFileName;
|
|
64
|
-
return parsedPath.join("/");
|
|
65
|
-
}
|
|
66
|
-
function urlToFilename(url) {
|
|
67
|
-
try {
|
|
68
|
-
const urlObj = new URL(url);
|
|
69
|
-
const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
|
|
70
|
-
if (index === 0) {
|
|
71
|
-
return acc;
|
|
72
|
-
}
|
|
73
|
-
if (acc.length >= 2) {
|
|
74
|
-
return acc;
|
|
75
|
-
}
|
|
76
|
-
if (part === "www") {
|
|
77
|
-
return acc;
|
|
78
|
-
}
|
|
79
|
-
return [part, ...acc];
|
|
80
|
-
}, []);
|
|
81
|
-
const domainString = domainParts.reverse().join("-");
|
|
82
|
-
const pathParts = urlObj.pathname.split("/").filter(Boolean);
|
|
83
|
-
const relevantPathParts = pathParts.slice(-2);
|
|
84
|
-
const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
|
|
85
|
-
let filename = [domainString, pathString].filter(Boolean).join("-");
|
|
86
|
-
filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
|
|
87
|
-
return filename || "output";
|
|
88
|
-
} catch {
|
|
89
|
-
return "output";
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
const sourcePathToOutputPath = (sourcePath) => {
|
|
93
|
-
return isUrl(sourcePath) ? `${urlToFilename(sourcePath)}.md` : changeFileExtension(sourcePath, "md");
|
|
94
|
-
};
|
|
95
|
-
function getNextAvailableFilePath(filePath) {
|
|
96
|
-
const parsedPath = import_node_path.default.parse(filePath);
|
|
97
|
-
const directory = parsedPath.dir;
|
|
98
|
-
const fullName = parsedPath.base;
|
|
99
|
-
const [firstPart, ...restParts] = fullName.split(".");
|
|
100
|
-
const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
|
|
101
|
-
const baseName = firstPart.replace(/_\d+$/, "");
|
|
102
|
-
let counter = 1;
|
|
103
|
-
let nextFilePath = filePath;
|
|
104
|
-
while (import_node_fs.default.existsSync(nextFilePath)) {
|
|
105
|
-
const match = firstPart.match(/_(\d+)$/);
|
|
106
|
-
if (match) {
|
|
107
|
-
counter = Number.parseInt(match[1], 10) + 1;
|
|
108
|
-
}
|
|
109
|
-
const newName = `${baseName}_${counter}${restName}`;
|
|
110
|
-
nextFilePath = import_node_path.default.join(directory, newName);
|
|
111
|
-
counter++;
|
|
112
|
-
}
|
|
113
|
-
return nextFilePath;
|
|
114
|
-
}
|
|
115
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
116
|
-
0 && (module.exports = {
|
|
117
|
-
changeFileExtension,
|
|
118
|
-
getNextAvailableFilePath,
|
|
119
|
-
isUrl,
|
|
120
|
-
sourcePathToOutputPath,
|
|
121
|
-
urlToFilename
|
|
122
|
-
});
|
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var import_fastest_levenshtein = require("fastest-levenshtein");
|
|
3
|
-
var import_vitest = require("vitest");
|
|
4
|
-
var import_html_to_markdown = require("./html-to-markdown");
|
|
5
|
-
var import_fetch = require("./loaders/fetch");
|
|
6
|
-
const html = `
|
|
7
|
-
<h1>Hello, world!</h1>
|
|
8
|
-
<p>This is a paragraph.</p>
|
|
9
|
-
<a href="/example.html">Example</a>
|
|
10
|
-
<img src="/example.jpg" alt="Example" />
|
|
11
|
-
<ul>
|
|
12
|
-
<li>Item 1</li>
|
|
13
|
-
<li>Item 2</li>
|
|
14
|
-
</ul>
|
|
15
|
-
`;
|
|
16
|
-
const expected = `# Hello, world!
|
|
17
|
-
|
|
18
|
-
This is a paragraph.
|
|
19
|
-
|
|
20
|
-
[Example](/example.html)
|
|
21
|
-
|
|
22
|
-

|
|
23
|
-
|
|
24
|
-
* Item 1
|
|
25
|
-
* Item 2
|
|
26
|
-
`;
|
|
27
|
-
const baseUrlReplaced = `# Hello, world!
|
|
28
|
-
|
|
29
|
-
This is a paragraph.
|
|
30
|
-
|
|
31
|
-
[Example](https://example.com/example.html)
|
|
32
|
-
|
|
33
|
-

|
|
34
|
-
|
|
35
|
-
* Item 1
|
|
36
|
-
* Item 2
|
|
37
|
-
`;
|
|
38
|
-
const linkAsText = `# Hello, world!
|
|
39
|
-
|
|
40
|
-
This is a paragraph.
|
|
41
|
-
|
|
42
|
-
Example
|
|
43
|
-
|
|
44
|
-

|
|
45
|
-
|
|
46
|
-
- Item 1
|
|
47
|
-
- Item 2
|
|
48
|
-
`;
|
|
49
|
-
const imageHidden = `# Hello, world!
|
|
50
|
-
|
|
51
|
-
This is a paragraph.
|
|
52
|
-
|
|
53
|
-
[Example](/example.html)
|
|
54
|
-
|
|
55
|
-
- Item 1
|
|
56
|
-
- Item 2
|
|
57
|
-
`;
|
|
58
|
-
const htmlTable = `
|
|
59
|
-
<table>
|
|
60
|
-
<tr>
|
|
61
|
-
<th>Header 1</th>
|
|
62
|
-
<th>Header 2</th>
|
|
63
|
-
</tr>
|
|
64
|
-
<tr>
|
|
65
|
-
<td>Cell 1</td>
|
|
66
|
-
<td>Cell 2</td>
|
|
67
|
-
</tr>
|
|
68
|
-
</table>
|
|
69
|
-
`;
|
|
70
|
-
const expectedTableMarkdown = `
|
|
71
|
-
| Header 1 | Header 2 |
|
|
72
|
-
| -------- | -------- |
|
|
73
|
-
| Cell 1 | Cell 2 |
|
|
74
|
-
`;
|
|
75
|
-
const expectedTableText = `Header 1 Header 2
|
|
76
|
-
Cell 1 Cell 2`;
|
|
77
|
-
(0, import_vitest.describe)("htmlToMarkdown", () => {
|
|
78
|
-
(0, import_vitest.it)("should convert HTML to Markdown", () => {
|
|
79
|
-
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { extractors: false });
|
|
80
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown, expected);
|
|
81
|
-
(0, import_vitest.expect)(d).lte(5);
|
|
82
|
-
});
|
|
83
|
-
(0, import_vitest.it)("should convert HTML to Markdown with replaced base URL", () => {
|
|
84
|
-
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { baseUrl: "https://example.com", extractors: false });
|
|
85
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown, baseUrlReplaced);
|
|
86
|
-
(0, import_vitest.expect)(d).lte(5);
|
|
87
|
-
});
|
|
88
|
-
(0, import_vitest.it)("should convert HTML to Markdown with links as text", () => {
|
|
89
|
-
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { linkAsText: true, extractors: false });
|
|
90
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown, linkAsText);
|
|
91
|
-
(0, import_vitest.expect)(d).lte(5);
|
|
92
|
-
});
|
|
93
|
-
(0, import_vitest.it)("should convert HTML to Markdown with hidden images", () => {
|
|
94
|
-
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(html, { hideImage: true, extractors: false });
|
|
95
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown, imageHidden);
|
|
96
|
-
(0, import_vitest.expect)(d).lte(5);
|
|
97
|
-
});
|
|
98
|
-
(0, import_vitest.it)("should convert HTML table to Markdown table", () => {
|
|
99
|
-
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(htmlTable, { extractors: false });
|
|
100
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown, expectedTableMarkdown);
|
|
101
|
-
(0, import_vitest.expect)(d).lte(5);
|
|
102
|
-
});
|
|
103
|
-
(0, import_vitest.it)("should convert HTML table with table as text option", () => {
|
|
104
|
-
const markdown = (0, import_html_to_markdown.htmlToMarkdown)(htmlTable, { tableAsText: true, extractors: false });
|
|
105
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown, expectedTableText);
|
|
106
|
-
(0, import_vitest.expect)(d).lte(10);
|
|
107
|
-
});
|
|
108
|
-
});
|
|
109
|
-
(0, import_vitest.describe)("htmlToMarkdown E2E", () => {
|
|
110
|
-
(0, import_vitest.it)("Converting for good", async () => {
|
|
111
|
-
const html1 = await (0, import_fetch.loadHtml)("https://www.npmjs.com/package/webforai");
|
|
112
|
-
const markdown1 = (0, import_html_to_markdown.htmlToMarkdown)(html1, { linkAsText: true, hideImage: true });
|
|
113
|
-
const html2 = await (0, import_fetch.loadHtml)("https://github.com/inaridiy/webforai");
|
|
114
|
-
const markdown2 = (0, import_html_to_markdown.htmlToMarkdown)(html2, { linkAsText: true, hideImage: true });
|
|
115
|
-
const d = (0, import_fastest_levenshtein.distance)(markdown1, markdown2);
|
|
116
|
-
(0, import_vitest.expect)(d).lte(2500);
|
|
117
|
-
});
|
|
118
|
-
});
|