webforai 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +972 -213
- package/dist/cjs/cli/bin.js +33 -0
- package/dist/cjs/cli/commands/webforai/index.js +86 -0
- package/dist/cjs/cli/commands/webforai/loadHtml.js +93 -0
- package/dist/cjs/cli/constants.js +34 -0
- package/dist/cjs/cli/helpers/assertContinue.js +34 -0
- package/dist/cjs/cli/helpers/inputOutputPath.js +89 -0
- package/dist/cjs/cli/helpers/inputSourcePath.js +64 -0
- package/dist/cjs/cli/helpers/selectExtractMode.js +39 -0
- package/dist/cjs/cli/helpers/selectLoader.js +44 -0
- package/dist/cjs/cli/utils.js +89 -0
- package/dist/cjs/extract-hast/index.js +4 -3
- package/dist/cjs/extract-hast/readability.js +28 -61
- package/dist/cjs/extract-hast/strongly.js +2 -1
- package/dist/cjs/extract-hast/utils.js +0 -15
- package/dist/cjs/html-to-mdast.js +6 -6
- package/dist/cjs/link-replacer.js +10 -5
- package/dist/cjs/loaders/playwright.js +36 -7
- package/dist/cjs/mdast-handlers/custom-br-handler.js +31 -0
- package/dist/cjs/mdast-to-markdown.js +1 -1
- package/dist/cjs/utils/hast-utils.js +71 -0
- package/dist/cli/bin.js +11 -0
- package/dist/cli/commands/webforai/index.js +53 -0
- package/dist/cli/commands/webforai/loadHtml.js +60 -0
- package/dist/cli/constants.js +9 -0
- package/dist/cli/helpers/assertContinue.js +11 -0
- package/dist/cli/helpers/inputOutputPath.js +56 -0
- package/dist/cli/helpers/inputSourcePath.js +31 -0
- package/dist/cli/helpers/selectExtractMode.js +16 -0
- package/dist/cli/helpers/selectLoader.js +21 -0
- package/dist/cli/utils.js +52 -0
- package/dist/extract-hast/index.js +4 -3
- package/dist/extract-hast/readability.js +29 -62
- package/dist/extract-hast/strongly.js +2 -1
- package/dist/extract-hast/utils.js +0 -14
- package/dist/html-to-mdast.js +6 -6
- package/dist/link-replacer.js +10 -5
- package/dist/loaders/playwright.js +36 -7
- package/dist/mdast-handlers/custom-br-handler.js +8 -0
- package/dist/mdast-to-markdown.js +1 -1
- package/dist/types/cli/bin.js.map +1 -0
- package/dist/types/cli/commands/webforai/index.d.ts +6 -0
- package/dist/types/cli/commands/webforai/index.js.map +1 -0
- package/dist/types/cli/commands/webforai/loadHtml.d.ts +3 -0
- package/dist/types/cli/commands/webforai/loadHtml.js.map +1 -0
- package/dist/types/cli/constants.d.ts +5 -0
- package/dist/types/cli/constants.js.map +1 -0
- package/dist/types/cli/helpers/assertContinue.d.ts +1 -0
- package/dist/types/cli/helpers/assertContinue.js.map +1 -0
- package/dist/types/cli/helpers/inputOutputPath.d.ts +1 -0
- package/dist/types/cli/helpers/inputOutputPath.js.map +1 -0
- package/dist/types/cli/helpers/inputSourcePath.d.ts +1 -0
- package/dist/types/cli/helpers/inputSourcePath.js.map +1 -0
- package/dist/types/cli/helpers/selectExtractMode.d.ts +1 -0
- package/dist/types/cli/helpers/selectExtractMode.js.map +1 -0
- package/dist/types/cli/helpers/selectLoader.d.ts +1 -0
- package/dist/types/cli/helpers/selectLoader.js.map +1 -0
- package/dist/types/cli/utils.d.ts +32 -0
- package/dist/types/cli/utils.js.map +1 -0
- package/dist/types/extract-hast/index.d.ts +9 -4
- package/dist/types/extract-hast/index.js.map +1 -1
- package/dist/types/extract-hast/readability.d.ts +2 -1
- package/dist/types/extract-hast/readability.js.map +1 -1
- package/dist/types/extract-hast/strongly.d.ts +2 -1
- package/dist/types/extract-hast/strongly.js.map +1 -1
- package/dist/types/extract-hast/utils.d.ts +0 -1
- package/dist/types/extract-hast/utils.js.map +1 -1
- package/dist/types/html-to-mdast.d.ts +4 -0
- package/dist/types/html-to-mdast.js.map +1 -1
- package/dist/types/link-replacer.d.ts +1 -1
- package/dist/types/link-replacer.js.map +1 -1
- package/dist/types/loaders/playwright.d.ts +8 -2
- package/dist/types/loaders/playwright.js.map +1 -1
- package/dist/types/mdast-handlers/custom-br-handler.d.ts +2 -0
- package/dist/types/mdast-handlers/custom-br-handler.js.map +1 -0
- package/dist/types/mdast-to-markdown.js.map +1 -1
- package/dist/types/utils/hast-utils.d.ts +4 -0
- package/dist/types/utils/hast-utils.js.map +1 -0
- package/dist/utils/hast-utils.js +46 -0
- package/package.json +18 -17
- package/dist/cjs/bin.js +0 -254
- package/dist/cjs/html-to-markdown.test.js +0 -146
- package/dist/cjs/link-replacer.test.js +0 -47
- package/dist/cjs/loaders/fetch.test.js +0 -9
- package/dist/cjs/loaders/playwright.test.js +0 -17
- package/dist/cjs/loaders/puppeteer.test.js +0 -17
- package/dist/cjs/utils/bin-utils.js +0 -124
- package/dist/html-to-markdown.test.js +0 -124
- package/dist/link-replacer.test.js +0 -47
- package/dist/loaders/fetch.test.js +0 -9
- package/dist/loaders/playwright.test.js +0 -17
- package/dist/loaders/puppeteer.test.js +0 -17
- package/dist/types/bin.js.map +0 -1
- package/dist/types/utils/bin-utils.d.ts +0 -5
- package/dist/types/utils/bin-utils.js.map +0 -1
- package/dist/utils/bin-utils.js +0 -87
- /package/dist/types/{bin.d.ts → cli/bin.d.ts} +0 -0
package/dist/utils/bin-utils.js
DELETED
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
// src/utils/bin-utils.ts
|
|
2
|
-
import fs from "node:fs";
|
|
3
|
-
import path from "node:path";
|
|
4
|
-
import { log } from "@clack/prompts";
|
|
5
|
-
function logError(error) {
|
|
6
|
-
log.error("An error occurred during processing");
|
|
7
|
-
console.error(error.message);
|
|
8
|
-
}
|
|
9
|
-
function isUrl(str) {
|
|
10
|
-
try {
|
|
11
|
-
new URL(str);
|
|
12
|
-
return true;
|
|
13
|
-
} catch {
|
|
14
|
-
return false;
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
function getNextAvailableFilePath(filePath) {
|
|
18
|
-
const parsedPath = path.parse(filePath);
|
|
19
|
-
const directory = parsedPath.dir;
|
|
20
|
-
const fullName = parsedPath.base;
|
|
21
|
-
const [firstPart, ...restParts] = fullName.split(".");
|
|
22
|
-
const restName = restParts.length > 0 ? `.${restParts.join(".")}` : "";
|
|
23
|
-
const baseName = firstPart.replace(/_\d+$/, "");
|
|
24
|
-
let counter = 1;
|
|
25
|
-
let nextFilePath = filePath;
|
|
26
|
-
while (fs.existsSync(nextFilePath)) {
|
|
27
|
-
const match = firstPart.match(/_(\d+)$/);
|
|
28
|
-
if (match) {
|
|
29
|
-
counter = Number.parseInt(match[1], 10) + 1;
|
|
30
|
-
}
|
|
31
|
-
const newName = `${baseName}_${counter}${restName}`;
|
|
32
|
-
nextFilePath = path.join(directory, newName);
|
|
33
|
-
counter++;
|
|
34
|
-
}
|
|
35
|
-
return nextFilePath;
|
|
36
|
-
}
|
|
37
|
-
function changeFileExtension(filePath, newExtension) {
|
|
38
|
-
const parsedPath = filePath.split("/");
|
|
39
|
-
const fileName = parsedPath[parsedPath.length - 1];
|
|
40
|
-
const formattedNewExtension = newExtension.startsWith(".") ? newExtension : `.${newExtension}`;
|
|
41
|
-
if (fileName.startsWith(".")) {
|
|
42
|
-
const parts = fileName.split(".");
|
|
43
|
-
if (parts.length === 2) {
|
|
44
|
-
return parsedPath.slice(0, -1).concat(`${fileName}${formattedNewExtension}`).join("/");
|
|
45
|
-
}
|
|
46
|
-
parts[parts.length - 1] = newExtension.replace(/^\./, "");
|
|
47
|
-
return parsedPath.slice(0, -1).concat(parts.join(".")).join("/");
|
|
48
|
-
}
|
|
49
|
-
const lastDotIndex = fileName.lastIndexOf(".");
|
|
50
|
-
const baseName = lastDotIndex !== -1 ? fileName.slice(0, lastDotIndex) : fileName;
|
|
51
|
-
const newFileName = `${baseName}${formattedNewExtension}`;
|
|
52
|
-
parsedPath[parsedPath.length - 1] = newFileName;
|
|
53
|
-
return parsedPath.join("/");
|
|
54
|
-
}
|
|
55
|
-
function urlToFilename(url) {
|
|
56
|
-
try {
|
|
57
|
-
const urlObj = new URL(url);
|
|
58
|
-
const domainParts = urlObj.hostname.split(".").reverse().reduce((acc, part, index) => {
|
|
59
|
-
if (index === 0) {
|
|
60
|
-
return acc;
|
|
61
|
-
}
|
|
62
|
-
if (acc.length >= 2) {
|
|
63
|
-
return acc;
|
|
64
|
-
}
|
|
65
|
-
if (part === "www") {
|
|
66
|
-
return acc;
|
|
67
|
-
}
|
|
68
|
-
return [part, ...acc];
|
|
69
|
-
}, []);
|
|
70
|
-
const domainString = domainParts.reverse().join("-");
|
|
71
|
-
const pathParts = urlObj.pathname.split("/").filter(Boolean);
|
|
72
|
-
const relevantPathParts = pathParts.slice(-2);
|
|
73
|
-
const pathString = relevantPathParts.map((part) => decodeURIComponent(part)).join("-");
|
|
74
|
-
let filename = [domainString, pathString].filter(Boolean).join("-");
|
|
75
|
-
filename = filename.toLowerCase().replace(/[<>:"/\\|?*\x00-\x1F]/g, "").replace(/[\s.]+/g, "-").replace(/^-+|-+$/g, "");
|
|
76
|
-
return filename || "output";
|
|
77
|
-
} catch {
|
|
78
|
-
return "output";
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
export {
|
|
82
|
-
changeFileExtension,
|
|
83
|
-
getNextAvailableFilePath,
|
|
84
|
-
isUrl,
|
|
85
|
-
logError,
|
|
86
|
-
urlToFilename
|
|
87
|
-
};
|
|
File without changes
|