single-file-cli 2.1.1 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/compile.sh +5 -5
- package/deno.json +3 -3
- package/deno.lock +14 -14
- package/lib/browser.js +7 -2
- package/lib/cdp-client.js +11 -8
- package/lib/deno-polyfill.js +19 -2
- package/lib/version.js +1 -1
- package/options.js +192 -94
- package/package.json +5 -5
- package/single-file-cli-api.js +39 -19
- package/single-file-launcher.js +5 -93
- package/test/e2e/errors-file.test.js +31 -0
- package/test/e2e/frame-gate.test.js +2 -1
- package/test/e2e/timeouts.test.js +70 -0
- package/test/e2e/urls-file.test.js +37 -0
- package/test/unit/cli-usage.test.js +56 -0
- package/test/unit/options.test.js +136 -0
- package/test/unit/write-file.test.js +37 -0
package/compile.sh
CHANGED
|
@@ -12,11 +12,11 @@ cp -R ./resources "$stage_dir/resources"
|
|
|
12
12
|
mkdir -p "$dist_dir"
|
|
13
13
|
cd "$stage_dir"
|
|
14
14
|
|
|
15
|
-
deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-apple-darwin" --target=aarch64-apple-darwin ./single-file
|
|
16
|
-
deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-apple-darwin" --target=x86_64-apple-darwin ./single-file
|
|
17
|
-
deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-linux" --target=x86_64-unknown-linux-gnu ./single-file
|
|
18
|
-
deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-linux" --target=aarch64-unknown-linux-gnu ./single-file
|
|
19
|
-
deno compile --allow-all --ext=js --output="$dist_dir/single-file.exe" --target=x86_64-pc-windows-msvc --icon=./resources/single-file.ico ./single-file
|
|
15
|
+
deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-apple-darwin" --target=aarch64-apple-darwin ./single-file
|
|
16
|
+
deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-apple-darwin" --target=x86_64-apple-darwin ./single-file
|
|
17
|
+
deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-linux" --target=x86_64-unknown-linux-gnu ./single-file
|
|
18
|
+
deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-linux" --target=aarch64-unknown-linux-gnu ./single-file
|
|
19
|
+
deno compile --minimum-dependency-age=0 --allow-all --ext=js --output="$dist_dir/single-file.exe" --target=x86_64-pc-windows-msvc --icon=./resources/single-file.ico ./single-file
|
|
20
20
|
|
|
21
21
|
cd - > /dev/null
|
|
22
22
|
|
package/deno.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@single-file/single-file-cli",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.2",
|
|
4
4
|
"description": "SingleFile CLI",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./single-file-cli-api.js"
|
|
7
7
|
},
|
|
8
8
|
"imports": {
|
|
9
|
-
"simple-cdp": "jsr:@simple-cdp/simple-cdp@^1.10.
|
|
10
|
-
"path": "jsr:@std/path@^1.1.
|
|
9
|
+
"simple-cdp": "jsr:@simple-cdp/simple-cdp@^1.10.1",
|
|
10
|
+
"path": "jsr:@std/path@^1.1.6"
|
|
11
11
|
}
|
|
12
12
|
}
|
package/deno.lock
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": "5",
|
|
3
3
|
"specifiers": {
|
|
4
|
-
"jsr:@simple-cdp/simple-cdp@^1.10.
|
|
4
|
+
"jsr:@simple-cdp/simple-cdp@^1.10.1": "1.10.1",
|
|
5
5
|
"jsr:@std/internal@^1.0.14": "1.0.14",
|
|
6
|
-
"jsr:@std/path@^1.1.
|
|
7
|
-
"npm:@eslint/js@^9.39.
|
|
8
|
-
"npm:esbuild@~0.27.
|
|
6
|
+
"jsr:@std/path@^1.1.6": "1.1.6",
|
|
7
|
+
"npm:@eslint/js@^9.39.5": "9.39.5",
|
|
8
|
+
"npm:esbuild@~0.27.7": "0.27.7",
|
|
9
9
|
"npm:eslint@^10.8.1": "10.8.1",
|
|
10
|
-
"npm:simple-cdp@^1.10.
|
|
10
|
+
"npm:simple-cdp@^1.10.1": "1.10.1"
|
|
11
11
|
},
|
|
12
12
|
"jsr": {
|
|
13
|
-
"@simple-cdp/simple-cdp@1.10.
|
|
14
|
-
"integrity": "
|
|
13
|
+
"@simple-cdp/simple-cdp@1.10.1": {
|
|
14
|
+
"integrity": "33c686242e812a53d13ba7fcebd18ed81b4685fb02da37c2ee6ed9a5123f9b8c"
|
|
15
15
|
},
|
|
16
16
|
"@std/internal@1.0.14": {
|
|
17
17
|
"integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7"
|
|
@@ -527,8 +527,8 @@
|
|
|
527
527
|
"shebang-regex@3.0.0": {
|
|
528
528
|
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
|
|
529
529
|
},
|
|
530
|
-
"simple-cdp@1.10.
|
|
531
|
-
"integrity": "sha512-
|
|
530
|
+
"simple-cdp@1.10.1": {
|
|
531
|
+
"integrity": "sha512-svGX76zc2MY2ErGAWGb9NWP9aF5XQgDyx00iqAGCgRi1qQd5MQjtBwuC7mcktliFH263gpHdnHicfsB+n7Vnyg=="
|
|
532
532
|
},
|
|
533
533
|
"type-check@0.4.0": {
|
|
534
534
|
"integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
|
|
@@ -558,15 +558,15 @@
|
|
|
558
558
|
},
|
|
559
559
|
"workspace": {
|
|
560
560
|
"dependencies": [
|
|
561
|
-
"jsr:@simple-cdp/simple-cdp@^1.10.
|
|
562
|
-
"jsr:@std/path@^1.1.
|
|
561
|
+
"jsr:@simple-cdp/simple-cdp@^1.10.1",
|
|
562
|
+
"jsr:@std/path@^1.1.6"
|
|
563
563
|
],
|
|
564
564
|
"packageJson": {
|
|
565
565
|
"dependencies": [
|
|
566
|
-
"npm:@eslint/js@^9.39.
|
|
567
|
-
"npm:esbuild@~0.27.
|
|
566
|
+
"npm:@eslint/js@^9.39.5",
|
|
567
|
+
"npm:esbuild@~0.27.7",
|
|
568
568
|
"npm:eslint@^10.8.1",
|
|
569
|
-
"npm:simple-cdp@^1.10.
|
|
569
|
+
"npm:simple-cdp@^1.10.1"
|
|
570
570
|
]
|
|
571
571
|
}
|
|
572
572
|
}
|
package/lib/browser.js
CHANGED
|
@@ -80,8 +80,13 @@ async function launchBrowser(options = {}, indexPath = 0) {
|
|
|
80
80
|
} catch (error) {
|
|
81
81
|
await remove(profilePath, { recursive: true }).catch(() => { });
|
|
82
82
|
profilePath = undefined;
|
|
83
|
-
if (error instanceof errors.NotFound
|
|
84
|
-
|
|
83
|
+
if (error instanceof errors.NotFound) {
|
|
84
|
+
if (!options.executablePath && indexPath + 1 < BROWSER_PATHS[build.os].length) {
|
|
85
|
+
return launchBrowser(options, indexPath + 1);
|
|
86
|
+
}
|
|
87
|
+
throw new Error(options.executablePath ?
|
|
88
|
+
`The browser executable was not found at ${JSON.stringify(executablePath)}` :
|
|
89
|
+
"The browser executable was not found, use --browser-executable-path to set its location");
|
|
85
90
|
}
|
|
86
91
|
throw error;
|
|
87
92
|
}
|
package/lib/cdp-client.js
CHANGED
|
@@ -102,7 +102,7 @@ function relaunchBrowser() {
|
|
|
102
102
|
|
|
103
103
|
async function getPageData(options) {
|
|
104
104
|
const EMPTY_PAGE_URL = "about:blank";
|
|
105
|
-
const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {} };
|
|
105
|
+
const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, fetchAbortController: new AbortController() };
|
|
106
106
|
let targetInfo, cdp;
|
|
107
107
|
try {
|
|
108
108
|
logData(["Loading page", EMPTY_PAGE_URL], pageContext);
|
|
@@ -131,6 +131,7 @@ async function getPageData(options) {
|
|
|
131
131
|
throw error;
|
|
132
132
|
} finally {
|
|
133
133
|
logData(["Closing page"], pageContext);
|
|
134
|
+
pageContext.fetchAbortController.abort();
|
|
134
135
|
await closeTarget();
|
|
135
136
|
logData(["Finishing"], pageContext);
|
|
136
137
|
}
|
|
@@ -454,14 +455,16 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
|
|
|
454
455
|
]),
|
|
455
456
|
waitForTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime, LOAD_TIMEOUT_ERROR_MESSAGE, LOAD_TIMEOUT_ERROR)
|
|
456
457
|
]);
|
|
458
|
+
// only called when the page loaded, these calls can hang forever on an
|
|
459
|
+
// unresponsive page and would mask the load timeout error
|
|
460
|
+
await Page.setLifecycleEventsEnabled({ enabled: false });
|
|
461
|
+
await Runtime.disable();
|
|
462
|
+
await Page.disable();
|
|
457
463
|
return contextId;
|
|
458
464
|
} finally {
|
|
459
465
|
if (!loadTimeoutAbortSignal.aborted) {
|
|
460
466
|
loadTimeoutAbortController.abort();
|
|
461
467
|
}
|
|
462
|
-
await Page.setLifecycleEventsEnabled({ enabled: false });
|
|
463
|
-
await Runtime.disable();
|
|
464
|
-
await Page.disable();
|
|
465
468
|
}
|
|
466
469
|
}
|
|
467
470
|
|
|
@@ -571,7 +574,7 @@ function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages
|
|
|
571
574
|
});
|
|
572
575
|
}
|
|
573
576
|
|
|
574
|
-
async function setupBindings({ Page, Runtime }, contextId, { options, debugMessages }) {
|
|
577
|
+
async function setupBindings({ Page, Runtime }, contextId, { options, debugMessages, fetchAbortController }) {
|
|
575
578
|
await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextId: contextId });
|
|
576
579
|
if (options.embedScreenshot && options.compressContent) {
|
|
577
580
|
await setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages });
|
|
@@ -582,7 +585,7 @@ async function setupBindings({ Page, Runtime }, contextId, { options, debugMessa
|
|
|
582
585
|
await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextId: contextId });
|
|
583
586
|
Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
|
|
584
587
|
if (params.name === FETCH_FUNCTION_NAME) {
|
|
585
|
-
await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages });
|
|
588
|
+
await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController });
|
|
586
589
|
}
|
|
587
590
|
}, { options, debugMessages }));
|
|
588
591
|
}
|
|
@@ -645,12 +648,12 @@ async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMes
|
|
|
645
648
|
}
|
|
646
649
|
}
|
|
647
650
|
|
|
648
|
-
async function handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages }) {
|
|
651
|
+
async function handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController }) {
|
|
649
652
|
const { payload } = params;
|
|
650
653
|
const { requestId, url, options: fetchOptions } = JSON.parse(payload);
|
|
651
654
|
logData(["Fetching URL", url], { options, debugMessages });
|
|
652
655
|
try {
|
|
653
|
-
const response = await fetch(url, fetchOptions);
|
|
656
|
+
const response = await fetch(url, Object.assign({}, fetchOptions, { signal: fetchAbortController.signal }));
|
|
654
657
|
const arrayBuffer = await response.arrayBuffer();
|
|
655
658
|
const base64Data = arrayBufferToBase64(arrayBuffer);
|
|
656
659
|
const result = {
|
package/lib/deno-polyfill.js
CHANGED
|
@@ -53,6 +53,12 @@ const errors = {
|
|
|
53
53
|
super(message);
|
|
54
54
|
this.name = "NotFound";
|
|
55
55
|
}
|
|
56
|
+
},
|
|
57
|
+
AlreadyExists: DENO_RUNTIME_DETECTED ? Deno.errors.AlreadyExists : class AlreadyExists extends Error {
|
|
58
|
+
constructor(message) {
|
|
59
|
+
super(message);
|
|
60
|
+
this.name = "AlreadyExists";
|
|
61
|
+
}
|
|
56
62
|
}
|
|
57
63
|
};
|
|
58
64
|
|
|
@@ -157,7 +163,7 @@ async function writeTextFile(path, data, options = {}) {
|
|
|
157
163
|
if (options.append) {
|
|
158
164
|
return fsPromise.appendFile(path, data, options);
|
|
159
165
|
} else {
|
|
160
|
-
return fsPromise
|
|
166
|
+
return nodeWriteFile(fsPromise, path, data, options);
|
|
161
167
|
}
|
|
162
168
|
}
|
|
163
169
|
}
|
|
@@ -167,7 +173,18 @@ async function writeFile(path, data, options = {}) {
|
|
|
167
173
|
return Deno.writeFile(path, data, options);
|
|
168
174
|
} else {
|
|
169
175
|
const fsPromise = await import(NODE_MODULES["fs"]);
|
|
170
|
-
return fsPromise
|
|
176
|
+
return nodeWriteFile(fsPromise, path, data, options);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function nodeWriteFile(fsPromise, path, data, options) {
|
|
181
|
+
if (options.createNew) {
|
|
182
|
+
options.flag = "wx";
|
|
183
|
+
}
|
|
184
|
+
try {
|
|
185
|
+
return await fsPromise.writeFile(path, data, options);
|
|
186
|
+
} catch (error) {
|
|
187
|
+
throw error.code == "EEXIST" ? new errors.AlreadyExists(error.message) : error;
|
|
171
188
|
}
|
|
172
189
|
}
|
|
173
190
|
|
package/lib/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const version = "2.1.
|
|
1
|
+
export const version = "2.1.2";
|
package/options.js
CHANGED
|
@@ -203,76 +203,155 @@ const OPTIONS_INFO = [{
|
|
|
203
203
|
}];
|
|
204
204
|
|
|
205
205
|
const { args, exit } = Deno;
|
|
206
|
-
|
|
207
|
-
export { options, parseArgs };
|
|
206
|
+
export { getOptions, parseArgs, applySettings, parseUrlsFile };
|
|
208
207
|
|
|
209
|
-
function
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
const optionDescription = optionInfo.description;
|
|
231
|
-
const optionDefaultValue = optionInfo.defaultValue === undefined ? "" : `(default: ${JSON.stringify(optionInfo.defaultValue)})`;
|
|
232
|
-
console.log(` --${optionName}: ${optionDescription} <${optionType}> ${optionDefaultValue}`); // eslint-disable-line no-console
|
|
233
|
-
});
|
|
234
|
-
console.log(""); // eslint-disable-line no-console
|
|
208
|
+
function parseUrlsFile(content) {
|
|
209
|
+
return content.split("\n")
|
|
210
|
+
.map(line => line.trim())
|
|
211
|
+
.filter(line => line)
|
|
212
|
+
.map(line => {
|
|
213
|
+
let optionPosition = line.indexOf(" --");
|
|
214
|
+
if (optionPosition < 0) {
|
|
215
|
+
optionPosition = line.indexOf("\t--");
|
|
216
|
+
}
|
|
217
|
+
if (optionPosition > 0) {
|
|
218
|
+
const url = line.substring(0, optionPosition).trim();
|
|
219
|
+
const { options, positionals, invalidOptions } = parseArgs(tokenizeArgs(line.substring(optionPosition + 1).trim()), false);
|
|
220
|
+
positionals.filter(positional => positional.startsWith("--")).forEach(option =>
|
|
221
|
+
console.warn(`Warning: Unknown option ${option} (${url})`)); // eslint-disable-line no-console
|
|
222
|
+
invalidOptions.forEach(({ name, value }) => console.warn(value === undefined ? // eslint-disable-line no-console
|
|
223
|
+
`Warning: Missing value for --${name} (${url})` :
|
|
224
|
+
`Warning: Invalid value for --${name}: ${JSON.stringify(value)} (${url})`));
|
|
225
|
+
return [url, options];
|
|
226
|
+
} else {
|
|
227
|
+
return line;
|
|
228
|
+
}
|
|
235
229
|
});
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function tokenizeArgs(argsString) {
|
|
233
|
+
const args = [];
|
|
234
|
+
let previousCharacter, previousPreviousCharacter, lastQuoteCharacter;
|
|
235
|
+
let lastCharIndex = 0;
|
|
236
|
+
for (let currentCharIndex = 0; currentCharIndex < argsString.length; currentCharIndex++) {
|
|
237
|
+
const character = argsString[currentCharIndex];
|
|
238
|
+
if (character == lastQuoteCharacter && (previousCharacter != "\\" || previousPreviousCharacter == "\\")) {
|
|
239
|
+
args.push(argsString.substring(lastCharIndex, currentCharIndex));
|
|
240
|
+
lastQuoteCharacter = null;
|
|
241
|
+
lastCharIndex = currentCharIndex + 1;
|
|
242
|
+
} else if (!lastQuoteCharacter) {
|
|
243
|
+
if (character == "'" || character == "\"") {
|
|
244
|
+
lastQuoteCharacter = character;
|
|
245
|
+
lastCharIndex = currentCharIndex + 1;
|
|
246
|
+
} else if (character == " " || character == "\t" || character == "=") {
|
|
247
|
+
if (lastCharIndex < currentCharIndex) {
|
|
248
|
+
args.push(argsString.substring(lastCharIndex, currentCharIndex));
|
|
249
|
+
}
|
|
250
|
+
lastCharIndex = currentCharIndex + 1;
|
|
251
|
+
}
|
|
239
252
|
}
|
|
253
|
+
previousPreviousCharacter = previousCharacter;
|
|
254
|
+
previousCharacter = character;
|
|
255
|
+
}
|
|
256
|
+
if (lastCharIndex < argsString.length) {
|
|
257
|
+
args.push(argsString.substring(lastCharIndex).trim());
|
|
258
|
+
}
|
|
259
|
+
return args;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function applySettings(options, settings, explicitOptions = parseArgs(Array.from(args), false).options) {
|
|
263
|
+
const profiles = settings.profiles || {};
|
|
264
|
+
let profileName = options.settingsFileProfile;
|
|
265
|
+
if (profileName == "default" || profileName === undefined) {
|
|
266
|
+
profileName = "__Default_Settings__";
|
|
267
|
+
} else if (!profiles[profileName]) {
|
|
268
|
+
const profileNames = Object.keys(profiles).filter(name => name != "__Default_Settings__");
|
|
269
|
+
throw new Error(`Unknown profile ${JSON.stringify(profileName)}, available profiles: ${profileNames.join(", ")}`);
|
|
270
|
+
}
|
|
271
|
+
Object.assign(options, profiles[profileName], explicitOptions);
|
|
272
|
+
delete options.settingsFile;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function getOptions() {
|
|
276
|
+
const { positionals, options, invalidOptions } = parseArgs(Array.from(args));
|
|
277
|
+
const unknownOptions = positionals.filter(positional => positional.startsWith("--"));
|
|
278
|
+
const urls = positionals.filter(positional => !positional.startsWith("--"));
|
|
279
|
+
if (options.help) {
|
|
280
|
+
printUsage();
|
|
240
281
|
exit(0);
|
|
241
282
|
}
|
|
242
283
|
if (options.version) {
|
|
243
284
|
console.log(version); // eslint-disable-line no-console
|
|
244
285
|
exit(0);
|
|
245
286
|
}
|
|
246
|
-
|
|
287
|
+
const errorMessages = [];
|
|
288
|
+
unknownOptions.forEach(option => errorMessages.push(`Unknown option ${option}`));
|
|
289
|
+
invalidOptions.forEach(({ name, value }) => errorMessages.push(value === undefined ?
|
|
290
|
+
`Missing value for --${name}` :
|
|
291
|
+
`Invalid value for --${name}: ${JSON.stringify(value)}`));
|
|
292
|
+
if (!urls.length && !options.urlsFile) {
|
|
293
|
+
errorMessages.push("The URL or path of the page to save is required");
|
|
294
|
+
}
|
|
295
|
+
if (urls.length > 2) {
|
|
296
|
+
errorMessages.push(`Unexpected arguments: ${urls.slice(2).join(", ")}`);
|
|
297
|
+
}
|
|
298
|
+
if (errorMessages.length) {
|
|
299
|
+
printUsage();
|
|
300
|
+
errorMessages.forEach(message => console.error(`Error: ${message}`)); // eslint-disable-line no-console
|
|
301
|
+
exit(1);
|
|
302
|
+
}
|
|
303
|
+
return { ...options, url: urls[0], output: urls[1] };
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function printUsage() {
|
|
307
|
+
console.log(USAGE_TEXT + "\n"); // eslint-disable-line no-console
|
|
308
|
+
console.log("Options:"); // eslint-disable-line no-console
|
|
309
|
+
OPTIONS_INFO.forEach(category => {
|
|
310
|
+
const categoryName = CATEGORIES[OPTIONS_INFO.indexOf(category)];
|
|
311
|
+
console.log(` * ${categoryName}:`); // eslint-disable-line no-console
|
|
312
|
+
Object.keys(category).forEach(optionName => {
|
|
313
|
+
const optionInfo = category[optionName];
|
|
314
|
+
let optionType = optionInfo.type;
|
|
315
|
+
if (isArray(optionType)) {
|
|
316
|
+
optionType = optionType.replace("[]", "*");
|
|
317
|
+
}
|
|
318
|
+
const optionDescription = optionInfo.description;
|
|
319
|
+
const optionDefaultValue = optionInfo.defaultValue === undefined ? "" : `(default: ${JSON.stringify(optionInfo.defaultValue)})`;
|
|
320
|
+
console.log(` --${optionName}: ${optionDescription} <${optionType}> ${optionDefaultValue}`); // eslint-disable-line no-console
|
|
321
|
+
});
|
|
322
|
+
console.log(""); // eslint-disable-line no-console
|
|
323
|
+
});
|
|
247
324
|
}
|
|
248
325
|
|
|
249
326
|
function parseArgs(args, setDefaultValues = true) {
|
|
250
327
|
const positionals = [];
|
|
251
328
|
const options = {};
|
|
252
|
-
const
|
|
329
|
+
const invalidOptions = [];
|
|
330
|
+
const result = { positionals, options: {}, invalidOptions };
|
|
253
331
|
let argIndex = 0;
|
|
254
332
|
while (argIndex < args.length) {
|
|
255
333
|
const arg = args[argIndex];
|
|
256
|
-
const {
|
|
257
|
-
if (
|
|
258
|
-
|
|
259
|
-
|
|
334
|
+
const { argValue, option } = parseArg(arg);
|
|
335
|
+
if (option) {
|
|
336
|
+
const optionName = option.name;
|
|
337
|
+
if (options[optionName] === undefined) {
|
|
338
|
+
options[optionName] = [];
|
|
260
339
|
}
|
|
261
|
-
let nextArgName;
|
|
262
340
|
if (argValue === undefined) {
|
|
263
|
-
if (
|
|
264
|
-
argIndex + 1
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
(
|
|
269
|
-
|
|
270
|
-
|
|
341
|
+
if (argIndex + 1 < args.length && !parseArg(args[argIndex + 1]).option) {
|
|
342
|
+
const nextArg = args[argIndex + 1];
|
|
343
|
+
if (isValid(option.info.type, nextArg)) {
|
|
344
|
+
options[optionName].push(nextArg);
|
|
345
|
+
argIndex++;
|
|
346
|
+
} else if (option.info.type.startsWith("number")) {
|
|
347
|
+
invalidOptions.push({ name: optionName, value: nextArg });
|
|
348
|
+
argIndex++;
|
|
349
|
+
}
|
|
271
350
|
}
|
|
272
|
-
} else if (isValid(
|
|
273
|
-
options[
|
|
351
|
+
} else if (isValid(option.info.type, argValue)) {
|
|
352
|
+
options[optionName].push(argValue);
|
|
274
353
|
} else {
|
|
275
|
-
|
|
354
|
+
invalidOptions.push({ name: optionName, value: argValue });
|
|
276
355
|
}
|
|
277
356
|
} else {
|
|
278
357
|
positionals.push(arg);
|
|
@@ -280,34 +359,38 @@ function parseArgs(args, setDefaultValues = true) {
|
|
|
280
359
|
argIndex++;
|
|
281
360
|
}
|
|
282
361
|
Object.keys(options).forEach(optionName => {
|
|
283
|
-
const optionInfo = getOptionInfo(optionName);
|
|
362
|
+
const optionInfo = getOptionInfo(optionName).info;
|
|
284
363
|
const optionKey = getOptionKey(optionName, optionInfo);
|
|
285
364
|
let optionValue = options[optionName];
|
|
286
365
|
const isArrayType = isArray(optionInfo.type);
|
|
366
|
+
if (!optionInfo.type.startsWith("boolean") && !optionValue.length &&
|
|
367
|
+
!invalidOptions.some(invalidOption => invalidOption.name == optionName)) {
|
|
368
|
+
invalidOptions.push({ name: optionName });
|
|
369
|
+
}
|
|
287
370
|
if (optionInfo.type.startsWith("boolean")) {
|
|
288
371
|
optionValue = optionValue.map(value => value == "true");
|
|
289
372
|
optionValue = isArrayType ?
|
|
290
373
|
optionValue.length ? optionValue : true :
|
|
291
|
-
optionValue.length ? optionValue[
|
|
374
|
+
optionValue.length ? optionValue[optionValue.length - 1] : true;
|
|
292
375
|
} else if (optionInfo.type.startsWith("number")) {
|
|
293
376
|
optionValue = optionValue.map(value => Number(value));
|
|
294
377
|
optionValue = isArrayType ?
|
|
295
378
|
optionValue.length ? optionValue : optionInfo.defaultValue || 0 :
|
|
296
|
-
optionValue.length ? optionValue[
|
|
379
|
+
optionValue.length ? optionValue[optionValue.length - 1] : optionInfo.defaultValue || 0;
|
|
297
380
|
} else {
|
|
298
381
|
optionValue = isArrayType ?
|
|
299
382
|
optionValue.length ? optionValue : optionInfo.defaultValue || "" :
|
|
300
|
-
optionValue.length ? optionValue[
|
|
383
|
+
optionValue.length ? optionValue[optionValue.length - 1] : optionInfo.defaultValue || "";
|
|
301
384
|
}
|
|
302
385
|
result.options[optionKey] = optionValue;
|
|
303
386
|
});
|
|
304
387
|
if (setDefaultValues) {
|
|
305
388
|
OPTIONS_INFO.forEach(categoryOptions => {
|
|
306
389
|
Object.keys(categoryOptions).forEach(optionName => {
|
|
307
|
-
const optionInfo =
|
|
390
|
+
const optionInfo = categoryOptions[optionName];
|
|
308
391
|
const optionKey = getOptionKey(optionName, optionInfo);
|
|
309
392
|
if (result.options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
|
|
310
|
-
result.options[optionKey] =
|
|
393
|
+
result.options[optionKey] = optionInfo.defaultValue;
|
|
311
394
|
}
|
|
312
395
|
});
|
|
313
396
|
});
|
|
@@ -332,7 +415,11 @@ function parseArgs(args, setDefaultValues = true) {
|
|
|
332
415
|
}
|
|
333
416
|
if (result.options.browserArgs) {
|
|
334
417
|
const browserArguments = result.options.browserArguments || [];
|
|
335
|
-
|
|
418
|
+
try {
|
|
419
|
+
browserArguments.push(...JSON.parse(result.options.browserArgs));
|
|
420
|
+
} catch {
|
|
421
|
+
invalidOptions.push({ name: "browser-args", value: result.options.browserArgs });
|
|
422
|
+
}
|
|
336
423
|
result.options.browserArgs = browserArguments;
|
|
337
424
|
delete result.options.browserArguments;
|
|
338
425
|
}
|
|
@@ -340,46 +427,14 @@ function parseArgs(args, setDefaultValues = true) {
|
|
|
340
427
|
result.options.browserArgs = result.options.browserArguments;
|
|
341
428
|
delete result.options.browserArguments;
|
|
342
429
|
}
|
|
343
|
-
if (result.options.errorFile) {
|
|
430
|
+
if (result.options.errorFile !== undefined) {
|
|
344
431
|
result.options.errorsFile = result.options.errorFile;
|
|
345
432
|
delete result.options.errorFile;
|
|
346
433
|
}
|
|
347
|
-
if (result.options.errorTracesDisabled) {
|
|
434
|
+
if (result.options.errorTracesDisabled !== undefined) {
|
|
348
435
|
result.options.errorsTracesDisabled = result.options.errorTracesDisabled;
|
|
349
436
|
delete result.options.errorTracesDisabled;
|
|
350
437
|
}
|
|
351
|
-
if (result.options.crawlReplaceUrls) {
|
|
352
|
-
result.options.crawlReplaceURLs = result.options.crawlReplaceUrls;
|
|
353
|
-
delete result.options.crawlReplaceUrls;
|
|
354
|
-
}
|
|
355
|
-
if (result.options.saveOriginalUrls) {
|
|
356
|
-
result.options.saveOriginalURLs = result.options.saveOriginalUrls;
|
|
357
|
-
delete result.options.saveOriginalUrls;
|
|
358
|
-
}
|
|
359
|
-
if (result.options.browserRemoteDebuggingUrl) {
|
|
360
|
-
result.options.browserRemoteDebuggingURL = result.options.browserRemoteDebuggingUrl;
|
|
361
|
-
delete result.options.browserRemoteDebuggingUrl;
|
|
362
|
-
}
|
|
363
|
-
if (result.options.crawlRemoveUrlFragment) {
|
|
364
|
-
result.options.crawlRemoveURLFragment = result.options.crawlRemoveUrlFragment;
|
|
365
|
-
delete result.options.crawlRemoveUrlFragment;
|
|
366
|
-
}
|
|
367
|
-
if (result.options.compressCss) {
|
|
368
|
-
result.options.compressCSS = result.options.compressCss;
|
|
369
|
-
delete result.options.compressCss;
|
|
370
|
-
}
|
|
371
|
-
if (result.options.compressHtml) {
|
|
372
|
-
result.options.compressHTML = result.options.compressHtml;
|
|
373
|
-
delete result.options.compressHtml;
|
|
374
|
-
}
|
|
375
|
-
if (result.options.insertMetaCsp) {
|
|
376
|
-
result.options.insertMetaCSP = result.options.insertMetaCsp;
|
|
377
|
-
delete result.options.insertMetaCsp;
|
|
378
|
-
}
|
|
379
|
-
if (result.options.blockedUrlPatterns) {
|
|
380
|
-
result.options.blockedURLPatterns = result.options.blockedUrlPatterns;
|
|
381
|
-
delete result.options.blockedUrlPatterns;
|
|
382
|
-
}
|
|
383
438
|
if (result.options.filenameReplacedCharacters) {
|
|
384
439
|
const filenameReplacedCharacters = result.options.filenameReplacedCharacters;
|
|
385
440
|
result.options.filenameReplacedCharacters = [];
|
|
@@ -404,6 +459,49 @@ function parseArgs(args, setDefaultValues = true) {
|
|
|
404
459
|
}
|
|
405
460
|
});
|
|
406
461
|
}
|
|
462
|
+
if (result.options.httpHeaders) {
|
|
463
|
+
const headers = {};
|
|
464
|
+
result.options.httpHeaders.forEach(header => {
|
|
465
|
+
const separatorIndex = header.indexOf("=");
|
|
466
|
+
if (separatorIndex <= 0) {
|
|
467
|
+
invalidOptions.push({ name: "http-header", value: header });
|
|
468
|
+
} else {
|
|
469
|
+
headers[header.substring(0, separatorIndex).trim()] = header.substring(separatorIndex + 1).trim();
|
|
470
|
+
}
|
|
471
|
+
});
|
|
472
|
+
result.options.httpHeaders = headers;
|
|
473
|
+
}
|
|
474
|
+
if (result.options.emulateMediaFeatures) {
|
|
475
|
+
result.options.emulateMediaFeatures = result.options.emulateMediaFeatures
|
|
476
|
+
.map(feature => {
|
|
477
|
+
const separatorIndex = feature.indexOf(":");
|
|
478
|
+
if (separatorIndex <= 0) {
|
|
479
|
+
invalidOptions.push({ name: "emulate-media-feature", value: feature });
|
|
480
|
+
} else {
|
|
481
|
+
return {
|
|
482
|
+
name: feature.substring(0, separatorIndex),
|
|
483
|
+
value: feature.substring(separatorIndex + 1)
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
})
|
|
487
|
+
.filter(feature => feature);
|
|
488
|
+
}
|
|
489
|
+
if (result.options.browserCookies) {
|
|
490
|
+
result.options.browserCookies = result.options.browserCookies.map(cookie => {
|
|
491
|
+
const [name, value, domain, path, expires, httpOnly, secure, sameSite, url] = cookie.split(",");
|
|
492
|
+
return {
|
|
493
|
+
name,
|
|
494
|
+
value,
|
|
495
|
+
url,
|
|
496
|
+
domain,
|
|
497
|
+
path,
|
|
498
|
+
secure: secure === "true",
|
|
499
|
+
httpOnly: httpOnly === "true",
|
|
500
|
+
sameSite,
|
|
501
|
+
expires: expires && !isNaN(Number(expires)) ? Number(expires) : undefined
|
|
502
|
+
};
|
|
503
|
+
});
|
|
504
|
+
}
|
|
407
505
|
return result;
|
|
408
506
|
}
|
|
409
507
|
|
|
@@ -423,13 +521,13 @@ function parseArg(arg) {
|
|
|
423
521
|
const parsedArg = arg.match(ARGS_REGEX);
|
|
424
522
|
if (parsedArg && parsedArg.length) {
|
|
425
523
|
let [_, argName, argValue] = parsedArg; // eslint-disable-line no-unused-vars
|
|
426
|
-
const
|
|
524
|
+
const option = getOptionInfo(argName);
|
|
427
525
|
if (argValue !== undefined &&
|
|
428
526
|
((argValue.startsWith("\"") && argValue.endsWith("\"")) ||
|
|
429
527
|
(argValue.startsWith("'") && argValue.endsWith("'")))) {
|
|
430
528
|
argValue = argValue.substring(1, argValue.length - 1);
|
|
431
529
|
}
|
|
432
|
-
return { argName, argValue,
|
|
530
|
+
return { argName, argValue, option };
|
|
433
531
|
} else {
|
|
434
532
|
return {};
|
|
435
533
|
}
|
|
@@ -440,7 +538,7 @@ function getOptionInfo(optionName) {
|
|
|
440
538
|
OPTIONS_INFO.forEach(categoryOptions => {
|
|
441
539
|
Object.keys(categoryOptions).forEach(keyName => {
|
|
442
540
|
if (keyName.toLowerCase() == optionName.toLowerCase() || categoryOptions[keyName].alias == optionName.toLowerCase()) {
|
|
443
|
-
result = categoryOptions[keyName];
|
|
541
|
+
result = { name: keyName, info: categoryOptions[keyName] };
|
|
444
542
|
}
|
|
445
543
|
});
|
|
446
544
|
});
|
|
@@ -455,7 +553,7 @@ function isValid(type, value) {
|
|
|
455
553
|
if (type.startsWith("boolean")) {
|
|
456
554
|
return value == "true" || value == "false";
|
|
457
555
|
} else if (type.startsWith("number")) {
|
|
458
|
-
return !isNaN(value);
|
|
556
|
+
return value.trim() != "" && !isNaN(value);
|
|
459
557
|
} else {
|
|
460
558
|
return true;
|
|
461
559
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "single-file-cli",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.2",
|
|
4
4
|
"description": "SingleFile CLI",
|
|
5
5
|
"author": "Gildas Lormeau",
|
|
6
6
|
"engines": {
|
|
@@ -10,18 +10,18 @@
|
|
|
10
10
|
},
|
|
11
11
|
"type": "module",
|
|
12
12
|
"scripts": {
|
|
13
|
-
"test": "node --test \"test/**/*.test.js\"",
|
|
13
|
+
"test": "node --test --test-concurrency=2 \"test/**/*.test.js\"",
|
|
14
14
|
"lint": "eslint ."
|
|
15
15
|
},
|
|
16
16
|
"bin": {
|
|
17
17
|
"single-file": "single-file-node.js"
|
|
18
18
|
},
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"simple-cdp": "^1.10.
|
|
20
|
+
"simple-cdp": "^1.10.1"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|
|
23
|
-
"@eslint/js": "^9.39.
|
|
24
|
-
"esbuild": "^0.27.
|
|
23
|
+
"@eslint/js": "^9.39.5",
|
|
24
|
+
"esbuild": "^0.27.7",
|
|
25
25
|
"eslint": "^10.8.1"
|
|
26
26
|
}
|
|
27
27
|
}
|
package/single-file-cli-api.js
CHANGED
|
@@ -106,7 +106,6 @@ async function initialize(options) {
|
|
|
106
106
|
|
|
107
107
|
async function capture(urls, options) {
|
|
108
108
|
let newTasks;
|
|
109
|
-
const taskUrls = tasks.map(task => task.url);
|
|
110
109
|
newTasks = await Promise.all(urls.map(value => {
|
|
111
110
|
let url, taskOptions;
|
|
112
111
|
if (Array.isArray(value)) {
|
|
@@ -118,7 +117,7 @@ async function capture(urls, options) {
|
|
|
118
117
|
}
|
|
119
118
|
return createTask(url, taskOptions);
|
|
120
119
|
}));
|
|
121
|
-
newTasks = newTasks.filter(task => task && !
|
|
120
|
+
newTasks = newTasks.filter((task, taskIndex) => task && !mergeDuplicateTask(task, tasks.concat(newTasks.slice(0, taskIndex))));
|
|
122
121
|
if (newTasks.length) {
|
|
123
122
|
tasks = tasks.concat(newTasks);
|
|
124
123
|
await saveTasks();
|
|
@@ -300,37 +299,27 @@ async function capturePage(options) {
|
|
|
300
299
|
pageData.comment = undefined;
|
|
301
300
|
content = JSON.stringify(pageData, null, 2);
|
|
302
301
|
}
|
|
303
|
-
if (options.output) {
|
|
304
|
-
filename = await getFilename(options.output, options);
|
|
305
|
-
} else if (options.dumpContent) {
|
|
302
|
+
if (options.dumpContent && !options.output) {
|
|
306
303
|
if (options.compressContent) {
|
|
307
304
|
await stdout.write(content);
|
|
308
305
|
} else {
|
|
309
306
|
console.log(content || ""); // eslint-disable-line no-console
|
|
310
307
|
}
|
|
311
308
|
} else {
|
|
312
|
-
|
|
309
|
+
let outputFilename = options.output || pageData.filename;
|
|
310
|
+
if (options.outputJson && !outputFilename.endsWith(".json")) {
|
|
311
|
+
outputFilename += ".json";
|
|
312
|
+
}
|
|
313
|
+
filename = await writeOutputFile(outputFilename, content, options);
|
|
313
314
|
}
|
|
314
315
|
if (filename) {
|
|
315
|
-
if (options.outputJson) {
|
|
316
|
-
filename += filename.endsWith(".json") ? "" : ".json";
|
|
317
|
-
}
|
|
318
|
-
const directoryName = path.dirname(filename);
|
|
319
|
-
if (directoryName !== ".") {
|
|
320
|
-
await mkdir(directoryName, { recursive: true });
|
|
321
|
-
}
|
|
322
|
-
if (content instanceof Uint8Array) {
|
|
323
|
-
await writeFile(filename, content);
|
|
324
|
-
} else {
|
|
325
|
-
await writeTextFile(filename, content);
|
|
326
|
-
}
|
|
327
316
|
const outputDirectory = getOutputDirectory(options);
|
|
328
317
|
pageData.filename = filename.startsWith(outputDirectory) ? filename.substring(outputDirectory.length) : filename;
|
|
329
318
|
}
|
|
330
319
|
return pageData;
|
|
331
320
|
} catch (error) {
|
|
332
321
|
const date = new Date();
|
|
333
|
-
let message = `[${date.toISOString()}] URL: ${options.url}`;
|
|
322
|
+
let message = `[${date.toISOString()}] URL: ${options.url} Error: ${error.message || error}`;
|
|
334
323
|
if (!options.errorsTracesDisabled) {
|
|
335
324
|
message += "\nStack: " + error.stack;
|
|
336
325
|
}
|
|
@@ -366,6 +355,37 @@ function getOutputDirectory(options) {
|
|
|
366
355
|
return outputDirectory;
|
|
367
356
|
}
|
|
368
357
|
|
|
358
|
+
async function writeOutputFile(outputFilename, content, options) {
|
|
359
|
+
while (true) {
|
|
360
|
+
const filename = await getFilename(outputFilename, options);
|
|
361
|
+
if (!filename) {
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
const directoryName = path.dirname(filename);
|
|
365
|
+
if (directoryName !== ".") {
|
|
366
|
+
await mkdir(directoryName, { recursive: true });
|
|
367
|
+
}
|
|
368
|
+
// exclusive creation prevents parallel tasks writing the same filename
|
|
369
|
+
// from silently overwriting each other
|
|
370
|
+
const writeOptions = { createNew: options.filenameConflictAction != "overwrite" };
|
|
371
|
+
try {
|
|
372
|
+
if (content instanceof Uint8Array) {
|
|
373
|
+
await writeFile(filename, content, writeOptions);
|
|
374
|
+
} else {
|
|
375
|
+
await writeTextFile(filename, content, writeOptions);
|
|
376
|
+
}
|
|
377
|
+
return filename;
|
|
378
|
+
} catch (error) {
|
|
379
|
+
if (!(error instanceof errors.AlreadyExists)) {
|
|
380
|
+
throw error;
|
|
381
|
+
}
|
|
382
|
+
if (options.filenameConflictAction == "skip") {
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
369
389
|
async function getFilename(filename, options, index = 1) {
|
|
370
390
|
let newFilename = getOutputDirectory(options) + filename;
|
|
371
391
|
if (options.filenameConflictAction == "overwrite") {
|
package/single-file-launcher.js
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
import { initialize } from "./single-file-cli-api.js";
|
|
25
25
|
import { closeBrowser } from "./lib/browser.js";
|
|
26
26
|
import { Deno } from "./lib/deno-polyfill.js";
|
|
27
|
-
import {
|
|
27
|
+
import { getOptions, applySettings, parseUrlsFile } from "./options.js";
|
|
28
28
|
|
|
29
29
|
const { readTextFile, readFile, exit, addSignalListener } = Deno;
|
|
30
30
|
|
|
@@ -43,39 +43,17 @@ export { run };
|
|
|
43
43
|
|
|
44
44
|
async function run() {
|
|
45
45
|
try {
|
|
46
|
+
const options = getOptions();
|
|
46
47
|
let urls;
|
|
47
48
|
if (options.settingsFile) {
|
|
48
|
-
const settings = JSON.parse(await
|
|
49
|
-
|
|
50
|
-
if (profileName == "default" || !settings.profiles[profileName]) {
|
|
51
|
-
profileName = "__Default_Settings__";
|
|
52
|
-
}
|
|
53
|
-
Object.assign(options, settings.profiles[profileName]);
|
|
54
|
-
delete options.settingsFile;
|
|
49
|
+
const settings = JSON.parse(await readTextFile(options.settingsFile));
|
|
50
|
+
applySettings(options, settings);
|
|
55
51
|
}
|
|
56
52
|
if (options.urlsFile) {
|
|
57
53
|
urls = await getUrlsFile(options.urlsFile);
|
|
58
54
|
} else {
|
|
59
55
|
urls = [options.url];
|
|
60
56
|
}
|
|
61
|
-
if (options.browserCookies) {
|
|
62
|
-
const cookies = [];
|
|
63
|
-
for (const cookie of options.browserCookies) {
|
|
64
|
-
const [name, value, domain, path, expires, httpOnly, secure, sameSite, url] = cookie.split(",");
|
|
65
|
-
cookies.push({
|
|
66
|
-
name,
|
|
67
|
-
value,
|
|
68
|
-
url,
|
|
69
|
-
domain,
|
|
70
|
-
path,
|
|
71
|
-
secure: secure === "true",
|
|
72
|
-
httpOnly: httpOnly === "true",
|
|
73
|
-
sameSite,
|
|
74
|
-
expires: isNaN(Number(expires)) ? undefined : Number(expires)
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
options.browserCookies = cookies;
|
|
78
|
-
}
|
|
79
57
|
if (options.browserCookiesFile) {
|
|
80
58
|
const cookiesContent = await readTextFile(options.browserCookiesFile);
|
|
81
59
|
try {
|
|
@@ -84,23 +62,6 @@ async function run() {
|
|
|
84
62
|
options.browserCookies = parseCookies(cookiesContent);
|
|
85
63
|
}
|
|
86
64
|
}
|
|
87
|
-
if (options.emulateMediaFeatures) {
|
|
88
|
-
options.emulateMediaFeatures = options.emulateMediaFeatures.map(feature => {
|
|
89
|
-
const colonIndex = feature.indexOf(":");
|
|
90
|
-
return {
|
|
91
|
-
name: feature.substring(0, colonIndex),
|
|
92
|
-
value: feature.substring(colonIndex + 1)
|
|
93
|
-
};
|
|
94
|
-
});
|
|
95
|
-
}
|
|
96
|
-
if (options.httpHeaders) {
|
|
97
|
-
const headers = {};
|
|
98
|
-
for (const header of options.httpHeaders) {
|
|
99
|
-
const [name, value] = header.split("=");
|
|
100
|
-
headers[name] = value.trim();
|
|
101
|
-
}
|
|
102
|
-
options.httpHeaders = headers;
|
|
103
|
-
}
|
|
104
65
|
if (options.embeddedImage) {
|
|
105
66
|
options.embeddedImage = Array.from(await readFile(options.embeddedImage));
|
|
106
67
|
}
|
|
@@ -148,54 +109,5 @@ async function closeBrowserAndExit(code) {
|
|
|
148
109
|
}
|
|
149
110
|
|
|
150
111
|
async function getUrlsFile(urlsFile) {
|
|
151
|
-
|
|
152
|
-
urls = urls.map(value => {
|
|
153
|
-
value = value.trim();
|
|
154
|
-
let optionPosition = value.indexOf(" --");
|
|
155
|
-
if (optionPosition < 0) {
|
|
156
|
-
optionPosition = value.indexOf("\t--");
|
|
157
|
-
}
|
|
158
|
-
if (optionPosition > 0) {
|
|
159
|
-
const url = value.substring(0, optionPosition).trim();
|
|
160
|
-
const argsString = value.substring(optionPosition + 1).trim();
|
|
161
|
-
const args = [];
|
|
162
|
-
let previousCharacter, previousPreviousCharacter, lastQuoteCharacter;
|
|
163
|
-
let lastCharIndex = 0;
|
|
164
|
-
for (let currentCharIndex = 0; currentCharIndex < argsString.length; currentCharIndex++) {
|
|
165
|
-
const character = argsString[currentCharIndex];
|
|
166
|
-
if (character == lastQuoteCharacter && (previousCharacter != "\\" || previousPreviousCharacter == "\\")) {
|
|
167
|
-
args.push(argsString.substring(lastCharIndex, currentCharIndex));
|
|
168
|
-
lastQuoteCharacter = null;
|
|
169
|
-
lastCharIndex = currentCharIndex + 1;
|
|
170
|
-
} else if (!lastQuoteCharacter) {
|
|
171
|
-
if (character == "'" || character == "\"") {
|
|
172
|
-
lastQuoteCharacter = argsString[currentCharIndex];
|
|
173
|
-
lastCharIndex = currentCharIndex + 1;
|
|
174
|
-
} else {
|
|
175
|
-
const isSpaceCharacter = character == " " || character == "\t";
|
|
176
|
-
if (isSpaceCharacter || character == "=") {
|
|
177
|
-
if (isSpaceCharacter && (currentCharIndex == lastCharIndex + 1)) {
|
|
178
|
-
lastCharIndex++;
|
|
179
|
-
} else if (lastCharIndex < currentCharIndex) {
|
|
180
|
-
args.push(argsString.substring(lastCharIndex, currentCharIndex));
|
|
181
|
-
lastCharIndex = currentCharIndex + 1;
|
|
182
|
-
} else {
|
|
183
|
-
lastCharIndex = currentCharIndex + 1;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
previousPreviousCharacter = previousCharacter;
|
|
189
|
-
previousCharacter = character;
|
|
190
|
-
}
|
|
191
|
-
if (lastCharIndex < argsString.length) {
|
|
192
|
-
args.push(argsString.substring(lastCharIndex).trim());
|
|
193
|
-
}
|
|
194
|
-
const { options } = parseArgs(args, false);
|
|
195
|
-
return [url, options];
|
|
196
|
-
} else {
|
|
197
|
-
return value;
|
|
198
|
-
}
|
|
199
|
-
});
|
|
200
|
-
return urls;
|
|
112
|
+
return parseUrlsFile(await readTextFile(urlsFile));
|
|
201
113
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
6
|
+
import { tmpdir } from "node:os";
|
|
7
|
+
import { join, dirname } from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
import process from "node:process";
|
|
10
|
+
|
|
11
|
+
const execFileAsync = promisify(execFile);
|
|
12
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
13
|
+
|
|
14
|
+
test("errors file lines include the error message", { timeout: 120000 }, async () => {
|
|
15
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
16
|
+
try {
|
|
17
|
+
const errorsPath = join(directory, "errors.txt");
|
|
18
|
+
const url = "http://localhost:1/";
|
|
19
|
+
await execFileAsync(process.execPath, [
|
|
20
|
+
"single-file-node.js", url, join(directory, "out.html"),
|
|
21
|
+
"--errors-file", errorsPath
|
|
22
|
+
], { cwd: cliDirectory });
|
|
23
|
+
const content = await readFile(errorsPath, "utf8");
|
|
24
|
+
assert.ok(content.includes("URL: " + url));
|
|
25
|
+
const errorMessage = content.match(/Error: (.*)/);
|
|
26
|
+
assert.ok(errorMessage);
|
|
27
|
+
assert.ok(errorMessage[1].trim().length > 0);
|
|
28
|
+
} finally {
|
|
29
|
+
await rm(directory, { recursive: true });
|
|
30
|
+
}
|
|
31
|
+
});
|
|
@@ -51,7 +51,8 @@ test("capture waits for the top frame, not a fast iframe", { timeout: 120000 },
|
|
|
51
51
|
await execFileAsync(process.execPath, [
|
|
52
52
|
"single-file-node.js", url, outputPath,
|
|
53
53
|
"--browser-wait-until", "networkIdle",
|
|
54
|
-
"--browser-wait-until-delay", "1000"
|
|
54
|
+
"--browser-wait-until-delay", "1000",
|
|
55
|
+
"--browser-wait-until-fallback", "false"
|
|
55
56
|
], { cwd: cliDirectory });
|
|
56
57
|
const content = await readFile(outputPath, "utf8");
|
|
57
58
|
assert.ok(content.includes(LOAD_MARKER), "page was captured before the top frame finished loading");
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/* global setInterval, clearInterval */
|
|
2
|
+
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { createServer } from "node:http";
|
|
6
|
+
import { execFile } from "node:child_process";
|
|
7
|
+
import { promisify } from "node:util";
|
|
8
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
9
|
+
import { Buffer } from "node:buffer";
|
|
10
|
+
import { tmpdir } from "node:os";
|
|
11
|
+
import { join, dirname } from "node:path";
|
|
12
|
+
import { fileURLToPath } from "node:url";
|
|
13
|
+
import process from "node:process";
|
|
14
|
+
|
|
15
|
+
const execFileAsync = promisify(execFile);
|
|
16
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
17
|
+
|
|
18
|
+
test("load timeout is reported when the page never loads", { timeout: 120000 }, async () => {
|
|
19
|
+
const server = createServer(() => { });
|
|
20
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
21
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
22
|
+
try {
|
|
23
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
24
|
+
const { stderr } = await runCli([
|
|
25
|
+
url, join(directory, "out.html"),
|
|
26
|
+
"--browser-load-max-time", "5000",
|
|
27
|
+
"--browser-wait-until-fallback", "false"
|
|
28
|
+
]);
|
|
29
|
+
assert.ok(stderr.includes("Load timeout"), "stderr: " + stderr);
|
|
30
|
+
} finally {
|
|
31
|
+
await rm(directory, { recursive: true });
|
|
32
|
+
server.closeAllConnections();
|
|
33
|
+
server.close();
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("capture timeout is reported when capturing an endless download", { timeout: 120000 }, async () => {
|
|
38
|
+
const chunk = Buffer.alloc(65536);
|
|
39
|
+
const server = createServer((_, response) => {
|
|
40
|
+
response.writeHead(200, { "content-type": "application/octet-stream", "content-disposition": "attachment; filename=big.bin" });
|
|
41
|
+
const intervalId = setInterval(() => response.write(chunk), 50);
|
|
42
|
+
response.on("close", () => clearInterval(intervalId));
|
|
43
|
+
});
|
|
44
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
45
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
46
|
+
try {
|
|
47
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
48
|
+
const { stderr } = await runCli([
|
|
49
|
+
url, join(directory, "out.html"),
|
|
50
|
+
"--browser-load-max-time", "5000",
|
|
51
|
+
"--browser-capture-max-time", "10000"
|
|
52
|
+
]);
|
|
53
|
+
assert.ok(stderr.includes("Capture timeout") || stderr.includes("Load timeout"), "stderr: " + stderr);
|
|
54
|
+
} finally {
|
|
55
|
+
await rm(directory, { recursive: true });
|
|
56
|
+
server.closeAllConnections();
|
|
57
|
+
server.close();
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
async function runCli(args) {
|
|
62
|
+
try {
|
|
63
|
+
return await execFileAsync(process.execPath, ["single-file-node.js", ...args], { cwd: cliDirectory, timeout: 90000, killSignal: "SIGKILL" });
|
|
64
|
+
} catch (error) {
|
|
65
|
+
if (error.killed) {
|
|
66
|
+
throw new Error("the process did not exit before the timeout");
|
|
67
|
+
}
|
|
68
|
+
return { stdout: error.stdout, stderr: error.stderr };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { execFile } from "node:child_process";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import { join, dirname } from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import process from "node:process";
|
|
11
|
+
|
|
12
|
+
const execFileAsync = promisify(execFile);
|
|
13
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
14
|
+
|
|
15
|
+
test("duplicate urls in a urls file are captured once", { timeout: 120000 }, async () => {
|
|
16
|
+
const server = createServer((_, response) => response
|
|
17
|
+
.writeHead(200, { "content-type": "text/html" })
|
|
18
|
+
.end("<html><head><title>Same Page</title></head><body>content</body></html>"));
|
|
19
|
+
await new Promise(resolve => server.listen(0, "localhost", resolve));
|
|
20
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
21
|
+
try {
|
|
22
|
+
const url = "http://localhost:" + server.address().port + "/";
|
|
23
|
+
const urlsFilePath = join(directory, "urls.txt");
|
|
24
|
+
await writeFile(urlsFilePath, url + "\n\n" + url + "\n");
|
|
25
|
+
const { stderr } = await execFileAsync(process.execPath, [
|
|
26
|
+
"single-file-node.js",
|
|
27
|
+
"--urls-file", urlsFilePath,
|
|
28
|
+
"--output-directory", directory,
|
|
29
|
+
"--filename-template", "{page-title}.html"
|
|
30
|
+
], { cwd: cliDirectory });
|
|
31
|
+
const files = (await readdir(directory)).filter(file => file.endsWith(".html"));
|
|
32
|
+
assert.deepEqual(files, ["Same Page.html"], "stderr: " + stderr);
|
|
33
|
+
} finally {
|
|
34
|
+
await rm(directory, { recursive: true });
|
|
35
|
+
server.close();
|
|
36
|
+
}
|
|
37
|
+
});
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { join, dirname } from "node:path";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
import process from "node:process";
|
|
8
|
+
|
|
9
|
+
const execFileAsync = promisify(execFile);
|
|
10
|
+
const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
11
|
+
|
|
12
|
+
async function runCli(args) {
|
|
13
|
+
try {
|
|
14
|
+
const { stdout, stderr } = await execFileAsync(process.execPath, ["single-file-node.js", ...args], { cwd: cliDirectory });
|
|
15
|
+
return { code: 0, stdout, stderr };
|
|
16
|
+
} catch (error) {
|
|
17
|
+
return { code: error.code, stdout: error.stdout, stderr: error.stderr };
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
test("help is displayed with an exit code of 0", async () => {
|
|
22
|
+
const { code, stdout } = await runCli(["--help"]);
|
|
23
|
+
assert.equal(code, 0);
|
|
24
|
+
assert.ok(stdout.includes("--browser-executable-path"));
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("a missing url is reported as an error", async () => {
|
|
28
|
+
const { code, stderr } = await runCli(["--browser-executable-path", "/path/to/chrome"]);
|
|
29
|
+
assert.equal(code, 1);
|
|
30
|
+
assert.ok(stderr.includes("The URL or path of the page to save is required"));
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("unknown options are reported as errors", async () => {
|
|
34
|
+
const { code, stderr } = await runCli(["--foo", "--bar", "https://example.com", "out.html"]);
|
|
35
|
+
assert.equal(code, 1);
|
|
36
|
+
assert.ok(stderr.includes("Unknown option --foo"));
|
|
37
|
+
assert.ok(stderr.includes("Unknown option --bar"));
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("invalid option values are reported as errors", async () => {
|
|
41
|
+
const { code, stderr } = await runCli(["--browser-width", "abc", "https://example.com"]);
|
|
42
|
+
assert.equal(code, 1);
|
|
43
|
+
assert.ok(stderr.includes("Invalid value for --browser-width: \"abc\""));
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("a wrong browser executable path is reported with the path", async () => {
|
|
47
|
+
const { code, stderr } = await runCli(["https://localhost:1/", "out.html", "--browser-executable-path", "/nonexistent/chrome"]);
|
|
48
|
+
assert.notEqual(code, 0);
|
|
49
|
+
assert.ok(stderr.includes("The browser executable was not found at \"/nonexistent/chrome\""));
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("unexpected extra arguments are reported as errors", async () => {
|
|
53
|
+
const { code, stderr } = await runCli(["https://example.com", "out.html", "extra.html"]);
|
|
54
|
+
assert.equal(code, 1);
|
|
55
|
+
assert.ok(stderr.includes("Unexpected arguments: extra.html"));
|
|
56
|
+
});
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { parseArgs, applySettings, parseUrlsFile } from "../../options.js";
|
|
4
|
+
|
|
5
|
+
const parse = args => parseArgs(args).options;
|
|
6
|
+
|
|
7
|
+
test("boolean options with uppercase canonical names can be disabled", () => {
|
|
8
|
+
assert.equal(parse(["--compress-html", "false"]).compressHTML, false);
|
|
9
|
+
assert.equal(parse(["--compress-HTML", "false"]).compressHTML, false);
|
|
10
|
+
assert.equal(parse(["--crawl-remove-url-fragment", "false"]).crawlRemoveURLFragment, false);
|
|
11
|
+
assert.equal(parse(["--insert-meta-csp", "false"]).insertMetaCSP, false);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test("option names are canonicalized regardless of case", () => {
|
|
15
|
+
assert.equal(parse(["--include-bom"]).includeBOM, true);
|
|
16
|
+
assert.equal(parse(["--Include-BOM"]).includeBOM, true);
|
|
17
|
+
assert.equal(parse(["--crawl-replace-urls"]).crawlReplaceURLs, true);
|
|
18
|
+
assert.equal(parse(["--save-original-urls"]).saveOriginalURLs, true);
|
|
19
|
+
assert.deepEqual(parse(["--blocked-url-pattern", "ads"]).blockedURLPatterns, ["ads"]);
|
|
20
|
+
const options = parse(["--compress-html", "false"]);
|
|
21
|
+
assert.equal("compressHtml" in options, false);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test("aliases map to the canonical option", () => {
|
|
25
|
+
assert.equal(parse(["--error-file", "errors.txt"]).errorsFile, "errors.txt");
|
|
26
|
+
assert.equal(parse(["--errors-file", "errors.txt"]).errorsFile, "errors.txt");
|
|
27
|
+
assert.equal(parse(["--error-traces-disabled", "false"]).errorsTracesDisabled, false);
|
|
28
|
+
assert.equal(parse(["--errors-traces-disabled", "false"]).errorsTracesDisabled, false);
|
|
29
|
+
assert.equal(parse([]).errorsTracesDisabled, true);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("browser arguments are merged", () => {
|
|
33
|
+
assert.deepEqual(parse(["--browser-arg", "--disable-gpu"]).browserArgs, ["--disable-gpu"]);
|
|
34
|
+
assert.deepEqual(parse(["--browser-args", "[\"--mute-audio\"]"]).browserArgs, ["--mute-audio"]);
|
|
35
|
+
assert.deepEqual(parse(["--browser-arg", "--disable-gpu", "--browser-args", "[\"--mute-audio\"]"]).browserArgs, ["--disable-gpu", "--mute-audio"]);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("default values are applied under canonical keys", () => {
|
|
39
|
+
const options = parse([]);
|
|
40
|
+
assert.equal(options.compressHTML, true);
|
|
41
|
+
assert.equal(options.crawlRemoveURLFragment, true);
|
|
42
|
+
assert.equal(options.insertMetaCSP, true);
|
|
43
|
+
assert.equal(options.browserWidth, 1280);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test("invalid and missing values are reported", () => {
|
|
47
|
+
assert.deepEqual(parseArgs(["--browser-width", "abc"]).invalidOptions, [{ name: "browser-width", value: "abc" }]);
|
|
48
|
+
assert.deepEqual(parseArgs(["--browser-width=abc"]).invalidOptions, [{ name: "browser-width", value: "abc" }]);
|
|
49
|
+
assert.deepEqual(parseArgs(["--browser-height="]).invalidOptions, [{ name: "browser-height", value: "" }]);
|
|
50
|
+
assert.deepEqual(parseArgs(["--output-directory"]).invalidOptions, [{ name: "output-directory" }]);
|
|
51
|
+
assert.deepEqual(parseArgs(["--browser-width", "abc"]).positionals, []);
|
|
52
|
+
assert.deepEqual(parseArgs(["--browser-debug"]).invalidOptions, []);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("the last value wins when a scalar option is repeated", () => {
|
|
56
|
+
const { options, positionals } = parseArgs(["--browser-width", "100", "--browser-width", "200"]);
|
|
57
|
+
assert.equal(options.browserWidth, 200);
|
|
58
|
+
assert.deepEqual(positionals, []);
|
|
59
|
+
assert.equal(parse(["--browser-width=100", "--browser-width=200"]).browserWidth, 200);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("http headers keep characters after the first equals sign", () => {
|
|
63
|
+
const options = parse(["--http-header", "authorization=Basic dGVzdA==", "--http-header", " x-test = value "]);
|
|
64
|
+
assert.deepEqual(options.httpHeaders, { authorization: "Basic dGVzdA==", "x-test": "value" });
|
|
65
|
+
assert.deepEqual(parseArgs(["--http-header", "no-separator"]).invalidOptions, [{ name: "http-header", value: "no-separator" }]);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("media features are split on the first colon", () => {
|
|
69
|
+
assert.deepEqual(parse(["--emulate-media-feature", "prefers-color-scheme:dark"]).emulateMediaFeatures, [{ name: "prefers-color-scheme", value: "dark" }]);
|
|
70
|
+
assert.deepEqual(parseArgs(["--emulate-media-feature", "no-separator"]).invalidOptions, [{ name: "emulate-media-feature", value: "no-separator" }]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("browser cookies are parsed into objects", () => {
|
|
74
|
+
const options = parse(["--browser-cookie", "name,value,example.com,/,,true,true,None,https://example.com"]);
|
|
75
|
+
assert.deepEqual(options.browserCookies, [{
|
|
76
|
+
name: "name",
|
|
77
|
+
value: "value",
|
|
78
|
+
url: "https://example.com",
|
|
79
|
+
domain: "example.com",
|
|
80
|
+
path: "/",
|
|
81
|
+
secure: true,
|
|
82
|
+
httpOnly: true,
|
|
83
|
+
sameSite: "None",
|
|
84
|
+
expires: undefined
|
|
85
|
+
}]);
|
|
86
|
+
assert.equal(parse(["--browser-cookie", "name,value,example.com,/,1893456000,false,false,Lax,https://example.com"]).browserCookies[0].expires, 1893456000);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("invalid browser args json is reported", () => {
|
|
90
|
+
assert.deepEqual(parseArgs(["--browser-args", "not json"]).invalidOptions, [{ name: "browser-args", value: "not json" }]);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("explicit options win over settings file profiles", () => {
|
|
94
|
+
const options = parse(["--settings-file-profile", "custom", "--browser-width", "1024"]);
|
|
95
|
+
const settings = { profiles: { custom: { compressHTML: false, browserWidth: 800 } } };
|
|
96
|
+
applySettings(options, settings, { browserWidth: 1024 });
|
|
97
|
+
assert.equal(options.compressHTML, false);
|
|
98
|
+
assert.equal(options.browserWidth, 1024);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("the default profile maps to the default settings", () => {
|
|
102
|
+
const options = parse([]);
|
|
103
|
+
applySettings(options, { profiles: { __Default_Settings__: { compressHTML: false } } }, {});
|
|
104
|
+
assert.equal(options.compressHTML, false);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("an unknown settings profile is reported", () => {
|
|
108
|
+
const options = parse(["--settings-file-profile", "missing"]);
|
|
109
|
+
assert.throws(
|
|
110
|
+
() => applySettings(options, { profiles: { __Default_Settings__: {}, work: {} } }, {}),
|
|
111
|
+
/Unknown profile "missing", available profiles: work/);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("blank lines in urls files are skipped", () => {
|
|
115
|
+
assert.deepEqual(parseUrlsFile("https://a.example\n\n \nhttps://b.example\r\n"), ["https://a.example", "https://b.example"]);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("single character values in urls file options are kept", () => {
|
|
119
|
+
const [url, options] = parseUrlsFile("https://a.example --crawl-max-depth 2 --browser-width 100")[0];
|
|
120
|
+
assert.equal(url, "https://a.example");
|
|
121
|
+
assert.equal(options.crawlMaxDepth, 2);
|
|
122
|
+
assert.equal(options.browserWidth, 100);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("quoted and equal form values in urls files are parsed", () => {
|
|
126
|
+
const [, options] = parseUrlsFile("https://a.example --filename-template \"My Page.html\" --browser-height=600")[0];
|
|
127
|
+
assert.equal(options.filenameTemplate, "My Page.html");
|
|
128
|
+
assert.equal(options.browserHeight, 600);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("values are parsed from both spaced and equal forms", () => {
|
|
132
|
+
assert.equal(parse(["--browser-width", "1024"]).browserWidth, 1024);
|
|
133
|
+
assert.equal(parse(["--browser-width=1024"]).browserWidth, 1024);
|
|
134
|
+
assert.equal(parse(["--browser-headless=false"]).browserHeadless, false);
|
|
135
|
+
assert.equal(parse(["--filename-template={page-title}.html"]).filenameTemplate, "{page-title}.html");
|
|
136
|
+
});
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { Deno } from "../../lib/deno-polyfill.js";
|
|
7
|
+
|
|
8
|
+
const { writeTextFile, writeFile, readTextFile, errors } = Deno;
|
|
9
|
+
|
|
10
|
+
test("exclusive creation fails when the file exists", async () => {
|
|
11
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
12
|
+
try {
|
|
13
|
+
const filename = join(directory, "out.txt");
|
|
14
|
+
await writeTextFile(filename, "first", { createNew: true });
|
|
15
|
+
await assert.rejects(
|
|
16
|
+
() => writeTextFile(filename, "second", { createNew: true }),
|
|
17
|
+
error => error instanceof errors.AlreadyExists);
|
|
18
|
+
await assert.rejects(
|
|
19
|
+
() => writeFile(filename, new Uint8Array([1]), { createNew: true }),
|
|
20
|
+
error => error instanceof errors.AlreadyExists);
|
|
21
|
+
assert.equal(await readTextFile(filename), "first");
|
|
22
|
+
} finally {
|
|
23
|
+
await rm(directory, { recursive: true });
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("writing without exclusive creation overwrites", async () => {
|
|
28
|
+
const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
|
|
29
|
+
try {
|
|
30
|
+
const filename = join(directory, "out.txt");
|
|
31
|
+
await writeTextFile(filename, "first");
|
|
32
|
+
await writeTextFile(filename, "second");
|
|
33
|
+
assert.equal(await readTextFile(filename), "second");
|
|
34
|
+
} finally {
|
|
35
|
+
await rm(directory, { recursive: true });
|
|
36
|
+
}
|
|
37
|
+
});
|