single-file-cli 2.1.1 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/compile.sh CHANGED
@@ -12,11 +12,11 @@ cp -R ./resources "$stage_dir/resources"
12
12
  mkdir -p "$dist_dir"
13
13
  cd "$stage_dir"
14
14
 
15
- deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-apple-darwin" --target=aarch64-apple-darwin ./single-file
16
- deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-apple-darwin" --target=x86_64-apple-darwin ./single-file
17
- deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-linux" --target=x86_64-unknown-linux-gnu ./single-file
18
- deno compile --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-linux" --target=aarch64-unknown-linux-gnu ./single-file
19
- deno compile --allow-all --ext=js --output="$dist_dir/single-file.exe" --target=x86_64-pc-windows-msvc --icon=./resources/single-file.ico ./single-file
15
+ deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-apple-darwin" --target=aarch64-apple-darwin ./single-file
16
+ deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-apple-darwin" --target=x86_64-apple-darwin ./single-file
17
+ deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-x86_64-linux" --target=x86_64-unknown-linux-gnu ./single-file
18
+ deno compile --minimum-dependency-age=0 --allow-read --allow-write --allow-net --allow-env --allow-run --ext=js --output="$dist_dir/single-file-aarch64-linux" --target=aarch64-unknown-linux-gnu ./single-file
19
+ deno compile --minimum-dependency-age=0 --allow-all --ext=js --output="$dist_dir/single-file.exe" --target=x86_64-pc-windows-msvc --icon=./resources/single-file.ico ./single-file
20
20
 
21
21
  cd - > /dev/null
22
22
 
package/deno.json CHANGED
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@single-file/single-file-cli",
3
- "version": "2.1.1",
3
+ "version": "2.1.2",
4
4
  "description": "SingleFile CLI",
5
5
  "exports": {
6
6
  ".": "./single-file-cli-api.js"
7
7
  },
8
8
  "imports": {
9
- "simple-cdp": "jsr:@simple-cdp/simple-cdp@^1.10.0",
10
- "path": "jsr:@std/path@^1.1.2"
9
+ "simple-cdp": "jsr:@simple-cdp/simple-cdp@^1.10.1",
10
+ "path": "jsr:@std/path@^1.1.6"
11
11
  }
12
12
  }
package/deno.lock CHANGED
@@ -1,17 +1,17 @@
1
1
  {
2
2
  "version": "5",
3
3
  "specifiers": {
4
- "jsr:@simple-cdp/simple-cdp@^1.10.0": "1.10.0",
4
+ "jsr:@simple-cdp/simple-cdp@^1.10.1": "1.10.1",
5
5
  "jsr:@std/internal@^1.0.14": "1.0.14",
6
- "jsr:@std/path@^1.1.2": "1.1.6",
7
- "npm:@eslint/js@^9.39.1": "9.39.5",
8
- "npm:esbuild@~0.27.4": "0.27.7",
6
+ "jsr:@std/path@^1.1.6": "1.1.6",
7
+ "npm:@eslint/js@^9.39.5": "9.39.5",
8
+ "npm:esbuild@~0.27.7": "0.27.7",
9
9
  "npm:eslint@^10.8.1": "10.8.1",
10
- "npm:simple-cdp@^1.10.0": "1.10.0"
10
+ "npm:simple-cdp@^1.10.1": "1.10.1"
11
11
  },
12
12
  "jsr": {
13
- "@simple-cdp/simple-cdp@1.10.0": {
14
- "integrity": "3f8c91e22598c3e15a680f14ee3d2aa174c21b9b528caad5ec8552e3df0c3897"
13
+ "@simple-cdp/simple-cdp@1.10.1": {
14
+ "integrity": "33c686242e812a53d13ba7fcebd18ed81b4685fb02da37c2ee6ed9a5123f9b8c"
15
15
  },
16
16
  "@std/internal@1.0.14": {
17
17
  "integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7"
@@ -527,8 +527,8 @@
527
527
  "shebang-regex@3.0.0": {
528
528
  "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
529
529
  },
530
- "simple-cdp@1.10.0": {
531
- "integrity": "sha512-UXDAaiges1GqNKtBy39po+CD4hXDJxQEDtR1GF9bK2J9MNN0epTo2GpoSmEc8o5ofW8dutpHafBjowEchSNgig=="
530
+ "simple-cdp@1.10.1": {
531
+ "integrity": "sha512-svGX76zc2MY2ErGAWGb9NWP9aF5XQgDyx00iqAGCgRi1qQd5MQjtBwuC7mcktliFH263gpHdnHicfsB+n7Vnyg=="
532
532
  },
533
533
  "type-check@0.4.0": {
534
534
  "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==",
@@ -558,15 +558,15 @@
558
558
  },
559
559
  "workspace": {
560
560
  "dependencies": [
561
- "jsr:@simple-cdp/simple-cdp@^1.10.0",
562
- "jsr:@std/path@^1.1.2"
561
+ "jsr:@simple-cdp/simple-cdp@^1.10.1",
562
+ "jsr:@std/path@^1.1.6"
563
563
  ],
564
564
  "packageJson": {
565
565
  "dependencies": [
566
- "npm:@eslint/js@^9.39.1",
567
- "npm:esbuild@~0.27.4",
566
+ "npm:@eslint/js@^9.39.5",
567
+ "npm:esbuild@~0.27.7",
568
568
  "npm:eslint@^10.8.1",
569
- "npm:simple-cdp@^1.10.0"
569
+ "npm:simple-cdp@^1.10.1"
570
570
  ]
571
571
  }
572
572
  }
package/lib/browser.js CHANGED
@@ -80,8 +80,13 @@ async function launchBrowser(options = {}, indexPath = 0) {
80
80
  } catch (error) {
81
81
  await remove(profilePath, { recursive: true }).catch(() => { });
82
82
  profilePath = undefined;
83
- if (error instanceof errors.NotFound && indexPath + 1 < BROWSER_PATHS[build.os].length) {
84
- return launchBrowser(options, indexPath + 1);
83
+ if (error instanceof errors.NotFound) {
84
+ if (!options.executablePath && indexPath + 1 < BROWSER_PATHS[build.os].length) {
85
+ return launchBrowser(options, indexPath + 1);
86
+ }
87
+ throw new Error(options.executablePath ?
88
+ `The browser executable was not found at ${JSON.stringify(executablePath)}` :
89
+ "The browser executable was not found, use --browser-executable-path to set its location");
85
90
  }
86
91
  throw error;
87
92
  }
package/lib/cdp-client.js CHANGED
@@ -102,7 +102,7 @@ function relaunchBrowser() {
102
102
 
103
103
  async function getPageData(options) {
104
104
  const EMPTY_PAGE_URL = "about:blank";
105
- const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {} };
105
+ const pageContext = { options, consoleMessages: [], debugMessages: [], httpInfo: {}, fetchAbortController: new AbortController() };
106
106
  let targetInfo, cdp;
107
107
  try {
108
108
  logData(["Loading page", EMPTY_PAGE_URL], pageContext);
@@ -131,6 +131,7 @@ async function getPageData(options) {
131
131
  throw error;
132
132
  } finally {
133
133
  logData(["Closing page"], pageContext);
134
+ pageContext.fetchAbortController.abort();
134
135
  await closeTarget();
135
136
  logData(["Finishing"], pageContext);
136
137
  }
@@ -454,14 +455,16 @@ async function loadPage({ Page, Runtime }, { options, debugMessages }) {
454
455
  ]),
455
456
  waitForTimeout(loadTimeoutAbortSignal, options.browserLoadMaxTime, LOAD_TIMEOUT_ERROR_MESSAGE, LOAD_TIMEOUT_ERROR)
456
457
  ]);
458
+ // only called when the page loaded, these calls can hang forever on an
459
+ // unresponsive page and would mask the load timeout error
460
+ await Page.setLifecycleEventsEnabled({ enabled: false });
461
+ await Runtime.disable();
462
+ await Page.disable();
457
463
  return contextId;
458
464
  } finally {
459
465
  if (!loadTimeoutAbortSignal.aborted) {
460
466
  loadTimeoutAbortController.abort();
461
467
  }
462
- await Page.setLifecycleEventsEnabled({ enabled: false });
463
- await Runtime.disable();
464
- await Page.disable();
465
468
  }
466
469
  }
467
470
 
@@ -571,7 +574,7 @@ function setupPageDataCapture({ Runtime }, _contextId, { options, debugMessages
571
574
  });
572
575
  }
573
576
 
574
- async function setupBindings({ Page, Runtime }, contextId, { options, debugMessages }) {
577
+ async function setupBindings({ Page, Runtime }, contextId, { options, debugMessages, fetchAbortController }) {
575
578
  await Runtime.addBinding({ name: SET_PAGE_DATA_FUNCTION_NAME, executionContextId: contextId });
576
579
  if (options.embedScreenshot && options.compressContent) {
577
580
  await setupScreenshotCapture({ Page, Runtime }, contextId, { options, debugMessages });
@@ -582,7 +585,7 @@ async function setupBindings({ Page, Runtime }, contextId, { options, debugMessa
582
585
  await Runtime.addBinding({ name: FETCH_FUNCTION_NAME, executionContextId: contextId });
583
586
  Runtime.addEventListener(BINDING_CALLED_EVENT_TYPE, ignoringErrors(async ({ params }) => {
584
587
  if (params.name === FETCH_FUNCTION_NAME) {
585
- await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages });
588
+ await handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController });
586
589
  }
587
590
  }, { options, debugMessages }));
588
591
  }
@@ -645,12 +648,12 @@ async function setupPdfCapture({ Page, Runtime }, contextId, { options, debugMes
645
648
  }
646
649
  }
647
650
 
648
- async function handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages }) {
651
+ async function handleFetchRequest({ Runtime }, params, contextId, { options, debugMessages, fetchAbortController }) {
649
652
  const { payload } = params;
650
653
  const { requestId, url, options: fetchOptions } = JSON.parse(payload);
651
654
  logData(["Fetching URL", url], { options, debugMessages });
652
655
  try {
653
- const response = await fetch(url, fetchOptions);
656
+ const response = await fetch(url, Object.assign({}, fetchOptions, { signal: fetchAbortController.signal }));
654
657
  const arrayBuffer = await response.arrayBuffer();
655
658
  const base64Data = arrayBufferToBase64(arrayBuffer);
656
659
  const result = {
@@ -53,6 +53,12 @@ const errors = {
53
53
  super(message);
54
54
  this.name = "NotFound";
55
55
  }
56
+ },
57
+ AlreadyExists: DENO_RUNTIME_DETECTED ? Deno.errors.AlreadyExists : class AlreadyExists extends Error {
58
+ constructor(message) {
59
+ super(message);
60
+ this.name = "AlreadyExists";
61
+ }
56
62
  }
57
63
  };
58
64
 
@@ -157,7 +163,7 @@ async function writeTextFile(path, data, options = {}) {
157
163
  if (options.append) {
158
164
  return fsPromise.appendFile(path, data, options);
159
165
  } else {
160
- return fsPromise.writeFile(path, data, options);
166
+ return nodeWriteFile(fsPromise, path, data, options);
161
167
  }
162
168
  }
163
169
  }
@@ -167,7 +173,18 @@ async function writeFile(path, data, options = {}) {
167
173
  return Deno.writeFile(path, data, options);
168
174
  } else {
169
175
  const fsPromise = await import(NODE_MODULES["fs"]);
170
- return fsPromise.writeFile(path, data, options);
176
+ return nodeWriteFile(fsPromise, path, data, options);
177
+ }
178
+ }
179
+
180
+ async function nodeWriteFile(fsPromise, path, data, options) {
181
+ if (options.createNew) {
182
+ options.flag = "wx";
183
+ }
184
+ try {
185
+ return await fsPromise.writeFile(path, data, options);
186
+ } catch (error) {
187
+ throw error.code == "EEXIST" ? new errors.AlreadyExists(error.message) : error;
171
188
  }
172
189
  }
173
190
 
package/lib/version.js CHANGED
@@ -1 +1 @@
1
- export const version = "2.1.1";
1
+ export const version = "2.1.2";
package/options.js CHANGED
@@ -203,76 +203,155 @@ const OPTIONS_INFO = [{
203
203
  }];
204
204
 
205
205
  const { args, exit } = Deno;
206
- const options = getOptions();
207
- export { options, parseArgs };
206
+ export { getOptions, parseArgs, applySettings, parseUrlsFile };
208
207
 
209
- function getOptions() {
210
- const { positionals, options } = parseArgs(Array.from(args));
211
- const unknownOptions = [];
212
- positionals.forEach(positional => {
213
- if (positional.startsWith("--")) {
214
- unknownOptions.push(positional);
215
- positionals.splice(positionals.indexOf(positional), 1);
216
- }
217
- });
218
- if ((!positionals.length || positionals.length > 2 || options.help || unknownOptions.length) && (!options.version && !options.urlsFile)) {
219
- console.log(USAGE_TEXT + "\n"); // eslint-disable-line no-console
220
- console.log("Options:"); // eslint-disable-line no-console
221
- OPTIONS_INFO.forEach(category => {
222
- const categoryName = CATEGORIES[OPTIONS_INFO.indexOf(category)];
223
- console.log(` * ${categoryName}:`); // eslint-disable-line no-console
224
- Object.keys(category).forEach(optionName => {
225
- const optionInfo = getOptionInfo(optionName);
226
- let optionType = optionInfo.type;
227
- if (isArray(optionType)) {
228
- optionType = optionType.replace("[]", "*");
229
- }
230
- const optionDescription = optionInfo.description;
231
- const optionDefaultValue = optionInfo.defaultValue === undefined ? "" : `(default: ${JSON.stringify(optionInfo.defaultValue)})`;
232
- console.log(` --${optionName}: ${optionDescription} <${optionType}> ${optionDefaultValue}`); // eslint-disable-line no-console
233
- });
234
- console.log(""); // eslint-disable-line no-console
208
+ function parseUrlsFile(content) {
209
+ return content.split("\n")
210
+ .map(line => line.trim())
211
+ .filter(line => line)
212
+ .map(line => {
213
+ let optionPosition = line.indexOf(" --");
214
+ if (optionPosition < 0) {
215
+ optionPosition = line.indexOf("\t--");
216
+ }
217
+ if (optionPosition > 0) {
218
+ const url = line.substring(0, optionPosition).trim();
219
+ const { options, positionals, invalidOptions } = parseArgs(tokenizeArgs(line.substring(optionPosition + 1).trim()), false);
220
+ positionals.filter(positional => positional.startsWith("--")).forEach(option =>
221
+ console.warn(`Warning: Unknown option ${option} (${url})`)); // eslint-disable-line no-console
222
+ invalidOptions.forEach(({ name, value }) => console.warn(value === undefined ? // eslint-disable-line no-console
223
+ `Warning: Missing value for --${name} (${url})` :
224
+ `Warning: Invalid value for --${name}: ${JSON.stringify(value)} (${url})`));
225
+ return [url, options];
226
+ } else {
227
+ return line;
228
+ }
235
229
  });
236
- if (unknownOptions.length) {
237
- console.log(`Error: Unknown option${unknownOptions.length > 1 ? "s" : ""} ${unknownOptions.join(", ")}`); // eslint-disable-line no-console
238
- console.log(""); // eslint-disable-line no-console
230
+ }
231
+
232
+ function tokenizeArgs(argsString) {
233
+ const args = [];
234
+ let previousCharacter, previousPreviousCharacter, lastQuoteCharacter;
235
+ let lastCharIndex = 0;
236
+ for (let currentCharIndex = 0; currentCharIndex < argsString.length; currentCharIndex++) {
237
+ const character = argsString[currentCharIndex];
238
+ if (character == lastQuoteCharacter && (previousCharacter != "\\" || previousPreviousCharacter == "\\")) {
239
+ args.push(argsString.substring(lastCharIndex, currentCharIndex));
240
+ lastQuoteCharacter = null;
241
+ lastCharIndex = currentCharIndex + 1;
242
+ } else if (!lastQuoteCharacter) {
243
+ if (character == "'" || character == "\"") {
244
+ lastQuoteCharacter = character;
245
+ lastCharIndex = currentCharIndex + 1;
246
+ } else if (character == " " || character == "\t" || character == "=") {
247
+ if (lastCharIndex < currentCharIndex) {
248
+ args.push(argsString.substring(lastCharIndex, currentCharIndex));
249
+ }
250
+ lastCharIndex = currentCharIndex + 1;
251
+ }
239
252
  }
253
+ previousPreviousCharacter = previousCharacter;
254
+ previousCharacter = character;
255
+ }
256
+ if (lastCharIndex < argsString.length) {
257
+ args.push(argsString.substring(lastCharIndex).trim());
258
+ }
259
+ return args;
260
+ }
261
+
262
+ function applySettings(options, settings, explicitOptions = parseArgs(Array.from(args), false).options) {
263
+ const profiles = settings.profiles || {};
264
+ let profileName = options.settingsFileProfile;
265
+ if (profileName == "default" || profileName === undefined) {
266
+ profileName = "__Default_Settings__";
267
+ } else if (!profiles[profileName]) {
268
+ const profileNames = Object.keys(profiles).filter(name => name != "__Default_Settings__");
269
+ throw new Error(`Unknown profile ${JSON.stringify(profileName)}, available profiles: ${profileNames.join(", ")}`);
270
+ }
271
+ Object.assign(options, profiles[profileName], explicitOptions);
272
+ delete options.settingsFile;
273
+ }
274
+
275
+ function getOptions() {
276
+ const { positionals, options, invalidOptions } = parseArgs(Array.from(args));
277
+ const unknownOptions = positionals.filter(positional => positional.startsWith("--"));
278
+ const urls = positionals.filter(positional => !positional.startsWith("--"));
279
+ if (options.help) {
280
+ printUsage();
240
281
  exit(0);
241
282
  }
242
283
  if (options.version) {
243
284
  console.log(version); // eslint-disable-line no-console
244
285
  exit(0);
245
286
  }
246
- return { ...options, url: positionals[0], output: positionals[1] };
287
+ const errorMessages = [];
288
+ unknownOptions.forEach(option => errorMessages.push(`Unknown option ${option}`));
289
+ invalidOptions.forEach(({ name, value }) => errorMessages.push(value === undefined ?
290
+ `Missing value for --${name}` :
291
+ `Invalid value for --${name}: ${JSON.stringify(value)}`));
292
+ if (!urls.length && !options.urlsFile) {
293
+ errorMessages.push("The URL or path of the page to save is required");
294
+ }
295
+ if (urls.length > 2) {
296
+ errorMessages.push(`Unexpected arguments: ${urls.slice(2).join(", ")}`);
297
+ }
298
+ if (errorMessages.length) {
299
+ printUsage();
300
+ errorMessages.forEach(message => console.error(`Error: ${message}`)); // eslint-disable-line no-console
301
+ exit(1);
302
+ }
303
+ return { ...options, url: urls[0], output: urls[1] };
304
+ }
305
+
306
+ function printUsage() {
307
+ console.log(USAGE_TEXT + "\n"); // eslint-disable-line no-console
308
+ console.log("Options:"); // eslint-disable-line no-console
309
+ OPTIONS_INFO.forEach(category => {
310
+ const categoryName = CATEGORIES[OPTIONS_INFO.indexOf(category)];
311
+ console.log(` * ${categoryName}:`); // eslint-disable-line no-console
312
+ Object.keys(category).forEach(optionName => {
313
+ const optionInfo = category[optionName];
314
+ let optionType = optionInfo.type;
315
+ if (isArray(optionType)) {
316
+ optionType = optionType.replace("[]", "*");
317
+ }
318
+ const optionDescription = optionInfo.description;
319
+ const optionDefaultValue = optionInfo.defaultValue === undefined ? "" : `(default: ${JSON.stringify(optionInfo.defaultValue)})`;
320
+ console.log(` --${optionName}: ${optionDescription} <${optionType}> ${optionDefaultValue}`); // eslint-disable-line no-console
321
+ });
322
+ console.log(""); // eslint-disable-line no-console
323
+ });
247
324
  }
248
325
 
249
326
  function parseArgs(args, setDefaultValues = true) {
250
327
  const positionals = [];
251
328
  const options = {};
252
- const result = { positionals, options: {} };
329
+ const invalidOptions = [];
330
+ const result = { positionals, options: {}, invalidOptions };
253
331
  let argIndex = 0;
254
332
  while (argIndex < args.length) {
255
333
  const arg = args[argIndex];
256
- const { argName, argValue, optionInfo } = parseArg(arg);
257
- if (optionInfo) {
258
- if (options[argName] === undefined) {
259
- options[argName] = [];
334
+ const { argValue, option } = parseArg(arg);
335
+ if (option) {
336
+ const optionName = option.name;
337
+ if (options[optionName] === undefined) {
338
+ options[optionName] = [];
260
339
  }
261
- let nextArgName;
262
340
  if (argValue === undefined) {
263
- if (
264
- argIndex + 1 < args.length &&
265
- ({ argName: nextArgName } = parseArg(args[argIndex + 1])) &&
266
- (nextArgName === undefined || !getOptionInfo(nextArgName)) &&
267
- isValid(optionInfo.type, args[argIndex + 1]) &&
268
- (isArray(optionInfo.type) || !options[argName].length)) {
269
- options[argName].push(args[argIndex + 1]);
270
- argIndex++;
341
+ if (argIndex + 1 < args.length && !parseArg(args[argIndex + 1]).option) {
342
+ const nextArg = args[argIndex + 1];
343
+ if (isValid(option.info.type, nextArg)) {
344
+ options[optionName].push(nextArg);
345
+ argIndex++;
346
+ } else if (option.info.type.startsWith("number")) {
347
+ invalidOptions.push({ name: optionName, value: nextArg });
348
+ argIndex++;
349
+ }
271
350
  }
272
- } else if (isValid(optionInfo.type, argValue) && (isArray(optionInfo.type) || !options[argName].length)) {
273
- options[argName].push(argValue);
351
+ } else if (isValid(option.info.type, argValue)) {
352
+ options[optionName].push(argValue);
274
353
  } else {
275
- positionals.push(arg);
354
+ invalidOptions.push({ name: optionName, value: argValue });
276
355
  }
277
356
  } else {
278
357
  positionals.push(arg);
@@ -280,34 +359,38 @@ function parseArgs(args, setDefaultValues = true) {
280
359
  argIndex++;
281
360
  }
282
361
  Object.keys(options).forEach(optionName => {
283
- const optionInfo = getOptionInfo(optionName);
362
+ const optionInfo = getOptionInfo(optionName).info;
284
363
  const optionKey = getOptionKey(optionName, optionInfo);
285
364
  let optionValue = options[optionName];
286
365
  const isArrayType = isArray(optionInfo.type);
366
+ if (!optionInfo.type.startsWith("boolean") && !optionValue.length &&
367
+ !invalidOptions.some(invalidOption => invalidOption.name == optionName)) {
368
+ invalidOptions.push({ name: optionName });
369
+ }
287
370
  if (optionInfo.type.startsWith("boolean")) {
288
371
  optionValue = optionValue.map(value => value == "true");
289
372
  optionValue = isArrayType ?
290
373
  optionValue.length ? optionValue : true :
291
- optionValue.length ? optionValue[0] : true;
374
+ optionValue.length ? optionValue[optionValue.length - 1] : true;
292
375
  } else if (optionInfo.type.startsWith("number")) {
293
376
  optionValue = optionValue.map(value => Number(value));
294
377
  optionValue = isArrayType ?
295
378
  optionValue.length ? optionValue : optionInfo.defaultValue || 0 :
296
- optionValue.length ? optionValue[0] : optionInfo.defaultValue || 0;
379
+ optionValue.length ? optionValue[optionValue.length - 1] : optionInfo.defaultValue || 0;
297
380
  } else {
298
381
  optionValue = isArrayType ?
299
382
  optionValue.length ? optionValue : optionInfo.defaultValue || "" :
300
- optionValue.length ? optionValue[0] : optionInfo.defaultValue || "";
383
+ optionValue.length ? optionValue[optionValue.length - 1] : optionInfo.defaultValue || "";
301
384
  }
302
385
  result.options[optionKey] = optionValue;
303
386
  });
304
387
  if (setDefaultValues) {
305
388
  OPTIONS_INFO.forEach(categoryOptions => {
306
389
  Object.keys(categoryOptions).forEach(optionName => {
307
- const optionInfo = getOptionInfo(optionName);
390
+ const optionInfo = categoryOptions[optionName];
308
391
  const optionKey = getOptionKey(optionName, optionInfo);
309
392
  if (result.options[optionKey] === undefined && optionInfo.defaultValue !== undefined) {
310
- result.options[optionKey] = categoryOptions[optionName].defaultValue;
393
+ result.options[optionKey] = optionInfo.defaultValue;
311
394
  }
312
395
  });
313
396
  });
@@ -332,7 +415,11 @@ function parseArgs(args, setDefaultValues = true) {
332
415
  }
333
416
  if (result.options.browserArgs) {
334
417
  const browserArguments = result.options.browserArguments || [];
335
- browserArguments.push(...JSON.parse(result.options.browserArgs));
418
+ try {
419
+ browserArguments.push(...JSON.parse(result.options.browserArgs));
420
+ } catch {
421
+ invalidOptions.push({ name: "browser-args", value: result.options.browserArgs });
422
+ }
336
423
  result.options.browserArgs = browserArguments;
337
424
  delete result.options.browserArguments;
338
425
  }
@@ -340,46 +427,14 @@ function parseArgs(args, setDefaultValues = true) {
340
427
  result.options.browserArgs = result.options.browserArguments;
341
428
  delete result.options.browserArguments;
342
429
  }
343
- if (result.options.errorFile) {
430
+ if (result.options.errorFile !== undefined) {
344
431
  result.options.errorsFile = result.options.errorFile;
345
432
  delete result.options.errorFile;
346
433
  }
347
- if (result.options.errorTracesDisabled) {
434
+ if (result.options.errorTracesDisabled !== undefined) {
348
435
  result.options.errorsTracesDisabled = result.options.errorTracesDisabled;
349
436
  delete result.options.errorTracesDisabled;
350
437
  }
351
- if (result.options.crawlReplaceUrls) {
352
- result.options.crawlReplaceURLs = result.options.crawlReplaceUrls;
353
- delete result.options.crawlReplaceUrls;
354
- }
355
- if (result.options.saveOriginalUrls) {
356
- result.options.saveOriginalURLs = result.options.saveOriginalUrls;
357
- delete result.options.saveOriginalUrls;
358
- }
359
- if (result.options.browserRemoteDebuggingUrl) {
360
- result.options.browserRemoteDebuggingURL = result.options.browserRemoteDebuggingUrl;
361
- delete result.options.browserRemoteDebuggingUrl;
362
- }
363
- if (result.options.crawlRemoveUrlFragment) {
364
- result.options.crawlRemoveURLFragment = result.options.crawlRemoveUrlFragment;
365
- delete result.options.crawlRemoveUrlFragment;
366
- }
367
- if (result.options.compressCss) {
368
- result.options.compressCSS = result.options.compressCss;
369
- delete result.options.compressCss;
370
- }
371
- if (result.options.compressHtml) {
372
- result.options.compressHTML = result.options.compressHtml;
373
- delete result.options.compressHtml;
374
- }
375
- if (result.options.insertMetaCsp) {
376
- result.options.insertMetaCSP = result.options.insertMetaCsp;
377
- delete result.options.insertMetaCsp;
378
- }
379
- if (result.options.blockedUrlPatterns) {
380
- result.options.blockedURLPatterns = result.options.blockedUrlPatterns;
381
- delete result.options.blockedUrlPatterns;
382
- }
383
438
  if (result.options.filenameReplacedCharacters) {
384
439
  const filenameReplacedCharacters = result.options.filenameReplacedCharacters;
385
440
  result.options.filenameReplacedCharacters = [];
@@ -404,6 +459,49 @@ function parseArgs(args, setDefaultValues = true) {
404
459
  }
405
460
  });
406
461
  }
462
+ if (result.options.httpHeaders) {
463
+ const headers = {};
464
+ result.options.httpHeaders.forEach(header => {
465
+ const separatorIndex = header.indexOf("=");
466
+ if (separatorIndex <= 0) {
467
+ invalidOptions.push({ name: "http-header", value: header });
468
+ } else {
469
+ headers[header.substring(0, separatorIndex).trim()] = header.substring(separatorIndex + 1).trim();
470
+ }
471
+ });
472
+ result.options.httpHeaders = headers;
473
+ }
474
+ if (result.options.emulateMediaFeatures) {
475
+ result.options.emulateMediaFeatures = result.options.emulateMediaFeatures
476
+ .map(feature => {
477
+ const separatorIndex = feature.indexOf(":");
478
+ if (separatorIndex <= 0) {
479
+ invalidOptions.push({ name: "emulate-media-feature", value: feature });
480
+ } else {
481
+ return {
482
+ name: feature.substring(0, separatorIndex),
483
+ value: feature.substring(separatorIndex + 1)
484
+ };
485
+ }
486
+ })
487
+ .filter(feature => feature);
488
+ }
489
+ if (result.options.browserCookies) {
490
+ result.options.browserCookies = result.options.browserCookies.map(cookie => {
491
+ const [name, value, domain, path, expires, httpOnly, secure, sameSite, url] = cookie.split(",");
492
+ return {
493
+ name,
494
+ value,
495
+ url,
496
+ domain,
497
+ path,
498
+ secure: secure === "true",
499
+ httpOnly: httpOnly === "true",
500
+ sameSite,
501
+ expires: expires && !isNaN(Number(expires)) ? Number(expires) : undefined
502
+ };
503
+ });
504
+ }
407
505
  return result;
408
506
  }
409
507
 
@@ -423,13 +521,13 @@ function parseArg(arg) {
423
521
  const parsedArg = arg.match(ARGS_REGEX);
424
522
  if (parsedArg && parsedArg.length) {
425
523
  let [_, argName, argValue] = parsedArg; // eslint-disable-line no-unused-vars
426
- const optionInfo = getOptionInfo(argName);
524
+ const option = getOptionInfo(argName);
427
525
  if (argValue !== undefined &&
428
526
  ((argValue.startsWith("\"") && argValue.endsWith("\"")) ||
429
527
  (argValue.startsWith("'") && argValue.endsWith("'")))) {
430
528
  argValue = argValue.substring(1, argValue.length - 1);
431
529
  }
432
- return { argName, argValue, optionInfo };
530
+ return { argName, argValue, option };
433
531
  } else {
434
532
  return {};
435
533
  }
@@ -440,7 +538,7 @@ function getOptionInfo(optionName) {
440
538
  OPTIONS_INFO.forEach(categoryOptions => {
441
539
  Object.keys(categoryOptions).forEach(keyName => {
442
540
  if (keyName.toLowerCase() == optionName.toLowerCase() || categoryOptions[keyName].alias == optionName.toLowerCase()) {
443
- result = categoryOptions[keyName];
541
+ result = { name: keyName, info: categoryOptions[keyName] };
444
542
  }
445
543
  });
446
544
  });
@@ -455,7 +553,7 @@ function isValid(type, value) {
455
553
  if (type.startsWith("boolean")) {
456
554
  return value == "true" || value == "false";
457
555
  } else if (type.startsWith("number")) {
458
- return !isNaN(value);
556
+ return value.trim() != "" && !isNaN(value);
459
557
  } else {
460
558
  return true;
461
559
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "single-file-cli",
3
- "version": "2.1.1",
3
+ "version": "2.1.2",
4
4
  "description": "SingleFile CLI",
5
5
  "author": "Gildas Lormeau",
6
6
  "engines": {
@@ -10,18 +10,18 @@
10
10
  },
11
11
  "type": "module",
12
12
  "scripts": {
13
- "test": "node --test \"test/**/*.test.js\"",
13
+ "test": "node --test --test-concurrency=2 \"test/**/*.test.js\"",
14
14
  "lint": "eslint ."
15
15
  },
16
16
  "bin": {
17
17
  "single-file": "single-file-node.js"
18
18
  },
19
19
  "dependencies": {
20
- "simple-cdp": "^1.10.0"
20
+ "simple-cdp": "^1.10.1"
21
21
  },
22
22
  "devDependencies": {
23
- "@eslint/js": "^9.39.1",
24
- "esbuild": "^0.27.4",
23
+ "@eslint/js": "^9.39.5",
24
+ "esbuild": "^0.27.7",
25
25
  "eslint": "^10.8.1"
26
26
  }
27
27
  }
@@ -106,7 +106,6 @@ async function initialize(options) {
106
106
 
107
107
  async function capture(urls, options) {
108
108
  let newTasks;
109
- const taskUrls = tasks.map(task => task.url);
110
109
  newTasks = await Promise.all(urls.map(value => {
111
110
  let url, taskOptions;
112
111
  if (Array.isArray(value)) {
@@ -118,7 +117,7 @@ async function capture(urls, options) {
118
117
  }
119
118
  return createTask(url, taskOptions);
120
119
  }));
121
- newTasks = newTasks.filter(task => task && !taskUrls.includes(task.url));
120
+ newTasks = newTasks.filter((task, taskIndex) => task && !mergeDuplicateTask(task, tasks.concat(newTasks.slice(0, taskIndex))));
122
121
  if (newTasks.length) {
123
122
  tasks = tasks.concat(newTasks);
124
123
  await saveTasks();
@@ -300,37 +299,27 @@ async function capturePage(options) {
300
299
  pageData.comment = undefined;
301
300
  content = JSON.stringify(pageData, null, 2);
302
301
  }
303
- if (options.output) {
304
- filename = await getFilename(options.output, options);
305
- } else if (options.dumpContent) {
302
+ if (options.dumpContent && !options.output) {
306
303
  if (options.compressContent) {
307
304
  await stdout.write(content);
308
305
  } else {
309
306
  console.log(content || ""); // eslint-disable-line no-console
310
307
  }
311
308
  } else {
312
- filename = await getFilename(pageData.filename, options);
309
+ let outputFilename = options.output || pageData.filename;
310
+ if (options.outputJson && !outputFilename.endsWith(".json")) {
311
+ outputFilename += ".json";
312
+ }
313
+ filename = await writeOutputFile(outputFilename, content, options);
313
314
  }
314
315
  if (filename) {
315
- if (options.outputJson) {
316
- filename += filename.endsWith(".json") ? "" : ".json";
317
- }
318
- const directoryName = path.dirname(filename);
319
- if (directoryName !== ".") {
320
- await mkdir(directoryName, { recursive: true });
321
- }
322
- if (content instanceof Uint8Array) {
323
- await writeFile(filename, content);
324
- } else {
325
- await writeTextFile(filename, content);
326
- }
327
316
  const outputDirectory = getOutputDirectory(options);
328
317
  pageData.filename = filename.startsWith(outputDirectory) ? filename.substring(outputDirectory.length) : filename;
329
318
  }
330
319
  return pageData;
331
320
  } catch (error) {
332
321
  const date = new Date();
333
- let message = `[${date.toISOString()}] URL: ${options.url}`;
322
+ let message = `[${date.toISOString()}] URL: ${options.url} Error: ${error.message || error}`;
334
323
  if (!options.errorsTracesDisabled) {
335
324
  message += "\nStack: " + error.stack;
336
325
  }
@@ -366,6 +355,37 @@ function getOutputDirectory(options) {
366
355
  return outputDirectory;
367
356
  }
368
357
 
358
+ async function writeOutputFile(outputFilename, content, options) {
359
+ while (true) {
360
+ const filename = await getFilename(outputFilename, options);
361
+ if (!filename) {
362
+ return;
363
+ }
364
+ const directoryName = path.dirname(filename);
365
+ if (directoryName !== ".") {
366
+ await mkdir(directoryName, { recursive: true });
367
+ }
368
+ // exclusive creation prevents parallel tasks writing the same filename
369
+ // from silently overwriting each other
370
+ const writeOptions = { createNew: options.filenameConflictAction != "overwrite" };
371
+ try {
372
+ if (content instanceof Uint8Array) {
373
+ await writeFile(filename, content, writeOptions);
374
+ } else {
375
+ await writeTextFile(filename, content, writeOptions);
376
+ }
377
+ return filename;
378
+ } catch (error) {
379
+ if (!(error instanceof errors.AlreadyExists)) {
380
+ throw error;
381
+ }
382
+ if (options.filenameConflictAction == "skip") {
383
+ return;
384
+ }
385
+ }
386
+ }
387
+ }
388
+
369
389
  async function getFilename(filename, options, index = 1) {
370
390
  let newFilename = getOutputDirectory(options) + filename;
371
391
  if (options.filenameConflictAction == "overwrite") {
@@ -24,7 +24,7 @@
24
24
  import { initialize } from "./single-file-cli-api.js";
25
25
  import { closeBrowser } from "./lib/browser.js";
26
26
  import { Deno } from "./lib/deno-polyfill.js";
27
- import { options, parseArgs } from "./options.js";
27
+ import { getOptions, applySettings, parseUrlsFile } from "./options.js";
28
28
 
29
29
  const { readTextFile, readFile, exit, addSignalListener } = Deno;
30
30
 
@@ -43,39 +43,17 @@ export { run };
43
43
 
44
44
  async function run() {
45
45
  try {
46
+ const options = getOptions();
46
47
  let urls;
47
48
  if (options.settingsFile) {
48
- const settings = JSON.parse(await Deno.readTextFile(options.settingsFile));
49
- let profileName = options.settingsProfile || "default";
50
- if (profileName == "default" || !settings.profiles[profileName]) {
51
- profileName = "__Default_Settings__";
52
- }
53
- Object.assign(options, settings.profiles[profileName]);
54
- delete options.settingsFile;
49
+ const settings = JSON.parse(await readTextFile(options.settingsFile));
50
+ applySettings(options, settings);
55
51
  }
56
52
  if (options.urlsFile) {
57
53
  urls = await getUrlsFile(options.urlsFile);
58
54
  } else {
59
55
  urls = [options.url];
60
56
  }
61
- if (options.browserCookies) {
62
- const cookies = [];
63
- for (const cookie of options.browserCookies) {
64
- const [name, value, domain, path, expires, httpOnly, secure, sameSite, url] = cookie.split(",");
65
- cookies.push({
66
- name,
67
- value,
68
- url,
69
- domain,
70
- path,
71
- secure: secure === "true",
72
- httpOnly: httpOnly === "true",
73
- sameSite,
74
- expires: isNaN(Number(expires)) ? undefined : Number(expires)
75
- });
76
- }
77
- options.browserCookies = cookies;
78
- }
79
57
  if (options.browserCookiesFile) {
80
58
  const cookiesContent = await readTextFile(options.browserCookiesFile);
81
59
  try {
@@ -84,23 +62,6 @@ async function run() {
84
62
  options.browserCookies = parseCookies(cookiesContent);
85
63
  }
86
64
  }
87
- if (options.emulateMediaFeatures) {
88
- options.emulateMediaFeatures = options.emulateMediaFeatures.map(feature => {
89
- const colonIndex = feature.indexOf(":");
90
- return {
91
- name: feature.substring(0, colonIndex),
92
- value: feature.substring(colonIndex + 1)
93
- };
94
- });
95
- }
96
- if (options.httpHeaders) {
97
- const headers = {};
98
- for (const header of options.httpHeaders) {
99
- const [name, value] = header.split("=");
100
- headers[name] = value.trim();
101
- }
102
- options.httpHeaders = headers;
103
- }
104
65
  if (options.embeddedImage) {
105
66
  options.embeddedImage = Array.from(await readFile(options.embeddedImage));
106
67
  }
@@ -148,54 +109,5 @@ async function closeBrowserAndExit(code) {
148
109
  }
149
110
 
150
111
  async function getUrlsFile(urlsFile) {
151
- let urls = (await readTextFile(urlsFile)).split("\n");
152
- urls = urls.map(value => {
153
- value = value.trim();
154
- let optionPosition = value.indexOf(" --");
155
- if (optionPosition < 0) {
156
- optionPosition = value.indexOf("\t--");
157
- }
158
- if (optionPosition > 0) {
159
- const url = value.substring(0, optionPosition).trim();
160
- const argsString = value.substring(optionPosition + 1).trim();
161
- const args = [];
162
- let previousCharacter, previousPreviousCharacter, lastQuoteCharacter;
163
- let lastCharIndex = 0;
164
- for (let currentCharIndex = 0; currentCharIndex < argsString.length; currentCharIndex++) {
165
- const character = argsString[currentCharIndex];
166
- if (character == lastQuoteCharacter && (previousCharacter != "\\" || previousPreviousCharacter == "\\")) {
167
- args.push(argsString.substring(lastCharIndex, currentCharIndex));
168
- lastQuoteCharacter = null;
169
- lastCharIndex = currentCharIndex + 1;
170
- } else if (!lastQuoteCharacter) {
171
- if (character == "'" || character == "\"") {
172
- lastQuoteCharacter = argsString[currentCharIndex];
173
- lastCharIndex = currentCharIndex + 1;
174
- } else {
175
- const isSpaceCharacter = character == " " || character == "\t";
176
- if (isSpaceCharacter || character == "=") {
177
- if (isSpaceCharacter && (currentCharIndex == lastCharIndex + 1)) {
178
- lastCharIndex++;
179
- } else if (lastCharIndex < currentCharIndex) {
180
- args.push(argsString.substring(lastCharIndex, currentCharIndex));
181
- lastCharIndex = currentCharIndex + 1;
182
- } else {
183
- lastCharIndex = currentCharIndex + 1;
184
- }
185
- }
186
- }
187
- }
188
- previousPreviousCharacter = previousCharacter;
189
- previousCharacter = character;
190
- }
191
- if (lastCharIndex < argsString.length) {
192
- args.push(argsString.substring(lastCharIndex).trim());
193
- }
194
- const { options } = parseArgs(args, false);
195
- return [url, options];
196
- } else {
197
- return value;
198
- }
199
- });
200
- return urls;
112
+ return parseUrlsFile(await readTextFile(urlsFile));
201
113
  }
@@ -0,0 +1,31 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { execFile } from "node:child_process";
4
+ import { promisify } from "node:util";
5
+ import { mkdtemp, readFile, rm } from "node:fs/promises";
6
+ import { tmpdir } from "node:os";
7
+ import { join, dirname } from "node:path";
8
+ import { fileURLToPath } from "node:url";
9
+ import process from "node:process";
10
+
11
+ const execFileAsync = promisify(execFile);
12
+ const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
13
+
14
+ test("errors file lines include the error message", { timeout: 120000 }, async () => {
15
+ const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
16
+ try {
17
+ const errorsPath = join(directory, "errors.txt");
18
+ const url = "http://localhost:1/";
19
+ await execFileAsync(process.execPath, [
20
+ "single-file-node.js", url, join(directory, "out.html"),
21
+ "--errors-file", errorsPath
22
+ ], { cwd: cliDirectory });
23
+ const content = await readFile(errorsPath, "utf8");
24
+ assert.ok(content.includes("URL: " + url));
25
+ const errorMessage = content.match(/Error: (.*)/);
26
+ assert.ok(errorMessage);
27
+ assert.ok(errorMessage[1].trim().length > 0);
28
+ } finally {
29
+ await rm(directory, { recursive: true });
30
+ }
31
+ });
@@ -51,7 +51,8 @@ test("capture waits for the top frame, not a fast iframe", { timeout: 120000 },
51
51
  await execFileAsync(process.execPath, [
52
52
  "single-file-node.js", url, outputPath,
53
53
  "--browser-wait-until", "networkIdle",
54
- "--browser-wait-until-delay", "1000"
54
+ "--browser-wait-until-delay", "1000",
55
+ "--browser-wait-until-fallback", "false"
55
56
  ], { cwd: cliDirectory });
56
57
  const content = await readFile(outputPath, "utf8");
57
58
  assert.ok(content.includes(LOAD_MARKER), "page was captured before the top frame finished loading");
@@ -0,0 +1,70 @@
1
+ /* global setInterval, clearInterval */
2
+
3
+ import { test } from "node:test";
4
+ import assert from "node:assert/strict";
5
+ import { createServer } from "node:http";
6
+ import { execFile } from "node:child_process";
7
+ import { promisify } from "node:util";
8
+ import { mkdtemp, rm } from "node:fs/promises";
9
+ import { Buffer } from "node:buffer";
10
+ import { tmpdir } from "node:os";
11
+ import { join, dirname } from "node:path";
12
+ import { fileURLToPath } from "node:url";
13
+ import process from "node:process";
14
+
15
+ const execFileAsync = promisify(execFile);
16
+ const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
17
+
18
+ test("load timeout is reported when the page never loads", { timeout: 120000 }, async () => {
19
+ const server = createServer(() => { });
20
+ await new Promise(resolve => server.listen(0, "localhost", resolve));
21
+ const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
22
+ try {
23
+ const url = "http://localhost:" + server.address().port + "/";
24
+ const { stderr } = await runCli([
25
+ url, join(directory, "out.html"),
26
+ "--browser-load-max-time", "5000",
27
+ "--browser-wait-until-fallback", "false"
28
+ ]);
29
+ assert.ok(stderr.includes("Load timeout"), "stderr: " + stderr);
30
+ } finally {
31
+ await rm(directory, { recursive: true });
32
+ server.closeAllConnections();
33
+ server.close();
34
+ }
35
+ });
36
+
37
+ test("capture timeout is reported when capturing an endless download", { timeout: 120000 }, async () => {
38
+ const chunk = Buffer.alloc(65536);
39
+ const server = createServer((_, response) => {
40
+ response.writeHead(200, { "content-type": "application/octet-stream", "content-disposition": "attachment; filename=big.bin" });
41
+ const intervalId = setInterval(() => response.write(chunk), 50);
42
+ response.on("close", () => clearInterval(intervalId));
43
+ });
44
+ await new Promise(resolve => server.listen(0, "localhost", resolve));
45
+ const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
46
+ try {
47
+ const url = "http://localhost:" + server.address().port + "/";
48
+ const { stderr } = await runCli([
49
+ url, join(directory, "out.html"),
50
+ "--browser-load-max-time", "5000",
51
+ "--browser-capture-max-time", "10000"
52
+ ]);
53
+ assert.ok(stderr.includes("Capture timeout") || stderr.includes("Load timeout"), "stderr: " + stderr);
54
+ } finally {
55
+ await rm(directory, { recursive: true });
56
+ server.closeAllConnections();
57
+ server.close();
58
+ }
59
+ });
60
+
61
+ async function runCli(args) {
62
+ try {
63
+ return await execFileAsync(process.execPath, ["single-file-node.js", ...args], { cwd: cliDirectory, timeout: 90000, killSignal: "SIGKILL" });
64
+ } catch (error) {
65
+ if (error.killed) {
66
+ throw new Error("the process did not exit before the timeout");
67
+ }
68
+ return { stdout: error.stdout, stderr: error.stderr };
69
+ }
70
+ }
@@ -0,0 +1,37 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { createServer } from "node:http";
4
+ import { execFile } from "node:child_process";
5
+ import { promisify } from "node:util";
6
+ import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
7
+ import { tmpdir } from "node:os";
8
+ import { join, dirname } from "node:path";
9
+ import { fileURLToPath } from "node:url";
10
+ import process from "node:process";
11
+
12
+ const execFileAsync = promisify(execFile);
13
+ const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
14
+
15
+ test("duplicate urls in a urls file are captured once", { timeout: 120000 }, async () => {
16
+ const server = createServer((_, response) => response
17
+ .writeHead(200, { "content-type": "text/html" })
18
+ .end("<html><head><title>Same Page</title></head><body>content</body></html>"));
19
+ await new Promise(resolve => server.listen(0, "localhost", resolve));
20
+ const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
21
+ try {
22
+ const url = "http://localhost:" + server.address().port + "/";
23
+ const urlsFilePath = join(directory, "urls.txt");
24
+ await writeFile(urlsFilePath, url + "\n\n" + url + "\n");
25
+ const { stderr } = await execFileAsync(process.execPath, [
26
+ "single-file-node.js",
27
+ "--urls-file", urlsFilePath,
28
+ "--output-directory", directory,
29
+ "--filename-template", "{page-title}.html"
30
+ ], { cwd: cliDirectory });
31
+ const files = (await readdir(directory)).filter(file => file.endsWith(".html"));
32
+ assert.deepEqual(files, ["Same Page.html"], "stderr: " + stderr);
33
+ } finally {
34
+ await rm(directory, { recursive: true });
35
+ server.close();
36
+ }
37
+ });
@@ -0,0 +1,56 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { execFile } from "node:child_process";
4
+ import { promisify } from "node:util";
5
+ import { join, dirname } from "node:path";
6
+ import { fileURLToPath } from "node:url";
7
+ import process from "node:process";
8
+
9
+ const execFileAsync = promisify(execFile);
10
+ const cliDirectory = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
11
+
12
+ async function runCli(args) {
13
+ try {
14
+ const { stdout, stderr } = await execFileAsync(process.execPath, ["single-file-node.js", ...args], { cwd: cliDirectory });
15
+ return { code: 0, stdout, stderr };
16
+ } catch (error) {
17
+ return { code: error.code, stdout: error.stdout, stderr: error.stderr };
18
+ }
19
+ }
20
+
21
+ test("help is displayed with an exit code of 0", async () => {
22
+ const { code, stdout } = await runCli(["--help"]);
23
+ assert.equal(code, 0);
24
+ assert.ok(stdout.includes("--browser-executable-path"));
25
+ });
26
+
27
+ test("a missing url is reported as an error", async () => {
28
+ const { code, stderr } = await runCli(["--browser-executable-path", "/path/to/chrome"]);
29
+ assert.equal(code, 1);
30
+ assert.ok(stderr.includes("The URL or path of the page to save is required"));
31
+ });
32
+
33
+ test("unknown options are reported as errors", async () => {
34
+ const { code, stderr } = await runCli(["--foo", "--bar", "https://example.com", "out.html"]);
35
+ assert.equal(code, 1);
36
+ assert.ok(stderr.includes("Unknown option --foo"));
37
+ assert.ok(stderr.includes("Unknown option --bar"));
38
+ });
39
+
40
+ test("invalid option values are reported as errors", async () => {
41
+ const { code, stderr } = await runCli(["--browser-width", "abc", "https://example.com"]);
42
+ assert.equal(code, 1);
43
+ assert.ok(stderr.includes("Invalid value for --browser-width: \"abc\""));
44
+ });
45
+
46
+ test("a wrong browser executable path is reported with the path", async () => {
47
+ const { code, stderr } = await runCli(["https://localhost:1/", "out.html", "--browser-executable-path", "/nonexistent/chrome"]);
48
+ assert.notEqual(code, 0);
49
+ assert.ok(stderr.includes("The browser executable was not found at \"/nonexistent/chrome\""));
50
+ });
51
+
52
+ test("unexpected extra arguments are reported as errors", async () => {
53
+ const { code, stderr } = await runCli(["https://example.com", "out.html", "extra.html"]);
54
+ assert.equal(code, 1);
55
+ assert.ok(stderr.includes("Unexpected arguments: extra.html"));
56
+ });
@@ -0,0 +1,136 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { parseArgs, applySettings, parseUrlsFile } from "../../options.js";
4
+
5
+ const parse = args => parseArgs(args).options;
6
+
7
+ test("boolean options with uppercase canonical names can be disabled", () => {
8
+ assert.equal(parse(["--compress-html", "false"]).compressHTML, false);
9
+ assert.equal(parse(["--compress-HTML", "false"]).compressHTML, false);
10
+ assert.equal(parse(["--crawl-remove-url-fragment", "false"]).crawlRemoveURLFragment, false);
11
+ assert.equal(parse(["--insert-meta-csp", "false"]).insertMetaCSP, false);
12
+ });
13
+
14
+ test("option names are canonicalized regardless of case", () => {
15
+ assert.equal(parse(["--include-bom"]).includeBOM, true);
16
+ assert.equal(parse(["--Include-BOM"]).includeBOM, true);
17
+ assert.equal(parse(["--crawl-replace-urls"]).crawlReplaceURLs, true);
18
+ assert.equal(parse(["--save-original-urls"]).saveOriginalURLs, true);
19
+ assert.deepEqual(parse(["--blocked-url-pattern", "ads"]).blockedURLPatterns, ["ads"]);
20
+ const options = parse(["--compress-html", "false"]);
21
+ assert.equal("compressHtml" in options, false);
22
+ });
23
+
24
+ test("aliases map to the canonical option", () => {
25
+ assert.equal(parse(["--error-file", "errors.txt"]).errorsFile, "errors.txt");
26
+ assert.equal(parse(["--errors-file", "errors.txt"]).errorsFile, "errors.txt");
27
+ assert.equal(parse(["--error-traces-disabled", "false"]).errorsTracesDisabled, false);
28
+ assert.equal(parse(["--errors-traces-disabled", "false"]).errorsTracesDisabled, false);
29
+ assert.equal(parse([]).errorsTracesDisabled, true);
30
+ });
31
+
32
+ test("browser arguments are merged", () => {
33
+ assert.deepEqual(parse(["--browser-arg", "--disable-gpu"]).browserArgs, ["--disable-gpu"]);
34
+ assert.deepEqual(parse(["--browser-args", "[\"--mute-audio\"]"]).browserArgs, ["--mute-audio"]);
35
+ assert.deepEqual(parse(["--browser-arg", "--disable-gpu", "--browser-args", "[\"--mute-audio\"]"]).browserArgs, ["--disable-gpu", "--mute-audio"]);
36
+ });
37
+
38
+ test("default values are applied under canonical keys", () => {
39
+ const options = parse([]);
40
+ assert.equal(options.compressHTML, true);
41
+ assert.equal(options.crawlRemoveURLFragment, true);
42
+ assert.equal(options.insertMetaCSP, true);
43
+ assert.equal(options.browserWidth, 1280);
44
+ });
45
+
46
+ test("invalid and missing values are reported", () => {
47
+ assert.deepEqual(parseArgs(["--browser-width", "abc"]).invalidOptions, [{ name: "browser-width", value: "abc" }]);
48
+ assert.deepEqual(parseArgs(["--browser-width=abc"]).invalidOptions, [{ name: "browser-width", value: "abc" }]);
49
+ assert.deepEqual(parseArgs(["--browser-height="]).invalidOptions, [{ name: "browser-height", value: "" }]);
50
+ assert.deepEqual(parseArgs(["--output-directory"]).invalidOptions, [{ name: "output-directory" }]);
51
+ assert.deepEqual(parseArgs(["--browser-width", "abc"]).positionals, []);
52
+ assert.deepEqual(parseArgs(["--browser-debug"]).invalidOptions, []);
53
+ });
54
+
55
+ test("the last value wins when a scalar option is repeated", () => {
56
+ const { options, positionals } = parseArgs(["--browser-width", "100", "--browser-width", "200"]);
57
+ assert.equal(options.browserWidth, 200);
58
+ assert.deepEqual(positionals, []);
59
+ assert.equal(parse(["--browser-width=100", "--browser-width=200"]).browserWidth, 200);
60
+ });
61
+
62
+ test("http headers keep characters after the first equals sign", () => {
63
+ const options = parse(["--http-header", "authorization=Basic dGVzdA==", "--http-header", " x-test = value "]);
64
+ assert.deepEqual(options.httpHeaders, { authorization: "Basic dGVzdA==", "x-test": "value" });
65
+ assert.deepEqual(parseArgs(["--http-header", "no-separator"]).invalidOptions, [{ name: "http-header", value: "no-separator" }]);
66
+ });
67
+
68
+ test("media features are split on the first colon", () => {
69
+ assert.deepEqual(parse(["--emulate-media-feature", "prefers-color-scheme:dark"]).emulateMediaFeatures, [{ name: "prefers-color-scheme", value: "dark" }]);
70
+ assert.deepEqual(parseArgs(["--emulate-media-feature", "no-separator"]).invalidOptions, [{ name: "emulate-media-feature", value: "no-separator" }]);
71
+ });
72
+
73
+ test("browser cookies are parsed into objects", () => {
74
+ const options = parse(["--browser-cookie", "name,value,example.com,/,,true,true,None,https://example.com"]);
75
+ assert.deepEqual(options.browserCookies, [{
76
+ name: "name",
77
+ value: "value",
78
+ url: "https://example.com",
79
+ domain: "example.com",
80
+ path: "/",
81
+ secure: true,
82
+ httpOnly: true,
83
+ sameSite: "None",
84
+ expires: undefined
85
+ }]);
86
+ assert.equal(parse(["--browser-cookie", "name,value,example.com,/,1893456000,false,false,Lax,https://example.com"]).browserCookies[0].expires, 1893456000);
87
+ });
88
+
89
+ test("invalid browser args json is reported", () => {
90
+ assert.deepEqual(parseArgs(["--browser-args", "not json"]).invalidOptions, [{ name: "browser-args", value: "not json" }]);
91
+ });
92
+
93
+ test("explicit options win over settings file profiles", () => {
94
+ const options = parse(["--settings-file-profile", "custom", "--browser-width", "1024"]);
95
+ const settings = { profiles: { custom: { compressHTML: false, browserWidth: 800 } } };
96
+ applySettings(options, settings, { browserWidth: 1024 });
97
+ assert.equal(options.compressHTML, false);
98
+ assert.equal(options.browserWidth, 1024);
99
+ });
100
+
101
+ test("the default profile maps to the default settings", () => {
102
+ const options = parse([]);
103
+ applySettings(options, { profiles: { __Default_Settings__: { compressHTML: false } } }, {});
104
+ assert.equal(options.compressHTML, false);
105
+ });
106
+
107
+ test("an unknown settings profile is reported", () => {
108
+ const options = parse(["--settings-file-profile", "missing"]);
109
+ assert.throws(
110
+ () => applySettings(options, { profiles: { __Default_Settings__: {}, work: {} } }, {}),
111
+ /Unknown profile "missing", available profiles: work/);
112
+ });
113
+
114
+ test("blank lines in urls files are skipped", () => {
115
+ assert.deepEqual(parseUrlsFile("https://a.example\n\n \nhttps://b.example\r\n"), ["https://a.example", "https://b.example"]);
116
+ });
117
+
118
+ test("single character values in urls file options are kept", () => {
119
+ const [url, options] = parseUrlsFile("https://a.example --crawl-max-depth 2 --browser-width 100")[0];
120
+ assert.equal(url, "https://a.example");
121
+ assert.equal(options.crawlMaxDepth, 2);
122
+ assert.equal(options.browserWidth, 100);
123
+ });
124
+
125
+ test("quoted and equal form values in urls files are parsed", () => {
126
+ const [, options] = parseUrlsFile("https://a.example --filename-template \"My Page.html\" --browser-height=600")[0];
127
+ assert.equal(options.filenameTemplate, "My Page.html");
128
+ assert.equal(options.browserHeight, 600);
129
+ });
130
+
131
+ test("values are parsed from both spaced and equal forms", () => {
132
+ assert.equal(parse(["--browser-width", "1024"]).browserWidth, 1024);
133
+ assert.equal(parse(["--browser-width=1024"]).browserWidth, 1024);
134
+ assert.equal(parse(["--browser-headless=false"]).browserHeadless, false);
135
+ assert.equal(parse(["--filename-template={page-title}.html"]).filenameTemplate, "{page-title}.html");
136
+ });
@@ -0,0 +1,37 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { mkdtemp, rm } from "node:fs/promises";
4
+ import { tmpdir } from "node:os";
5
+ import { join } from "node:path";
6
+ import { Deno } from "../../lib/deno-polyfill.js";
7
+
8
+ const { writeTextFile, writeFile, readTextFile, errors } = Deno;
9
+
10
+ test("exclusive creation fails when the file exists", async () => {
11
+ const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
12
+ try {
13
+ const filename = join(directory, "out.txt");
14
+ await writeTextFile(filename, "first", { createNew: true });
15
+ await assert.rejects(
16
+ () => writeTextFile(filename, "second", { createNew: true }),
17
+ error => error instanceof errors.AlreadyExists);
18
+ await assert.rejects(
19
+ () => writeFile(filename, new Uint8Array([1]), { createNew: true }),
20
+ error => error instanceof errors.AlreadyExists);
21
+ assert.equal(await readTextFile(filename), "first");
22
+ } finally {
23
+ await rm(directory, { recursive: true });
24
+ }
25
+ });
26
+
27
+ test("writing without exclusive creation overwrites", async () => {
28
+ const directory = await mkdtemp(join(tmpdir(), "single-file-test-"));
29
+ try {
30
+ const filename = join(directory, "out.txt");
31
+ await writeTextFile(filename, "first");
32
+ await writeTextFile(filename, "second");
33
+ assert.equal(await readTextFile(filename), "second");
34
+ } finally {
35
+ await rm(directory, { recursive: true });
36
+ }
37
+ });