single-file-cli 1.1.54 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/{.eslintrc.js → .eslintrc.json} +5 -6
  2. package/Dockerfile +2 -1
  3. package/README.MD +56 -60
  4. package/args.js +252 -334
  5. package/back-ends/chromium-back-end.js +223 -0
  6. package/back-ends/chromium-browser.js +89 -0
  7. package/back-ends/chromium-constants.js +157 -0
  8. package/back-ends/{common/scripts.js → single-file-script.js} +21 -49
  9. package/build-lib-dev.sh +84 -0
  10. package/build-lib.sh +80 -3
  11. package/compile.sh +13 -0
  12. package/deno-polyfill.js +223 -0
  13. package/deno.json +13 -0
  14. package/deno.lock +49 -0
  15. package/lib/single-file-bundle.js +1 -0
  16. package/package.json +9 -33
  17. package/single-file +39 -14
  18. package/single-file-cli-api.js +32 -44
  19. package/back-ends/extensions/bypass-csp/index.js +0 -36
  20. package/back-ends/extensions/bypass-csp/manifest.json +0 -20
  21. package/back-ends/extensions/disable-web-security/index.js +0 -54
  22. package/back-ends/extensions/disable-web-security/manifest.json +0 -20
  23. package/back-ends/extensions/network-idle/bg.js +0 -61
  24. package/back-ends/extensions/network-idle/content.js +0 -29
  25. package/back-ends/extensions/network-idle/manifest.json +0 -31
  26. package/back-ends/extensions/signed/bypass_csp-0.0.3-an+fx.xpi +0 -0
  27. package/back-ends/extensions/signed/disable_web_security-0.0.3-an+fx.xpi +0 -0
  28. package/back-ends/extensions/signed/network_idle-0.0.2-an+fx.xpi +0 -0
  29. package/back-ends/jsdom.js +0 -159
  30. package/back-ends/playwright-chromium.js +0 -126
  31. package/back-ends/playwright-firefox.js +0 -115
  32. package/back-ends/playwright-webkit.js +0 -126
  33. package/back-ends/puppeteer-firefox.js +0 -172
  34. package/back-ends/puppeteer.js +0 -201
  35. package/back-ends/webdriver-chromium.js +0 -172
  36. package/back-ends/webdriver-gecko.js +0 -190
  37. package/lib/single-file-bootstrap.js +0 -1
  38. package/lib/single-file-frames.js +0 -1
  39. package/lib/single-file-hooks-frames.js +0 -1
  40. package/lib/single-file-zip.min.js +0 -1
  41. package/lib/single-file.js +0 -1
  42. package/rollup.config.dev.js +0 -57
  43. package/rollup.config.js +0 -58
package/args.js CHANGED
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright 2010-2020 Gildas Lormeau
2
+ * Copyright 2010-2024 Gildas Lormeau
3
3
  * contact : gildas.lormeau <at> gmail.com
4
4
  *
5
5
  * This file is part of SingleFile.
@@ -21,341 +21,259 @@
21
21
  * Source.
22
22
  */
23
23
 
24
- /* global require, module */
24
+ import { exit, args } from "./deno-polyfill.js";
25
25
 
26
- const args = require("yargs")
27
- .wrap(null)
28
- .command("$0 [url] [output]", "Save a page into a single HTML file.", yargs => {
29
- yargs.positional("url", { description: "URL or path on the filesystem of the page to save", type: "string" });
30
- yargs.positional("output", { description: "Output filename", type: "string" });
31
- })
32
- .default({
33
- "accept-headers": {
34
- "font": "application/font-woff2;q=1.0,application/font-woff;q=0.9,*/*;q=0.8",
35
- "image": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
36
- "stylesheet": "text/css,*/*;q=0.1",
37
- "script": "*/*",
38
- "document": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
39
- },
40
- "back-end": "puppeteer",
41
- "block-mixed-content": false,
42
- "browser-server": "",
43
- "browser-headless": true,
44
- "browser-executable-path": "",
45
- "browser-width": 1280,
46
- "browser-height": 720,
47
- "browser-load-max-time": 60000,
48
- "browser-wait-delay": 0,
49
- "browser-wait-until": "networkidle0",
50
- "browser-wait-until-fallback": true,
51
- "browser-debug": false,
52
- "browser-script": [],
53
- "browser-stylesheet": [],
54
- "browser-args": "",
55
- "browser-start-minimized": false,
56
- "browser-cookie": [],
57
- "browser-cookies-file": "",
58
- "browser-ignore-insecure-certs": false,
59
- "browser-freeze-prototypes": false,
60
- "compress-content": false,
61
- "compress-CSS": false,
62
- "compress-HTML": true,
63
- "dump-content": false,
64
- "emulateMediaFeature": [],
65
- "filename-template": "%if-empty<{page-title}|No title> ({date-locale} {time-locale}).{filename-extension}",
66
- "filename-conflict-action": "uniquify",
67
- "filename-replacement-character": "_",
68
- "filename-max-length": 192,
69
- "filename-max-length-unit": "bytes",
70
- "replace-emojis-in-filename": false,
71
- "group-duplicate-images": true,
72
- "max-size-duplicate-images": 512 * 1024,
73
- "http-header": [],
74
- "http-proxy-server": "",
75
- "http-proxy-username": "",
76
- "http-proxy-password": "",
77
- "include-infobar": false,
78
- "insert-meta-csp": true,
79
- "load-deferred-images": true,
80
- "load-deferred-images-dispatch-scroll-event": false,
81
- "load-deferred-images-max-idle-time": 1500,
82
- "load-deferred-images-keep-zoom-level": false,
83
- "load-deferred-images-before-frames": false,
84
- "max-parallel-workers": 8,
85
- "max-resource-size-enabled": false,
86
- "max-resource-size": 10,
87
- "move-styles-in-head": false,
88
- "output-directory": "",
89
- "password": "",
90
- "remove-hidden-elements": true,
91
- "remove-unused-styles": true,
92
- "remove-unused-fonts": true,
93
- "remove-saved-date": false,
94
- "remove-frames": false,
95
- "block-scripts": true,
96
- "block-audios": true,
97
- "block-videos": true,
98
- "remove-alternative-fonts": true,
99
- "remove-alternative-medias": true,
100
- "remove-alternative-images": true,
101
- "save-original-urls": false,
102
- "save-raw-page": false,
103
- "web-driver-executable-path": "",
104
- "user-script-enabled": true,
105
- "include-BOM": false,
106
- "crawl-links": false,
107
- "crawl-inner-links-only": true,
108
- "crawl-remove-url-fragment": true,
109
- "crawl-max-depth": 1,
110
- "crawl-external-links-max-depth": 1,
111
- "crawl-replace-urls": false,
112
- "crawl-rewrite-rule": [],
113
- "insert-text-body": false,
114
- "create-root-directory": false,
115
- "self-extracting-archive": true,
116
- "extract-data-from-page": true,
117
- "prevent-appended-data": false
118
- })
119
- .options("back-end", { description: "Back-end to use" })
120
- .choices("back-end", ["jsdom", "puppeteer", "webdriver-chromium", "webdriver-gecko", "puppeteer-firefox", "playwright-firefox", "playwright-chromium", "playwright-webkit"])
121
- .options("block-audios", { description: "Block audios" })
122
- .boolean("block-audios")
123
- .options("block-fonts", { description: "Block fonts" })
124
- .boolean("block-fonts")
125
- .options("block-images", { description: "Block images" })
126
- .boolean("block-images")
127
- .options("block-scripts", { description: "Block scripts" })
128
- .boolean("block-scripts")
129
- .options("block-videos", { description: "Block videos" })
130
- .boolean("block-videos")
131
- .options("block-mixed-content", { description: "Block mixed contents" })
132
- .boolean("block-mixed-content")
133
- .options("browser-server", { description: "Server to connect to (puppeteer only for now)" })
134
- .string("browser-server")
135
- .options("browser-headless", { description: "Run the browser in headless mode (puppeteer, webdriver-gecko, webdriver-chromium)" })
136
- .boolean("browser-headless")
137
- .options("browser-executable-path", { description: "Path to chrome/chromium executable (puppeteer, webdriver-gecko, webdriver-chromium)" })
138
- .string("browser-executable-path")
139
- .options("browser-width", { description: "Width of the browser viewport in pixels" })
140
- .number("browser-width")
141
- .options("browser-height", { description: "Height of the browser viewport in pixels" })
142
- .number("browser-height")
143
- .options("browser-load-max-time", { description: "Maximum delay of time to wait for page loading in ms (puppeteer, webdriver-gecko, webdriver-chromium)" })
144
- .number("browser-load-max-time")
145
- .options("browser-wait-delay", { description: "Time to wait before capturing the page in ms" })
146
- .number("browser-wait-delay")
147
- .options("browser-wait-until", { description: "When to consider the page is loaded (puppeteer, webdriver-gecko, webdriver-chromium)" })
148
- .choices("browser-wait-until", ["networkidle0", "networkidle2", "load", "domcontentloaded"])
149
- .options("browser-wait-until-fallback", { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown" })
150
- .boolean("browser-wait-until-fallback")
151
- .options("browser-debug", { description: "Enable debug mode (puppeteer, webdriver-gecko, webdriver-chromium)" })
152
- .boolean("browser-debug")
153
- .options("browser-script", { description: "Path of a script executed in the page (and all the frames) before it is loaded" })
154
- .array("browser-script")
155
- .options("browser-stylesheet", { description: "Path of a stylesheet file inserted into the page (and all the frames) after it is loaded" })
156
- .array("browser-stylesheet")
157
- .options("browser-args", { description: "Arguments provided as a JSON array and passed to the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
158
- .string("browser-args")
159
- .options("browser-start-minimized", { description: "Minimize the browser (puppeteer)" })
160
- .boolean("browser-start-minimized")
161
- .options("browser-cookie", { description: "Ordered list of cookie parameters separated by a comma: name,value,domain,path,expires,httpOnly,secure,sameSite,url (puppeteer, webdriver-gecko, webdriver-chromium, jsdom)" })
162
- .array("browser-cookie")
163
- .options("browser-cookies-file", { description: "Path of the cookies file formatted as a JSON file or a Netscape text file (puppeteer, webdriver-gecko, webdriver-chromium, jsdom)" })
164
- .string("browser-cookies-file")
165
- .options("browser-ignore-insecure-certs", { description: "Ignore HTTPs errors" })
166
- .boolean("browser-ignore-insecure-certs")
167
- .options("browser-freeze-prototypes", { description: "Freeze prototypes of built-in objects in the page" })
168
- .boolean("browser-freeze-prototypes")
169
- .options("compress-content", { description: "Compress the output file into a ZIP file" })
170
- .boolean("compress-content")
171
- .options("compress-CSS", { description: "Compress CSS stylesheets" })
172
- .boolean("compress-CSS")
173
- .options("compress-HTML", { description: "Compress HTML content" })
174
- .boolean("compress-HTML")
175
- .options("crawl-links", { description: "Crawl and save pages found via inner links" })
176
- .boolean("crawl-links")
177
- .options("crawl-inner-links-only", { description: "Crawl pages found via inner links only if they are hosted on the same domain" })
178
- .boolean("crawl-inner-links-only")
179
- .options("crawl-no-parent", { description: "Crawl pages found via inner links only if their URLs are not parent of the URL to crawl" })
180
- .boolean("crawl-no-parent")
181
- .options("crawl-load-session", { description: "Name of the file of the session to load (previously saved with --crawl-save-session or --crawl-sync-session)" })
182
- .string("crawl-load-session")
183
- .options("crawl-remove-url-fragment", { description: "Remove URL fragments found in links" })
184
- .boolean("crawl-remove-url-fragment")
185
- .options("crawl-save-session", { description: "Name of the file where to save the state of the session" })
186
- .string("crawl-save-session")
187
- .options("crawl-sync-session", { description: "Name of the file where to load and save the state of the session" })
188
- .string("crawl-sync-session")
189
- .options("crawl-max-depth", { description: "Max depth when crawling pages found in internal and external links (0: infinite)" })
190
- .number("crawl-max-depth")
191
- .options("crawl-external-links-max-depth", { description: "Max depth when crawling pages found in external links (0: infinite)" })
192
- .number("crawl-external-links-max-depth")
193
- .options("crawl-replace-urls", { description: "Replace URLs of saved pages with relative paths of saved pages on the filesystem" })
194
- .boolean("crawl-replace-urls")
195
- .options("crawl-rewrite-rule", { description: "Rewrite rule used to rewrite URLs of crawled pages" })
196
- .array("crawl-rewrite-rule")
197
- .options("dump-content", { description: "Dump the content of the processed page in the console ('true' when running in Docker)" })
198
- .boolean("dump-content")
199
- .options("emulate-media-feature", { description: "Emulate a media feature. The syntax is <name>:<value>, e.g. \"prefers-color-scheme:dark\" (puppeteer)" })
200
- .array("emulate-media-feature")
201
- .options("error-file")
202
- .string("error-file")
203
- .options("filename-template", { description: "Template used to generate the output filename (see help page of the extension for more info)" })
204
- .string("filename-template")
205
- .options("filename-conflict-action", { description: "Action when the filename is conflicting with existing one on the filesystem. The possible values are \"uniquify\" (default), \"overwrite\" and \"skip\"" })
206
- .string("filename-conflict-action")
207
- .options("filename-replacement-character", { description: "The character used for replacing invalid characters in filenames" })
208
- .string("filename-replacement-character")
209
- .options("filename-max-length", { description: "Specify the maximum length of the filename" })
210
- .number("filename-max-length")
211
- .options("filename-max-length-unit", { description: "Specify the unit of the maximum length of the filename ('bytes' or 'chars')" })
212
- .string("filename-max-length-unit")
213
- .options("replace-emojis-in-filename", { description: "Replace emojis in the filename with their unicode text representation" })
214
- .boolean("replace-emojis-in-filename")
215
- .options("group-duplicate-images", { description: "Group duplicate images into CSS custom properties" })
216
- .boolean("group-duplicate-images")
217
- .options("max-size-duplicate-images", { description: "Maximum sie in bytes of duplicate images stored as CSS custom properties" })
218
- .number("max-size-duplicate-images")
219
- .options("http-header", { description: "Extra HTTP header (puppeteer, jsdom)" })
220
- .array("http-header")
221
- .options("http-proxy-server", { description: "Proxy address (puppeteer)" })
222
- .string("http-proxy-server")
223
- .options("http-proxy-username", { description: "HTTP username (puppeteer)" })
224
- .string("http-proxy-username")
225
- .options("http-proxy-password", { description: "HTTP password (puppeteer)" })
226
- .string("http-proxy-password")
227
- .options("include-BOM", { description: "Include the UTF-8 BOM into the HTML page" })
228
- .boolean("include-BOM")
229
- .options("include-infobar", { description: "Include the infobar" })
230
- .boolean("include-infobar")
231
- .options("insert-meta-csp", { description: "Include a <meta> tag with a CSP to avoid potential requests to internet when viewing a page" })
232
- .boolean("insert-meta-csp")
233
- .options("load-deferred-images", { description: "Load deferred (a.k.a. lazy-loaded) images (puppeteer, webdriver-gecko, webdriver-chromium)" })
234
- .boolean("load-deferred-images")
235
- .options("load-deferred-images-dispatch-scroll-event", { description: "Dispatch 'scroll' event when loading deferred images" })
236
- .boolean("load-deferred-images-dispatch-scroll-event")
237
- .options("load-deferred-images-max-idle-time", { description: "Maximum delay of time to wait for deferred images in ms (puppeteer, webdriver-gecko, webdriver-chromium)" })
238
- .number("load-deferred-images-max-idle-time")
239
- .options("load-deferred-images-keep-zoom-level", { description: "Load deferred images by keeping zoomed out the page" })
240
- .boolean("load-deferred-images-keep-zoom-level")
241
- .options("load-deferred-images-before-frames", { description: "Load deferred frames before before saving fame contents" })
242
- .boolean("load-deferred-images-before-frames")
243
- .options("max-parallel-workers", { description: "Maximum number of browsers launched in parallel when processing a list of URLs (cf --urls-file)" })
244
- .number("max-parallel-workers")
245
- .options("max-resource-size-enabled", { description: "Enable removal of embedded resources exceeding a given size" })
246
- .boolean("max-resource-size-enabled")
247
- .options("max-resource-size", { description: "Maximum size of embedded resources in MB (i.e. images, stylesheets, scripts and iframes)" })
248
- .number("max-resource-size")
249
- .options("move-styles-in-head", { description: "Move style elements outside the head element into the head element" })
250
- .boolean("move-styles-in-head")
251
- .options("password", { description: "Password of the zip file" })
252
- .string("password")
253
- .options("remove-frames", { description: "Remove frames (puppeteer, webdriver-gecko, webdriver-chromium)" })
254
- .boolean("remove-frames")
255
- .options("remove-hidden-elements", { description: "Remove HTML elements which are not displayed" })
256
- .boolean("remove-hidden-elements")
257
- .options("remove-unused-styles", { description: "Remove unused CSS rules and unneeded declarations" })
258
- .boolean("remove-unused-styles")
259
- .options("remove-unused-fonts", { description: "Remove unused CSS font rules" })
260
- .boolean("remove-unused-fonts")
261
- .options("remove-saved-date", { description: "Remove saved date metadata in HTML header" })
262
- .boolean("remove-saved-date")
263
- .options("block-scripts", { description: "Block scripts" })
264
- .boolean("block-scripts")
265
- .options("block-audios", { description: "Block audio elements" })
266
- .boolean("block-audios")
267
- .options("block-videos", { description: "Block video elements" })
268
- .boolean("block-videos")
269
- .options("remove-alternative-fonts", { description: "Remove alternative fonts to the ones displayed" })
270
- .boolean("remove-alternative-fonts")
271
- .options("remove-alternative-medias", { description: "Remove alternative CSS stylesheets" })
272
- .boolean("remove-alternative-medias")
273
- .options("remove-alternative-images", { description: "Remove images for alternative sizes of screen" })
274
- .boolean("remove-alternative-images")
275
- .options("save-original-urls", { description: "Save the original URLS in the embedded contents" })
276
- .boolean("save-original-urls")
277
- .options("save-raw-page", { description: "Save the original page without interpreting it into the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
278
- .boolean("save-raw-page")
279
- .options("urls-file", { description: "Path to a text file containing a list of URLs (separated by a newline) to save" })
280
- .string("urls-file")
281
- .options("user-agent", { description: "User-agent of the browser (puppeteer, webdriver-gecko, webdriver-chromium)" })
282
- .string("user-agent")
283
- .options("user-script-enabled", { description: "Enable the event API allowing to execute scripts before the page is saved" })
284
- .boolean("user-script-enabled")
285
- .options("web-driver-executable-path", { description: "Path to Selenium WebDriver executable (webdriver-gecko, webdriver-chromium)" })
286
- .string("web-driver-executable-path")
287
- .options("self-extracting-archive", { description: "Create a self extracting HTML file" })
288
- .boolean("self-extracting-archive")
289
- .options("insert-text-body", { description: "Insert the text of the page into the self-extracting HTML file" })
290
- .boolean("insert-text-body")
291
- .options("create-root-directory", { description: "Create a root directory based on the timestamp" })
292
- .boolean("create-root-directory")
293
- .options("extract-data-from-page", { description: "Extract compressed data from the page instead of fetching the page in order to create universal self-extracting HTML files" })
294
- .boolean("extract-data-from-page")
295
- .options("prevent-appended-data", { description: "Prevent appending data after the compressed data when creating self-extracting HTML files" })
296
- .boolean("prevent-appended-data")
297
- .options("output-directory", { description: "Path to where to save files, this path must exist." })
298
- .string("output-directory")
299
- .argv;
300
- args.backgroundSave = true;
301
- args.compressCSS = args.compressCss;
302
- args.compressHTML = args.compressHtml;
303
- args.includeBOM = args.includeBom;
304
- args.crawlReplaceURLs = args.crawlReplaceUrls;
305
- args.crawlRemoveURLFragment = args.crawlRemoveUrlFragment;
306
- args.insertMetaCSP = args.insertMetaCsp;
307
- args.saveOriginalURLs = args.saveOriginalUrls;
308
- if (args.removeScripts) {
309
- args.blockScripts = true;
310
- }
311
- if (args.removeAudioSrc) {
312
- args.blockAudios = true;
26
+ const USAGE_TEXT = `single-file [url] [output]
27
+
28
+ Positionals:
29
+ url URL or path on the filesystem of the page to save [string]
30
+ output Output filename [string]`;
31
+
32
+ const OPTIONS_INFO = {
33
+ "accept-header-font": { description: "Accept header for fonts", type: "string", defaultValue: "application/font-woff2;q=1.0,application/font-woff;q=0.9,*/*;q=0.8" },
34
+ "accept-header-image": { description: "Accept header for images", type: "string", defaultValue: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" },
35
+ "accept-header-stylesheet": { description: "Accept header for stylesheets", type: "string", defaultValue: "text/css,*/*;q=0.1" },
36
+ "accept-header-script": { description: "Accept header for scripts", type: "string", defaultValue: "*/*" },
37
+ "accept-header-document": { description: "Accept header for documents", type: "string", defaultValue: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" },
38
+ "block-audios": { description: "Block audios", type: "boolean", defaultValue: true },
39
+ "block-fonts": { description: "Block fonts", type: "boolean" },
40
+ "block-images": { description: "Block images", type: "boolean" },
41
+ "block-scripts": { description: "Block scripts", type: "boolean", defaultValue: true },
42
+ "block-videos": { description: "Block videos", type: "boolean", defaultValue: true },
43
+ "block-mixed-content": { description: "Block mixed contents", type: "boolean" },
44
+ "browser-server": { description: "Server to connect to", type: "string" },
45
+ "browser-headless": { description: "Run the browser in headless mode", type: "boolean", defaultValue: true },
46
+ "browser-executable-path": { description: "Path to chrome/chromium executable", type: "string" },
47
+ "browser-width": { description: "Width of the browser viewport in pixels", type: "number", defaultValue: 1280 },
48
+ "browser-height": { description: "Height of the browser viewport in pixels", type: "number", defaultValue: 720 },
49
+ "browser-load-max-time": { description: "Maximum delay of time to wait for page loading in ms", type: "number", defaultValue: 60000 },
50
+ "browser-wait-delay": { description: "Time to wait before capturing the page in ms", type: "number" },
51
+ "browser-wait-until": { description: "When to consider the page is loaded (netWorkIdle, netWorkAlmostIdle, load, domContentLoaded)", type: "string", defaultValue: "networkIdle" },
52
+ "browser-wait-until-fallback": { description: "Retry with the next value of --browser-wait-until when a timeout error is thrown", type: "boolean", defaultValue: true },
53
+ "browser-debug": { description: "Enable debug mode", type: "boolean" },
54
+ "browser-script": { description: "Path of a script executed in the page (and all the frames) before it is loaded", type: "string[]" },
55
+ "browser-stylesheet": { description: "Path of a stylesheet file inserted into the page (and all the frames) after it is loaded", type: "string[]" },
56
+ "browser-args": { description: "Arguments provided as a JSON array and passed to the browser", type: "string" },
57
+ "browser-start-minimized": { description: "Minimize the browser", type: "boolean" },
58
+ "browser-cookie": { description: "Ordered list of cookie parameters separated by a comma (name,value,domain,path,expires,httpOnly,secure,sameSite,url)", type: "string[]" },
59
+ "browser-cookies-file": { description: "Path of the cookies file formatted as a JSON file or a Netscape text file", type: "string" },
60
+ "browser-ignore-insecure-certs": { description: "Ignore HTTPs errors", type: "boolean" },
61
+ "compress-content": { description: "Compress the output file into a ZIP file", type: "boolean" },
62
+ "compress-CSS": { description: "Compress CSS stylesheets", type: "boolean" },
63
+ "compress-HTML": { description: "Compress HTML content", type: "boolean", defaultValue: true },
64
+ "crawl-links": { description: "Crawl and save pages found via inner links", type: "boolean" },
65
+ "crawl-inner-links-only": { description: "Crawl pages found via inner links only if they are hosted on the same domain", type: "boolean", defaultValue: true },
66
+ "crawl-no-parent": { description: "Crawl pages found via inner links only if their URLs are not parent of the URL to crawl", type: "boolean" },
67
+ "crawl-load-session": { description: "Name of the file of the session to load (previously saved with --crawl-save-session or --crawl-sync-session)", type: "string" },
68
+ "crawl-remove-URL-fragment": { description: "Remove URL fragments found in links", type: "boolean", defaultValue: true },
69
+ "crawl-save-session": { description: "Name of the file where to save the state of the session", type: "string" },
70
+ "crawl-sync-session": { description: "Name of the file where to load and save the state of the session", type: "string" },
71
+ "crawl-max-depth": { description: "Max depth when crawling pages found in internal and external links (0: infinite)", type: "number", defaultValue: 1 },
72
+ "crawl-external-links-max-depth": { description: "Max depth when crawling pages found in external links (0: infinite)", type: "number", defaultValue: 1 },
73
+ "crawl-replace-URLs": { description: "Replace URLs of saved pages with relative paths of saved pages on the filesystem", type: "boolean" },
74
+ "crawl-rewrite-rule": { description: "Rewrite rule used to rewrite URLs of crawled pages", type: "string[]" },
75
+ "dump-content": { description: "Dump the content of the processed page in the console ('true' when running in Docker)", type: "boolean" },
76
+ "emulate-media-feature": { description: "Emulate a media feature. The syntax is <name>:<value>, e.g. \"prefers-color-scheme:dark\"", type: "string[]" },
77
+ "error-file": { description: "Path of the file where to save the error messages", type: "string" },
78
+ "filename-template": { description: "Template used to generate the output filename (see help page of the extension for more info)", type: "string", defaultValue: "%if-empty<{page-title}|No title> ({date-locale} {time-locale}).{filename-extension}" },
79
+ "filename-conflict-action": { description: "Action when the filename is conflicting with existing one on the filesystem. The possible values are \"uniquify\" (default), \"overwrite\" and \"skip\"", type: "string", defaultValue: "uniquify" },
80
+ "filename-replacement-character": { description: "The character used for replacing invalid characters in filenames", type: "string", defaultValue: "_" },
81
+ "filename-max-length": { description: "Specify the maximum length of the filename", type: "number", defaultValue: 192 },
82
+ "filename-max-length-unit": { description: "Specify the unit of the maximum length of the filename ('bytes' or 'chars')", type: "string", defaultValue: "bytes" },
83
+ "replace-emojis-in-filename": { description: "Replace emojis in the filename with their unicode text representation", type: "boolean" },
84
+ "group-duplicate-images": { description: "Group duplicate images into CSS custom properties", type: "boolean", defaultValue: true },
85
+ "max-size-duplicate-images": { description: "Maximum size in bytes of duplicate images stored as CSS custom properties", type: "number", defaultValue: 512 * 1024 },
86
+ "help": { description: "Show help", type: "boolean" },
87
+ "http-header": { description: "Extra HTTP header", type: "string[]" },
88
+ "http-proxy-server": { description: "Proxy address", type: "string" },
89
+ "http-proxy-username": { description: "HTTP username", type: "string" },
90
+ "http-proxy-password": { description: "HTTP password", type: "string" },
91
+ "include-BOM": { description: "Include the UTF-8 BOM into the HTML page", type: "boolean" },
92
+ "include-infobar": { description: "Include the infobar", type: "boolean" },
93
+ "insert-meta-CSP": { description: "Include a <meta> tag with a CSP to avoid potential requests to internet when viewing a page", type: "boolean", defaultValue: true },
94
+ "load-deferred-images": { description: "Load deferred (a.k.a. lazy-loaded) images", type: "boolean", defaultValue: true },
95
+ "load-deferred-images-dispatch-scroll-event": { description: "Dispatch 'scroll' event when loading deferred images", type: "boolean" },
96
+ "load-deferred-images-max-idle-time": { description: "Maximum delay of time to wait for deferred images in ms", type: "number", defaultValue: 1500 },
97
+ "load-deferred-images-keep-zoom-level": { description: "Load deferred images by keeping zoomed out the page", type: "boolean" },
98
+ "load-deferred-images-before-frames": { description: "Load deferred frames before before saving fame contents", type: "boolean" },
99
+ "max-parallel-workers": { description: "Maximum number of browsers launched in parallel when processing a list of URLs (cf --urls-file)", type: "number", defaultValue: 8 },
100
+ "max-resource-size-enabled": { description: "Enable removal of embedded resources exceeding a given size", type: "boolean" },
101
+ "max-resource-size": { description: "Maximum size of embedded resources in MB (i.e. images, stylesheets, scripts and iframes)", type: "number", defaultValue: 10 },
102
+ "move-styles-in-head": { description: "Move style elements outside the head element into the head element", type: "boolean" },
103
+ "password": { description: "Password of the zip file", type: "string" },
104
+ "remove-frames": { description: "Remove frames", type: "boolean" },
105
+ "remove-hidden-elements": { description: "Remove HTML elements which are not displayed", type: "boolean", defaultValue: true },
106
+ "remove-unused-styles": { description: "Remove unused CSS rules and unneeded declarations", type: "boolean", defaultValue: true },
107
+ "remove-unused-fonts": { description: "Remove unused CSS font rules", type: "boolean", defaultValue: true },
108
+ "remove-saved-date": { description: "Remove saved date metadata in HTML header", type: "boolean" },
109
+ "remove-alternative-fonts": { description: "Remove alternative fonts to the ones displayed", type: "boolean", defaultValue: true },
110
+ "remove-alternative-medias": { description: "Remove alternative CSS stylesheets", type: "boolean", defaultValue: true },
111
+ "remove-alternative-images": { description: "Remove images for alternative sizes of screen", type: "boolean", defaultValue: true },
112
+ "save-original-URLs": { description: "Save the original URLS in the embedded contents", type: "boolean" },
113
+ "save-raw-page": { description: "Save the original page without interpreting it into the browser", type: "boolean" },
114
+ "urls-file": { description: "Path to a text file containing a list of URLs (separated by a newline) to save", type: "string" },
115
+ "user-agent": { description: "User-agent of the browser", type: "string" },
116
+ "user-script-enabled": { description: "Enable the event API allowing to execute scripts before the page is saved", type: "boolean", defaultValue: true },
117
+ "self-extracting-archive": { description: "Create a self extracting HTML file", type: "boolean", defaultValue: true },
118
+ "insert-text-body": { description: "Insert the text of the page into the self-extracting HTML file", type: "boolean" },
119
+ "create-root-directory": { description: "Create a root directory based on the timestamp", type: "boolean" },
120
+ "extract-data-from-page": { description: "Extract compressed data from the page instead of fetching the page in order to create universal self-extracting HTML files", type: "boolean", defaultValue: true },
121
+ "prevent-appended-data": { description: "Prevent appending data after the compressed data when creating self-extracting HTML files", type: "boolean" },
122
+ "output-directory": { description: "Path to where to save files, this path must exist.", type: "string" }
123
+ };
124
+
125
+ const options = getOptions();
126
+ export default options;
127
+
128
+ function getOptions() {
129
+ const { positionals, options } = parseArgs();
130
+ if ((!positionals.length && !Object.keys(options).length) || positionals.length > 2 || options.help) {
131
+ console.log(USAGE_TEXT + "\n"); // eslint-disable-line no-console
132
+ console.log("Options:"); // eslint-disable-line no-console
133
+ Object.keys(OPTIONS_INFO).forEach(optionName => {
134
+ const optionInfo = getOptionInfo(optionName);
135
+ let optionType = optionInfo.type;
136
+ if (isArray(optionType)) {
137
+ optionType = optionType.replace("[]", "*");
138
+ }
139
+ const optionDescription = optionInfo.description;
140
+ const optionDefaultValue = optionInfo.defaultValue === undefined ? "" : `(default: ${JSON.stringify(optionInfo.defaultValue)})`;
141
+ console.log(` --${optionName}: ${optionDescription} <${optionType}> ${optionDefaultValue}`); // eslint-disable-line no-console
142
+ });
143
+ console.log(""); // eslint-disable-line no-console
144
+ exit(0);
145
+ }
146
+ Object.keys(OPTIONS_INFO).forEach(optionName => {
147
+ const optionKey = getOptionKey(optionName);
148
+ if (options[optionKey] === undefined) {
149
+ const optionInfo = getOptionInfo(optionName);
150
+ if (optionInfo.defaultValue !== undefined) {
151
+ options[optionKey] = OPTIONS_INFO[optionName].defaultValue;
152
+ }
153
+ }
154
+ });
155
+ options.acceptHeaders = {
156
+ font: options.acceptHeaderFont,
157
+ image: options.acceptHeaderImage,
158
+ stylesheet: options.acceptHeaderStylesheet,
159
+ script: options.acceptHeaderScript,
160
+ document: options.acceptHeaderDocument
161
+ };
162
+ delete options.acceptHeaderFont;
163
+ delete options.acceptHeaderImage;
164
+ delete options.acceptHeaderStylesheet;
165
+ delete options.acceptHeaderScript;
166
+ delete options.acceptHeaderDocument;
167
+ return { ...options, url: positionals[0], output: positionals[1] };
313
168
  }
314
- if (args.removeVideoSrc) {
315
- args.blockVideos = true;
169
+
170
+ function parseArgs() {
171
+ const parsedArgs = Array.from(args);
172
+ const positionals = [];
173
+ const options = {};
174
+ const result = { positionals, options: {} };
175
+ let argIndex = 0;
176
+ while (argIndex < parsedArgs.length) {
177
+ const arg = parsedArgs[argIndex];
178
+ const { argName, argValue, optionInfo } = parseArg(arg);
179
+ if (optionInfo) {
180
+ if (options[argName] === undefined) {
181
+ options[argName] = [];
182
+ }
183
+ let nextArgName;
184
+ if (argValue === undefined) {
185
+ while (
186
+ argIndex + 1 < parsedArgs.length &&
187
+ ({ argName: nextArgName } = parseArg(parsedArgs[argIndex + 1])) &&
188
+ (nextArgName === undefined || !getOptionInfo(nextArgName)) &&
189
+ isValid(optionInfo.type, parsedArgs[argIndex + 1]) &&
190
+ (isArray(optionInfo.type) || !options[argName].length)) {
191
+ options[argName].push(parsedArgs[argIndex + 1]);
192
+ argIndex++;
193
+ }
194
+ } else if (isValid(optionInfo.type, argValue) && (isArray(optionInfo.type) || !options[argName].length)) {
195
+ options[argName].push(argValue);
196
+ } else {
197
+ positionals.push(arg);
198
+ }
199
+ } else {
200
+ positionals.push(arg);
201
+ }
202
+ argIndex++;
203
+ }
204
+ Object.keys(options).forEach(optionName => {
205
+ const optionKey = getOptionKey(optionName);
206
+ const optionInfo = getOptionInfo(optionName);
207
+ let optionValue = options[optionName];
208
+ const isArrayType = isArray(optionInfo.type);
209
+ if (optionInfo.type.startsWith("boolean")) {
210
+ optionValue = optionValue.map(value => value == "true");
211
+ optionValue = isArrayType ?
212
+ optionValue.length ? optionValue : true :
213
+ optionValue.length ? optionValue[0] : true;
214
+ } else if (optionInfo.type.startsWith("number")) {
215
+ optionValue = optionValue.map(value => Number(value));
216
+ optionValue = isArrayType ?
217
+ optionValue.length ? optionValue : optionInfo.defaultValue || 0 :
218
+ optionValue.length ? optionValue[0] : optionInfo.defaultValue || 0;
219
+ } else {
220
+ optionValue = isArrayType ?
221
+ optionValue.length ? optionValue : optionInfo.defaultValue || "" :
222
+ optionValue.length ? optionValue[0] : optionInfo.defaultValue || "";
223
+ }
224
+ result.options[optionKey] = optionValue;
225
+ });
226
+ return result;
316
227
  }
317
- const headers = args.httpHeader;
318
- delete args.httpHeader;
319
- args.httpHeaders = {};
320
- headers.forEach(header => {
321
- const matchedHeader = header.match(/^(.*?):(.*)$/);
322
- if (matchedHeader.length == 3) {
323
- args.httpHeaders[matchedHeader[1].trim()] = matchedHeader[2].trimLeft();
228
+
229
+ function getOptionKey(optionName) {
230
+ let optionKey;
231
+ const optionInfo = getOptionInfo(optionName);
232
+ if (optionInfo) {
233
+ if (isArray(optionInfo.type)) {
234
+ optionKey = kebabToCamelCase(optionName + "s");
235
+ } else {
236
+ optionKey = kebabToCamelCase(optionName);
237
+ }
238
+ return optionKey;
324
239
  }
325
- });
326
- const cookies = args.browserCookie;
327
- delete args.browserCookie;
328
- args.browserCookies = cookies.map(cookieValue => {
329
- const value = cookieValue.split(/(?<!\\),/);
330
- return {
331
- name: value[0],
332
- value: value[1],
333
- domain: value[2] || undefined,
334
- path: value[3] || undefined,
335
- expires: value[4] && Number(value[4]) || undefined,
336
- httpOnly: value[5] && value[5] == "true" || undefined,
337
- secure: value[6] && value[5] == "true" || undefined,
338
- sameSite: value[7] || undefined,
339
- url: value[8] || undefined
340
- };
341
- });
342
- args.browserScripts = args.browserScript;
343
- delete args.browserScript;
344
- args.browserStylesheets = args.browserStylesheet;
345
- delete args.browserStylesheet;
346
- args.crawlRewriteRules = args.crawlRewriteRule;
347
- delete args.crawlRewriteRule;
348
- args.emulateMediaFeatures = args.emulateMediaFeature
349
- .map(value => {
350
- const splitValue = value.match(/^([^:]+):(.*)$/);
351
- if (splitValue.length >= 3) {
352
- return { name: splitValue[1].trim(), value: splitValue[2].trim() };
240
+ }
241
+
242
+ function parseArg(arg) {
243
+ const ARGS_REGEX = /^--([^=]+)(?:=(.*))?$/;
244
+ const parsedArg = arg.match(ARGS_REGEX);
245
+ if (parsedArg && parsedArg.length) {
246
+ let [_, argName, argValue] = parsedArg; // eslint-disable-line no-unused-vars
247
+ const optionInfo = getOptionInfo(argName);
248
+ if (argValue !== undefined &&
249
+ ((argValue.startsWith("\"") && argValue.endsWith("\"")) ||
250
+ (argValue.startsWith("'") && argValue.endsWith("'")))) {
251
+ argValue = argValue.substring(1, argValue.length - 1);
353
252
  }
354
- })
355
- .filter(identity => identity);
356
- delete args.emulateMediaFeature;
357
- Object.keys(args).filter(optionName => optionName.includes("-"))
358
- .forEach(optionName => delete args[optionName]);
359
- delete args["$0"];
360
- delete args["_"];
361
- module.exports = args;
253
+ return { argName, argValue, optionInfo };
254
+ } else {
255
+ return {};
256
+ }
257
+ }
258
+
259
+ function getOptionInfo(optionName) {
260
+ return OPTIONS_INFO[Object.keys(OPTIONS_INFO).find(keyName => keyName.toLowerCase() == optionName.toLowerCase())];
261
+ }
262
+
263
+ function kebabToCamelCase(optionName) {
264
+ return optionName.replace(/-([a-zA-Z])/g, g => g[1].toUpperCase());
265
+ }
266
+
267
+ function isValid(type, value) {
268
+ if (type.startsWith("boolean")) {
269
+ return value == "true" || value == "false";
270
+ } else if (type.startsWith("number")) {
271
+ return !isNaN(value);
272
+ } else {
273
+ return true;
274
+ }
275
+ }
276
+
277
+ function isArray(type) {
278
+ return type.endsWith("[]");
279
+ }