single-file-core 1.6.1 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/filename.js +62 -0
- package/core/helper.js +9 -32
- package/modules/css-fonts-minifier.js +34 -13
- package/modules/template-formatter.js +8 -2
- package/package.json +10 -2
- package/processors/compression/compression-constants.js +28 -0
- package/processors/compression/compression.js +3 -1
- package/.github/workflows/publish.yml +0 -88
- package/.github/workflows/test.yml +0 -31
- package/.github/workflows/vendor.yml +0 -39
- package/css-tree-build/README.md +0 -39
- package/css-tree-build/build.js +0 -66
- package/css-tree-build/package-lock.json +0 -511
- package/css-tree-build/package.json +0 -13
- package/deno.lock +0 -19
- package/doc/assets/singlefile-archive-byte-map.svg +0 -276
- package/doc/assets/singlefile-archive-lenses.svg +0 -245
- package/doc/singlefile-archive.md +0 -1968
- package/eslint.config.mjs +0 -74
- package/test/capture/README.md +0 -74
- package/test/capture/canonical-link.js +0 -50
- package/test/capture/common.js +0 -73
- package/test/capture/deferred-content-options.js +0 -71
- package/test/capture/dom.js +0 -26
- package/test/capture/maff-metadata.js +0 -205
- package/test/capture/resource-cap.js +0 -79
- package/test/capture/script-uri-sanitization.js +0 -82
- package/test/fixtures/README.md +0 -18
- package/test/fixtures/classic-digest.html +0 -5
- package/test/fixtures/generate-fixtures.sh +0 -31
- package/test/fixtures/multi-page-dedup.zip.html +0 -70
- package/test/fixtures/multi-page.zip.html +0 -26
- package/test/fixtures/single-page.zip.html +0 -11
- package/test/fixtures/site/multi-page/alpha.html +0 -2
- package/test/fixtures/site/multi-page/beta.html +0 -2
- package/test/fixtures/site/multi-page/index.html +0 -2
- package/test/fixtures/site/multi-page/sub/gamma.html +0 -2
- package/test/fixtures/site/multi-page-dedup/index.html +0 -2
- package/test/fixtures/site/multi-page-dedup/logo.png +0 -0
- package/test/fixtures/site/multi-page-dedup/one.html +0 -2
- package/test/fixtures/site/multi-page-dedup/shared.css +0 -3
- package/test/fixtures/site/multi-page-dedup/two.html +0 -2
- package/test/run.js +0 -109
- package/test/sfz-harness/README.md +0 -87
- package/test/sfz-harness/adopted-stylesheets-hook.js +0 -240
- package/test/sfz-harness/byte-map.js +0 -137
- package/test/sfz-harness/charset-round-trip.js +0 -161
- package/test/sfz-harness/check-determinism.js +0 -57
- package/test/sfz-harness/common.js +0 -154
- package/test/sfz-harness/content-type-sniffing.js +0 -83
- package/test/sfz-harness/css-fonts-minifier.js +0 -235
- package/test/sfz-harness/css-property-filter.js +0 -113
- package/test/sfz-harness/dom-stub.js +0 -13
- package/test/sfz-harness/entry-compression.js +0 -77
- package/test/sfz-harness/filename-characters.js +0 -55
- package/test/sfz-harness/filename-max-length.js +0 -69
- package/test/sfz-harness/font-face-composite.js +0 -135
- package/test/sfz-harness/format-rules.js +0 -617
- package/test/sfz-harness/gen-e2e-page.js +0 -14
- package/test/sfz-harness/inlined-functions.js +0 -82
- package/test/sfz-harness/option-wiring.js +0 -60
- package/test/sfz-harness/pages-archive.js +0 -318
- package/test/sfz-harness/pages-router.js +0 -143
- package/test/sfz-harness/relocation-cost.js +0 -94
- package/test/sfz-harness/search-triggers.js +0 -35
- package/test/sfz-harness/smoke.js +0 -10
- package/test/sfz-harness/stored-trigger.js +0 -95
- package/test/sfz-harness/trigger-seeds.json +0 -120
- package/test/sfz-harness/zip64.js +0 -77
- package/zip-build/README.md +0 -56
- package/zip-build/lib/fflate-streams.js +0 -47
- package/zip-build/lib/fflate.js +0 -2692
- package/zip-build/lib/zip-vendor-worker.js +0 -3
- package/zip-build/lib/zip-vendor.js +0 -9
- package/zip-build/lib/zip.js +0 -25
- package/zip-build/package-lock.json +0 -665
- package/zip-build/package.json +0 -15
- package/zip-build/reserved-property-names.json +0 -10630
- package/zip-build/rollup.config.js +0 -61
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Copyright 2010-2026 Gildas Lormeau
|
|
3
|
-
* contact : gildas.lormeau <at> gmail.com
|
|
4
|
-
*
|
|
5
|
-
* This file is part of SingleFile.
|
|
6
|
-
*
|
|
7
|
-
* The code in this file is free software: you can redistribute it and/or
|
|
8
|
-
* modify it under the terms of the GNU Affero General Public License
|
|
9
|
-
* (GNU AGPL) as published by the Free Software Foundation, either version 3
|
|
10
|
-
* of the License, or (at your option) any later version.
|
|
11
|
-
*
|
|
12
|
-
* The code in this file is distributed in the hope that it will be useful,
|
|
13
|
-
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
-
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
|
|
15
|
-
* General Public License for more details.
|
|
16
|
-
*
|
|
17
|
-
* As additional permission under GNU AGPL version 3 section 7, you may
|
|
18
|
-
* distribute UNMODIFIED VERSIONS OF THIS file without the copy of the GNU
|
|
19
|
-
* AGPL normally required by section 4, provided you include this license
|
|
20
|
-
* notice and a URL through which recipients can access the Corresponding
|
|
21
|
-
* Source.
|
|
22
|
-
*/
|
|
23
|
-
|
|
24
|
-
// `createArchive` builds the script of a self-extracting archive by serializing functions with
|
|
25
|
-
// toString() and pasting the text into the generated page. That page has no module scope, so
|
|
26
|
-
// anything such a function names has to be declared inside it. An import is the trap: it survives
|
|
27
|
-
// bundling, the call still reads correctly in the source, and the archive then fails at runtime
|
|
28
|
-
// with a bare ReferenceError while the page stays blank.
|
|
29
|
-
//
|
|
30
|
-
// It is worse than it looks, because a minifier can hide it. `display` called an imported
|
|
31
|
-
// `getDoctypeString`, and terser inlined that helper into the release bundle, so released
|
|
32
|
-
// archives worked while every unminified build produced archives that threw
|
|
33
|
-
// `ReferenceError: getDoctypeString is not defined` and rendered nothing.
|
|
34
|
-
//
|
|
35
|
-
// The invariant below is what makes that impossible to reintroduce: a module holding an inlined
|
|
36
|
-
// function declares that function and nothing else, so there is no module scope to close over.
|
|
37
|
-
|
|
38
|
-
const MODULES = [
|
|
39
|
-
{ path: "../../processors/compression/compression-display.js", functions: ["display"] },
|
|
40
|
-
{ path: "../../processors/compression/compression-extract.js", functions: ["extract"] },
|
|
41
|
-
{ path: "../../processors/compression/compression-router.js", functions: ["router"] }
|
|
42
|
-
];
|
|
43
|
-
const TOP_LEVEL_DECLARATION = /^(?:export\s+)?(?:async\s+)?(?:function|const|let|var|class)\s+([A-Za-z_$][\w$]*)/;
|
|
44
|
-
|
|
45
|
-
let failures = 0;
|
|
46
|
-
|
|
47
|
-
for (const { path, functions } of MODULES) {
|
|
48
|
-
const source = await Deno.readTextFile(new URL(path, import.meta.url));
|
|
49
|
-
const name = path.split("/").pop();
|
|
50
|
-
const lines = source.split("\n");
|
|
51
|
-
const imports = lines.filter(line => /^import[\s{]/.test(line) || /^\s*import\s*\(/.test(line));
|
|
52
|
-
check(name + " imports nothing", imports.length === 0, imports.join(" | "));
|
|
53
|
-
const declared = [];
|
|
54
|
-
lines.forEach(line => {
|
|
55
|
-
if (line === line.trimStart()) {
|
|
56
|
-
const match = line.match(TOP_LEVEL_DECLARATION);
|
|
57
|
-
if (match) {
|
|
58
|
-
declared.push(match[1]);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
});
|
|
62
|
-
const unexpected = declared.filter(identifier => !functions.includes(identifier));
|
|
63
|
-
check(name + " declares nothing at module scope but " + functions.join(", "),
|
|
64
|
-
unexpected.length === 0, unexpected.join(", "));
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
// the one inlined function that does live beside module scope, in compression.js, states the rule
|
|
68
|
-
// in a comment and binds every global it uses itself; assert that binding is still there
|
|
69
|
-
const compression = await Deno.readTextFile(new URL("../../processors/compression/compression.js", import.meta.url));
|
|
70
|
-
const getContentBody = compression.slice(compression.indexOf("async function getContent()"));
|
|
71
|
-
check("getContent binds its globals locally",
|
|
72
|
-
/const \{[^}]*\} = globalThis;/.test(getContentBody.slice(0, 2000)));
|
|
73
|
-
|
|
74
|
-
console.log(failures ? `\n${failures} check(s) FAILED` : "\nall checks passed");
|
|
75
|
-
Deno.exit(failures ? 1 : 0);
|
|
76
|
-
|
|
77
|
-
function check(label, condition, detail) {
|
|
78
|
-
if (!condition) {
|
|
79
|
-
failures++;
|
|
80
|
-
}
|
|
81
|
-
console.log((condition ? "PASS" : "FAIL") + " " + label + (condition || !detail ? "" : ": " + detail));
|
|
82
|
-
}
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
// The compression processor is reached through a whitelist in single-file.js. An option this
|
|
2
|
-
// module reads but the whitelist omits is undefined at every call site, so the feature works on
|
|
3
|
-
// the multi-page path (which calls createArchive directly) and silently does nothing on every
|
|
4
|
-
// single-page capture. That is how --declare-appended-data and --include-BOM both shipped inert,
|
|
5
|
-
// and no test caught either: the other suites call compression.process() directly, one layer
|
|
6
|
-
// below the wiring they would have to see.
|
|
7
|
-
import { PROCESS_OPTION_NAMES } from "../../processors/compression/compression.js";
|
|
8
|
-
|
|
9
|
-
// names matched by the source scan that are NOT caller options. Keeping them listed here rather
|
|
10
|
-
// than filtering them out silently is the point of this test: a new name must be classified as
|
|
11
|
-
// one thing or the other before the suite goes green again
|
|
12
|
-
const INTERNAL_OPTION_NAMES = [
|
|
13
|
-
// state the module sets on itself between build passes
|
|
14
|
-
"extraDataSize",
|
|
15
|
-
"extractDataFromPageTags",
|
|
16
|
-
"preventEmbeddedPdfEntry",
|
|
17
|
-
// supplied by compression-packager.js, which calls createArchive directly
|
|
18
|
-
"multiPageArchive",
|
|
19
|
-
// read for the root directory name, but no caller has ever passed it
|
|
20
|
-
"tabId",
|
|
21
|
-
// a different `options` object: the per-entry zip options built inside addFile()
|
|
22
|
-
"comment",
|
|
23
|
-
"level"
|
|
24
|
-
];
|
|
25
|
-
|
|
26
|
-
let failed = false;
|
|
27
|
-
|
|
28
|
-
function check(label, actual, expected) {
|
|
29
|
-
const ok = actual === expected;
|
|
30
|
-
console.log(`${ok ? "PASS" : "FAIL"} ${label}: ${actual}${ok ? "" : " (expected " + expected + ")"}`);
|
|
31
|
-
failed ||= !ok;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
const compressionSource = await Deno.readTextFile(new URL("../../processors/compression/compression.js", import.meta.url));
|
|
35
|
-
const singleFileSource = await Deno.readTextFile(new URL("../../single-file.js", import.meta.url));
|
|
36
|
-
const readNames = [...new Set([...compressionSource.matchAll(/options\.([A-Za-z0-9_]+)/g)].map(match => match[1]))].sort();
|
|
37
|
-
|
|
38
|
-
const unclassified = readNames.filter(name => !PROCESS_OPTION_NAMES.includes(name) && !INTERNAL_OPTION_NAMES.includes(name));
|
|
39
|
-
check("every option compression.js reads is classified", unclassified.join(", "), "");
|
|
40
|
-
|
|
41
|
-
const unread = PROCESS_OPTION_NAMES.filter(name => !readNames.includes(name));
|
|
42
|
-
check("no declared caller option is dead", unread.join(", "), "");
|
|
43
|
-
|
|
44
|
-
const misclassified = INTERNAL_OPTION_NAMES.filter(name => PROCESS_OPTION_NAMES.includes(name));
|
|
45
|
-
check("no option is both a caller option and internal", misclassified.join(", "), "");
|
|
46
|
-
|
|
47
|
-
check("the declared list is sorted", PROCESS_OPTION_NAMES.join(), [...PROCESS_OPTION_NAMES].sort().join());
|
|
48
|
-
|
|
49
|
-
// the guard is only worth anything while single-file.js builds its argument from the list; a
|
|
50
|
-
// literal object there would drift again, which is exactly the bug this file exists to prevent
|
|
51
|
-
check("single-file.js builds its argument from the list", singleFileSource.includes("PROCESS_OPTION_NAMES"), true);
|
|
52
|
-
check("single-file.js keeps no literal option whitelist", /insertTextBody:\s*options\.insertTextBody/.test(singleFileSource), false);
|
|
53
|
-
|
|
54
|
-
check("includeBOM reaches the compressed path", PROCESS_OPTION_NAMES.includes("includeBOM"), true);
|
|
55
|
-
|
|
56
|
-
if (failed) {
|
|
57
|
-
console.log("FAILED");
|
|
58
|
-
Deno.exit(1);
|
|
59
|
-
}
|
|
60
|
-
console.log("OK");
|
|
@@ -1,318 +0,0 @@
|
|
|
1
|
-
// createPagesArchive packs several single-page archives into one. Nothing exercised it until this
|
|
2
|
-
// file: the module was reachable only through a crawl, so a change to the folder layout, the
|
|
3
|
-
// manifest, the deduplication or the table of contents broke nothing that anyone ran.
|
|
4
|
-
//
|
|
5
|
-
// Three of its rules are worth stating, because they look arbitrary in the code:
|
|
6
|
-
//
|
|
7
|
-
// - the first page is stored at the ROOT and the others under pages/N/, unless
|
|
8
|
-
// createRootDirectory asks for a folder for the first page too. A page's resources travel with
|
|
9
|
-
// it, so either layout resolves; what the root buys is a reader who unzips the archive and
|
|
10
|
-
// opens index.html without being told where to look, and what it costs is that the first page
|
|
11
|
-
// shares the root with the archive's own files.
|
|
12
|
-
// - a duplicate entry becomes a SYMLINK rather than being dropped. The router resolves it from
|
|
13
|
-
// the alias map in the manifest and never reads it, but a plain unzip has to produce complete
|
|
14
|
-
// page folders, and only a symlink gives both.
|
|
15
|
-
// - the titles written into the table of contents are CRAWLED, so they are attacker-controlled
|
|
16
|
-
// text going into an href attribute and into element content. Both escapers are checked here.
|
|
17
|
-
import "./dom-stub.js";
|
|
18
|
-
import { makePageData, makeOptions, runProcess, freezeDate } from "./common.js";
|
|
19
|
-
import { createPagesArchive } from "../../processors/compression/compression-packager.js";
|
|
20
|
-
import { ZipReader, ZipWriter, BlobReader, TextReader, TextWriter, Uint8ArrayWriter } from "../../vendor/zip/zip.js";
|
|
21
|
-
|
|
22
|
-
// a title as it comes back from a crawl: the quote closes the href it is written into, the angle
|
|
23
|
-
// bracket opens an element, and the ampersand is what a naive escaper double-encodes
|
|
24
|
-
const HOSTILE_TITLE = "Intro & \"start\" <b>";
|
|
25
|
-
const SYMLINK_UNIX_MODE = 0o120777;
|
|
26
|
-
const SOURCE_DATE = new Date("2021-03-04T05:06:08Z");
|
|
27
|
-
|
|
28
|
-
let failed = false;
|
|
29
|
-
|
|
30
|
-
const pages = [
|
|
31
|
-
await makePage(1, { url: "https://example.com/docs/intro.html", title: HOSTILE_TITLE, originalUrls: ["https://example.com/docs/"] }),
|
|
32
|
-
await makePage(2, { url: "https://example.com/docs/api/reference.html", title: "Reference" })
|
|
33
|
-
];
|
|
34
|
-
|
|
35
|
-
{
|
|
36
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions()));
|
|
37
|
-
const manifest = JSON.parse(await readEntry(entries, "sfz-pages.json"));
|
|
38
|
-
check("the first page is stored at the root of the archive", entries.has("index.html"), true);
|
|
39
|
-
check("a later page is stored in a folder of its own", entries.has("pages/2/index.html"), true);
|
|
40
|
-
check("the manifest names the path of every page",
|
|
41
|
-
manifest.pages.map(page => page.path).join(" "), " pages/2/");
|
|
42
|
-
check("the manifest names the url of every page",
|
|
43
|
-
manifest.pages.map(page => page.url).join(" "), "https://example.com/docs/intro.html https://example.com/docs/api/reference.html");
|
|
44
|
-
check("the manifest keeps the title a page was saved with", manifest.pages[0].title, HOSTILE_TITLE);
|
|
45
|
-
// a page reached through several urls has to answer to all of them, or a link to the url the
|
|
46
|
-
// crawler did not settle on leaves the archive
|
|
47
|
-
check("the manifest keeps the urls a page was reached by",
|
|
48
|
-
(manifest.pages[0].originalUrls || []).join(" "), "https://example.com/docs/");
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
// createRootDirectory gives the first page a folder of its own. Without it the first page is
|
|
52
|
-
// written at the root, mixed in with the archive's own files, which is the reason the router needs
|
|
53
|
-
// a special case at all: belongsToPage() has to read "everything not under pages/ and not named
|
|
54
|
-
// sfz-*" as the first page. With every page under pages/N/ that rule is a plain prefix match.
|
|
55
|
-
{
|
|
56
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions({ createRootDirectory: true, tocPage: true })));
|
|
57
|
-
const manifest = JSON.parse(await readEntry(entries, "sfz-pages.json"));
|
|
58
|
-
const toc = await readEntry(entries, "sfz-toc.html");
|
|
59
|
-
check("the first page is stored in a folder of its own when a root directory is asked for",
|
|
60
|
-
entries.has("pages/1/index.html"), true);
|
|
61
|
-
check("and the first page is no longer at the root", entries.has("index.html"), false);
|
|
62
|
-
check("the manifest names the folder of the first page too",
|
|
63
|
-
manifest.pages.map(page => page.path).join(" "), "pages/1/ pages/2/");
|
|
64
|
-
check("the table of contents links to the first page in its folder",
|
|
65
|
-
toc.includes("href=\"pages/1/index.html\""), true);
|
|
66
|
-
// the archive's own files stay at the root whatever the option says: the router finds them by
|
|
67
|
-
// exact name, and an archive whose sfz-pages.json moved stops being read as multi-page at all
|
|
68
|
-
check("the archive's own files are the only thing left at the root",
|
|
69
|
-
[...entries.keys()].filter(filename => !filename.includes("/")).sort().join(" "),
|
|
70
|
-
"sfz-pages.json sfz-toc.html");
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// deduplication writes the link target relative to the folder the repeated entry sits in. With the
|
|
74
|
-
// first page at the root that walk never has a common prefix to drop; with both pages in folders it
|
|
75
|
-
// has to climb out of one and back into the other, which nothing exercised before
|
|
76
|
-
{
|
|
77
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions({ createRootDirectory: true, dedupPages: true })));
|
|
78
|
-
check("a repeated entry points across folders at the one that was kept",
|
|
79
|
-
await readEntry(entries, "pages/2/styles.css"), "../1/styles.css");
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// the router reads these two out of the manifest, and "auto" is the absence of a choice rather
|
|
83
|
-
// than a value: writing it would pin the default of the day into every archive
|
|
84
|
-
{
|
|
85
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions({ markUnarchivedLinks: true, pageTransitions: "slide" })));
|
|
86
|
-
const manifest = JSON.parse(await readEntry(entries, "sfz-pages.json"));
|
|
87
|
-
check("the manifest records that unarchived links are marked", manifest.markUnarchivedLinks, true);
|
|
88
|
-
check("the manifest records the page transition it was given", manifest.pageTransitions, "slide");
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
{
|
|
92
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions({ pageTransitions: "auto" })));
|
|
93
|
-
const manifest = JSON.parse(await readEntry(entries, "sfz-pages.json"));
|
|
94
|
-
check("the default page transition is not written to the manifest", "pageTransitions" in manifest, false);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
// Both fixtures declare the same stylesheet, so pages/2/styles.css is byte-for-byte the entry
|
|
98
|
-
// already written at the root.
|
|
99
|
-
{
|
|
100
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions({ dedupPages: true })));
|
|
101
|
-
const manifest = JSON.parse(await readEntry(entries, "sfz-pages.json"));
|
|
102
|
-
const duplicate = entries.get("pages/2/styles.css");
|
|
103
|
-
check("a repeated entry is still present after deduplication", Boolean(duplicate), true);
|
|
104
|
-
check("the repeated entry points at the one that was kept",
|
|
105
|
-
await readEntry(entries, "pages/2/styles.css"), "../../styles.css");
|
|
106
|
-
// without the mode, tar and unzip write the path as the FILE CONTENT and the page folder ends
|
|
107
|
-
// up holding a text file where a stylesheet belongs
|
|
108
|
-
check("the repeated entry carries the unix symlink mode",
|
|
109
|
-
duplicate.externalFileAttributes >>> 16, SYMLINK_UNIX_MODE);
|
|
110
|
-
// read through a default, so that a manifest with no aliases at all reports as a failed check
|
|
111
|
-
// rather than throwing and taking every check after it down with it
|
|
112
|
-
check("the manifest maps the repeated entry to the one it aliases",
|
|
113
|
-
(manifest.aliases || {})["pages/2/styles.css"], "styles.css");
|
|
114
|
-
check("an entry that is not repeated is left alone",
|
|
115
|
-
"pages/2/index.html" in (manifest.aliases || {}), false);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
{
|
|
119
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions()));
|
|
120
|
-
const manifest = JSON.parse(await readEntry(entries, "sfz-pages.json"));
|
|
121
|
-
check("nothing is aliased when deduplication is off", "aliases" in manifest, false);
|
|
122
|
-
check("a repeated entry is stored whole when deduplication is off",
|
|
123
|
-
(await readEntry(entries, "pages/2/styles.css")).includes("font-family"), true);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// Every entry is copied with passThrough, i.e. its stored bytes are written back without being
|
|
127
|
-
// decompressed, so everything that DESCRIBES those bytes has to travel with them. Forwarding a
|
|
128
|
-
// subset does not make a partial copy, it makes a corrupt one: the writer cannot tell that a
|
|
129
|
-
// compression method it did not choose describes data it is about to store verbatim.
|
|
130
|
-
//
|
|
131
|
-
// The pages the rest of this file uses cannot show that, because they are written by the same
|
|
132
|
-
// writer with the same defaults as the archive they are copied into, so every value the copy
|
|
133
|
-
// drops is replaced by the one it had. This page carries values the packager's own defaults do
|
|
134
|
-
// not produce. It is the first page, so it is copied to the root under its own names.
|
|
135
|
-
{
|
|
136
|
-
const metadataPages = [await makeMetadataPage(), pages[1]];
|
|
137
|
-
const sourceEntries = await readArchive(await metadataPages[0].getData());
|
|
138
|
-
const entries = await readArchive(await createPagesArchive(metadataPages, packagerOptions()));
|
|
139
|
-
const copied = [...sourceEntries.keys()].filter(filename => entries.has(filename));
|
|
140
|
-
check("every entry of the first page is copied", copied.length, sourceEntries.size);
|
|
141
|
-
for (const property of ["comment", "compressionMethod", "uncompressedSize", "crc32", "filenameUTF8", "externalFileAttributes", "versionMadeBy", "internalFileAttributes", "uid", "gid", "directory"]) {
|
|
142
|
-
check("a copied entry keeps its " + property,
|
|
143
|
-
copied.every(filename => entries.get(filename)[property] === sourceEntries.get(filename)[property]), true);
|
|
144
|
-
}
|
|
145
|
-
for (const property of ["lastModDate", "creationDate", "lastAccessDate"]) {
|
|
146
|
-
check("a copied entry keeps its " + property,
|
|
147
|
-
copied.every(filename => dateOf(entries.get(filename)[property]) === dateOf(sourceEntries.get(filename)[property])), true);
|
|
148
|
-
}
|
|
149
|
-
// the level bits say how hard the deflater tried, and a copy that drops them reports the
|
|
150
|
-
// packager's default instead of the level the entry was actually written at
|
|
151
|
-
check("a copied entry keeps the deflate level it was written at",
|
|
152
|
-
copied.every(filename => entries.get(filename).bitFlag.level === sourceEntries.get(filename).bitFlag.level), true);
|
|
153
|
-
// zip.js rebuilds the fields it interprets itself, so what has to survive is the rest
|
|
154
|
-
check("a copied entry keeps an extra field zip.js does not interpret",
|
|
155
|
-
extraFieldOf(entries.get("styles.css")), extraFieldOf(sourceEntries.get("styles.css")));
|
|
156
|
-
const sameBytes = await Promise.all(copied.map(async filename => equalData(
|
|
157
|
-
await readRawData(entries.get(filename)),
|
|
158
|
-
await readRawData(sourceEntries.get(filename)))));
|
|
159
|
-
check("a copied entry holds the bytes it was read from", sameBytes.every(Boolean), true);
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
{
|
|
163
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions({ tocPage: true })));
|
|
164
|
-
const toc = await readEntry(entries, "sfz-toc.html");
|
|
165
|
-
check("the table of contents page is stored when it is asked for", entries.has("sfz-toc.html"), true);
|
|
166
|
-
check("the table of contents links to the page at the root", toc.includes("href=\"index.html\""), true);
|
|
167
|
-
check("the table of contents links to the page in its folder", toc.includes("href=\"pages/2/index.html\""), true);
|
|
168
|
-
// the escaped form has to be there AND the raw form has to be absent: a title written twice,
|
|
169
|
-
// once escaped and once not, passes any check that only looks for the escaped one
|
|
170
|
-
check("a crawled title is escaped into the table of contents",
|
|
171
|
-
toc.includes("Intro & "start" <b>"), true);
|
|
172
|
-
check("a crawled title is not also written raw", toc.includes(HOSTILE_TITLE), false);
|
|
173
|
-
// the groups are details/summary and nothing else on purpose: the page has to stay usable
|
|
174
|
-
// after a plain unzip, where no script runs
|
|
175
|
-
check("pages are grouped by the segments of their path",
|
|
176
|
-
toc.includes("<details open><summary>docs</summary>"), true);
|
|
177
|
-
check("the table of contents needs no script", toc.includes("<script"), false);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
{
|
|
181
|
-
const entries = await readArchive(await createPagesArchive(pages, packagerOptions()));
|
|
182
|
-
check("no table of contents page is stored when it is not asked for", entries.has("sfz-toc.html"), false);
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
// one origin is the whole archive's origin and adding it to every path would say nothing; two
|
|
186
|
-
// origins make it the first thing that tells two pages apart
|
|
187
|
-
{
|
|
188
|
-
const mixedPages = [pages[0], await makePage(3, { url: "https://other.example.org/notes.html", title: "Notes" })];
|
|
189
|
-
const entries = await readArchive(await createPagesArchive(mixedPages, packagerOptions({ tocPage: true })));
|
|
190
|
-
const toc = await readEntry(entries, "sfz-toc.html");
|
|
191
|
-
check("pages from several origins are grouped by origin",
|
|
192
|
-
toc.includes("<summary>https://example.com</summary>"), true);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
// the prelude list is read without decompressing anything, by tools that never extract the
|
|
196
|
-
// archive, so it is the only place the pages are named in plain text
|
|
197
|
-
{
|
|
198
|
-
const bytes = await createPagesArchive(pages, packagerOptions({ pageList: true }));
|
|
199
|
-
const prelude = new TextDecoder("windows-1252").decode(bytes);
|
|
200
|
-
check("the prelude lists the pages when the page list is asked for",
|
|
201
|
-
prelude.includes("<a href=\"https://example.com/docs/api/reference.html\">Reference</a>"), true);
|
|
202
|
-
// anchored on the link, not on the escaped text alone: the same title is also written into the
|
|
203
|
-
// wrapper's own <title>, which the writer escapes the same way, so a search for the escaped
|
|
204
|
-
// form anywhere in the archive passes even when the page list itself is written raw
|
|
205
|
-
check("the prelude escapes a crawled title too",
|
|
206
|
-
prelude.includes("<a href=\"https://example.com/docs/intro.html\">Intro & "start" <b></a>"), true);
|
|
207
|
-
check("the prelude is not written when the page list is not asked for",
|
|
208
|
-
new TextDecoder("windows-1252").decode(await createPagesArchive(pages, packagerOptions())).includes("<nav><ul>"), false);
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
// the options handed to the archive writer are DERIVED from PROCESS_OPTION_NAMES, not hand-listed.
|
|
212
|
-
// The hand copy carried eleven names and silently dropped maxAppendedDataLength, so
|
|
213
|
-
// --max-appended-data-length did nothing on any multi-page save and nothing failed for months.
|
|
214
|
-
// A one-byte budget has to reach the writer, where it is indistinguishable from refusing to append
|
|
215
|
-
{
|
|
216
|
-
const unfreeze = freezeDate();
|
|
217
|
-
try {
|
|
218
|
-
const budgeted = await createPagesArchive(pages, packagerOptions({ maxAppendedDataLength: 1 }));
|
|
219
|
-
const prevented = await createPagesArchive(pages, packagerOptions({ preventAppendedData: true }));
|
|
220
|
-
const unbudgeted = await createPagesArchive(pages, packagerOptions());
|
|
221
|
-
check("a one-byte appended-data budget reaches the archive writer", equalData(budgeted, prevented), true);
|
|
222
|
-
check("and appending is what the writer does without one", equalData(unbudgeted, prevented), false);
|
|
223
|
-
} finally {
|
|
224
|
-
unfreeze();
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
// `password` is the one name the derivation must NOT forward. An encrypted multi-page archive
|
|
229
|
-
// cannot be written yet, and forwarding the password would half-ship it: the writer would start
|
|
230
|
-
// withholding the prologue's title as if the archive were encrypted, while the table of contents
|
|
231
|
-
// and every entry comment — each one a resource URL — kept riding in that same cleartext prologue
|
|
232
|
-
{
|
|
233
|
-
const prologue = new TextDecoder("windows-1252").decode(await createPagesArchive(pages, packagerOptions({ password: "secret" })));
|
|
234
|
-
check("a password is not forwarded to the archive writer",
|
|
235
|
-
prologue.includes("<title>Intro & "start" <b></title>"), true);
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
console.log(failed ? "\nsome checks FAILED" : "\nall checks passed");
|
|
239
|
-
Deno.exit(failed ? 1 : 0);
|
|
240
|
-
|
|
241
|
-
// each page of a multi-page archive is a single-page archive, so the fixtures are built by the
|
|
242
|
-
// writer the rest of the harness already covers
|
|
243
|
-
async function makePage(seed, { url, title, originalUrls }) {
|
|
244
|
-
const pageData = makePageData(seed, 2 * 1024);
|
|
245
|
-
pageData.title = title;
|
|
246
|
-
const { bytes } = await runProcess(pageData, makeOptions({ url }));
|
|
247
|
-
return { url, title, originalUrls, getData: async () => bytes };
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
// a page archive holding, on purpose, nothing the packager's own writer would produce by default:
|
|
251
|
-
// a directory record, a name that needs the language encoding flag, a stored entry beside one
|
|
252
|
-
// deflated at the highest level, unix ownership, a non-default version-made-by spec byte, an
|
|
253
|
-
// extra field zip.js does not interpret, and dates outside the one the packager pins on its writer
|
|
254
|
-
async function makeMetadataPage() {
|
|
255
|
-
const zipWriter = new ZipWriter(new Uint8ArrayWriter(), { lastModDate: SOURCE_DATE });
|
|
256
|
-
await zipWriter.add("folder/", null, { directory: true, comment: "a folder" });
|
|
257
|
-
await zipWriter.add("styles.css", new TextReader("body{font-family:serif}"), {
|
|
258
|
-
level: 9,
|
|
259
|
-
comment: "https://example.com/café.css",
|
|
260
|
-
creationDate: SOURCE_DATE,
|
|
261
|
-
lastAccessDate: SOURCE_DATE,
|
|
262
|
-
internalFileAttributes: 1,
|
|
263
|
-
msDosCompatible: false,
|
|
264
|
-
// only the high byte is rewritten to host Unix, so the 0x32 spec byte survives and the
|
|
265
|
-
// copied value differs from the 0x0300 both writers land on by default
|
|
266
|
-
versionMadeBy: 0x0332,
|
|
267
|
-
unixMode: 0o100755,
|
|
268
|
-
uid: 501,
|
|
269
|
-
gid: 20,
|
|
270
|
-
extraField: new Map([[0x7777, new Uint8Array([1, 2, 3, 4])]])
|
|
271
|
-
});
|
|
272
|
-
await zipWriter.add("café.txt", new TextReader("un café"), { level: 0 });
|
|
273
|
-
const bytes = await zipWriter.close();
|
|
274
|
-
return { url: "https://example.com/metadata.html", title: "Metadata", getData: async () => bytes };
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
function dateOf(value) {
|
|
278
|
-
return value === undefined ? undefined : value.getTime();
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
function extraFieldOf(entry) {
|
|
282
|
-
const value = entry.extraField && entry.extraField.get(0x7777);
|
|
283
|
-
return value ? value.data.join(",") : undefined;
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
function packagerOptions(overrides = {}) {
|
|
287
|
-
return {
|
|
288
|
-
selfExtractingArchive: true,
|
|
289
|
-
extractDataFromPage: true,
|
|
290
|
-
zipScript: "/* zip script stub */",
|
|
291
|
-
...overrides
|
|
292
|
-
};
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
async function readArchive(bytes) {
|
|
296
|
-
const zipReader = new ZipReader(new BlobReader(new Blob([bytes])));
|
|
297
|
-
const entries = await zipReader.getEntries();
|
|
298
|
-
await zipReader.close();
|
|
299
|
-
return new Map(entries.map(entry => [entry.filename, entry]));
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
function readEntry(entries, filename) {
|
|
303
|
-
return entries.get(filename).getData(new TextWriter());
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
function readRawData(entry) {
|
|
307
|
-
return entry.getData(new Uint8ArrayWriter(), { passThrough: true, checkCrc32: false });
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
function equalData(dataLeft, dataRight) {
|
|
311
|
-
return dataLeft.length == dataRight.length && dataLeft.every((value, index) => value == dataRight[index]);
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
function check(label, actual, expected) {
|
|
315
|
-
const ok = actual === expected;
|
|
316
|
-
console.log(`${ok ? "PASS" : "FAIL"} ${label}: ${actual}${ok ? "" : " (expected " + expected + ")"}`);
|
|
317
|
-
failed ||= !ok;
|
|
318
|
-
}
|
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
// The router picks the page a multi-page archive opens on. Nothing drove it until this file: the
|
|
2
|
-
// function is inlined into the archive as source text, so it only ever ran inside a saved page,
|
|
3
|
-
// and the suites around it checked what the packager WROTE rather than what a reader would see.
|
|
4
|
-
// That is how the table of contents shipped unreachable. It was stored, the route existed, and
|
|
5
|
-
// no link and no landing rule pointed at it, so --crawl-save-archive-toc looked like it did
|
|
6
|
-
// nothing at all.
|
|
7
|
-
//
|
|
8
|
-
// The rule this file pins: an archive that stores a table of contents opens on it, and one that
|
|
9
|
-
// does not opens on the first page. Two cases guard the edges of that rule. A route in the hash
|
|
10
|
-
// names a page explicitly and has to win over the landing rule, or every deep link into an
|
|
11
|
-
// archive would land on the table of contents instead. A hash that is NOT a route is a plain
|
|
12
|
-
// fragment, and the only page it can mean is the first one, which is where the archive used to
|
|
13
|
-
// land before the fragment was ever read.
|
|
14
|
-
import "./dom-stub.js";
|
|
15
|
-
import { makePageData, makeOptions, runProcess } from "./common.js";
|
|
16
|
-
import { createPagesArchive } from "../../processors/compression/compression-packager.js";
|
|
17
|
-
import { router } from "../../processors/compression/compression-router.js";
|
|
18
|
-
import * as zip from "../../vendor/zip/zip.js";
|
|
19
|
-
|
|
20
|
-
const ARCHIVE_URL = "https://example.com/archive.html";
|
|
21
|
-
|
|
22
|
-
let failed = false;
|
|
23
|
-
let openedEntries;
|
|
24
|
-
|
|
25
|
-
const pages = [
|
|
26
|
-
await makePage(1, { url: "https://example.com/docs/intro.html", title: "Intro" }),
|
|
27
|
-
await makePage(2, { url: "https://example.com/docs/api/reference.html", title: "Reference" })
|
|
28
|
-
];
|
|
29
|
-
const withTOC = await createPagesArchive(pages, packagerOptions({ tocPage: true }));
|
|
30
|
-
const withoutTOC = await createPagesArchive(pages, packagerOptions());
|
|
31
|
-
|
|
32
|
-
{
|
|
33
|
-
const content = await open(withTOC);
|
|
34
|
-
check("an archive holding a table of contents opens on it", content.includes("<h1>Table of contents</h1>"), true);
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
{
|
|
38
|
-
const content = await open(withoutTOC);
|
|
39
|
-
check("an archive holding no table of contents opens on the first page", content, "page at \"\"");
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
{
|
|
43
|
-
const content = await open(withTOC, "#sfz/pages/2/");
|
|
44
|
-
check("a route in the hash names the page to open", content, "page at \"pages/2/\"");
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// a bare fragment is what a hand-written link into the saved page looks like. The router scrolls
|
|
48
|
-
// to it after rendering, and the table of contents is not the document it belongs to
|
|
49
|
-
{
|
|
50
|
-
const content = await open(withTOC, "#introduction");
|
|
51
|
-
check("a hash that is not a route opens the first page", content, "page at \"\"");
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
// createRootDirectory moves the first page into pages/1/, so the landing rule has to come from the
|
|
55
|
-
// manifest rather than from the root. belongsToPage() also leaves its special case behind: while
|
|
56
|
-
// the first page is at the root it can only be described as "everything not under pages/ and not
|
|
57
|
-
// named sfz-*", and a page in a folder is selected by prefix like any other. The entries handed to
|
|
58
|
-
// extract are the assertion, because a landing path alone would still read right if that selection
|
|
59
|
-
// silently picked up the archive's own files
|
|
60
|
-
{
|
|
61
|
-
const rooted = await createPagesArchive(pages, packagerOptions({ createRootDirectory: true }));
|
|
62
|
-
const content = await open(rooted);
|
|
63
|
-
check("an archive with a root directory opens on the first page in its folder", content, "page at \"pages/1/\"");
|
|
64
|
-
check("and the router hands it only the entries of that folder",
|
|
65
|
-
openedEntries.join(" "), "pages/1/index.html pages/1/manifest.json pages/1/styles.css");
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
{
|
|
69
|
-
const rooted = await createPagesArchive(pages, packagerOptions({ createRootDirectory: true, tocPage: true }));
|
|
70
|
-
check("an archive with a root directory still opens on its table of contents",
|
|
71
|
-
(await open(rooted)).includes("<h1>Table of contents</h1>"), true);
|
|
72
|
-
check("and a route still names a page in it",
|
|
73
|
-
await open(rooted, "#sfz/pages/1/"), "page at \"pages/1/\"");
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
console.log(failed ? "\nsome checks FAILED" : "\nall checks passed");
|
|
77
|
-
Deno.exit(failed ? 1 : 0);
|
|
78
|
-
|
|
79
|
-
// the router reads its world out of globalThis and renders through the two functions it is given,
|
|
80
|
-
// so a stub of each is enough to see the page it chose. Only what the first render touches is
|
|
81
|
-
// stubbed here; navigation, scroll restoration and link marking read more of the DOM than this
|
|
82
|
-
async function open(bytes, hash = "") {
|
|
83
|
-
let displayed;
|
|
84
|
-
installEnvironment(hash);
|
|
85
|
-
await router(new Blob([bytes]), {
|
|
86
|
-
extract: (content, { entries, pagePath }) => {
|
|
87
|
-
openedEntries = entries.map(entry => entry.filename).sort();
|
|
88
|
-
return { docContent: "page at " + JSON.stringify(pagePath) };
|
|
89
|
-
},
|
|
90
|
-
display: (document, docContent) => displayed = docContent
|
|
91
|
-
});
|
|
92
|
-
return displayed;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
function installEnvironment(hash) {
|
|
96
|
-
// the router asks for web workers, which the archive serves from its own extension URL. There
|
|
97
|
-
// is no such URL here, so the request is answered with the synchronous codec instead
|
|
98
|
-
globalThis.zip = { ...zip, configure: options => zip.configure({ ...options, useWebWorkers: false }) };
|
|
99
|
-
globalThis.document = {
|
|
100
|
-
head: { appendChild() { } },
|
|
101
|
-
styleSheets: [],
|
|
102
|
-
createElement: () => ({ setAttribute() { }, remove() { } }),
|
|
103
|
-
querySelectorAll: () => [],
|
|
104
|
-
querySelector: () => null,
|
|
105
|
-
getElementById: () => null
|
|
106
|
-
};
|
|
107
|
-
globalThis.history = {
|
|
108
|
-
state: null,
|
|
109
|
-
scrollRestoration: "auto",
|
|
110
|
-
replaceState(state) {
|
|
111
|
-
this.state = state;
|
|
112
|
-
}
|
|
113
|
-
};
|
|
114
|
-
// Deno defines location as a getter that throws without --location, so it is replaced rather
|
|
115
|
-
// than assigned
|
|
116
|
-
Object.defineProperty(globalThis, "location", {
|
|
117
|
-
value: { href: ARCHIVE_URL + hash, hash },
|
|
118
|
-
configurable: true,
|
|
119
|
-
writable: true
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
async function makePage(seed, { url, title }) {
|
|
124
|
-
const pageData = makePageData(seed, 2 * 1024);
|
|
125
|
-
pageData.title = title;
|
|
126
|
-
const { bytes } = await runProcess(pageData, makeOptions({ url }));
|
|
127
|
-
return { url, title, getData: async () => bytes };
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
function packagerOptions(overrides = {}) {
|
|
131
|
-
return {
|
|
132
|
-
selfExtractingArchive: true,
|
|
133
|
-
extractDataFromPage: true,
|
|
134
|
-
zipScript: "/* zip script stub */",
|
|
135
|
-
...overrides
|
|
136
|
-
};
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
function check(label, actual, expected) {
|
|
140
|
-
const ok = actual === expected;
|
|
141
|
-
console.log(`${ok ? "PASS" : "FAIL"} ${label}: ${actual}${ok ? "" : " (expected " + expected + ")"}`);
|
|
142
|
-
failed ||= !ok;
|
|
143
|
-
}
|