sparkbun 0.2.7 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/bun/SparkBunConfig.ts +23 -3
- package/src/cli/archive-stream.ts +292 -0
- package/src/cli/index.ts +176 -73
package/package.json
CHANGED
|
@@ -167,6 +167,12 @@ export interface SparkBunConfig {
|
|
|
167
167
|
* Source directory (relative to project root or absolute). The
|
|
168
168
|
* directory's own name becomes the archive's top-level folder,
|
|
169
169
|
* so it extracts into the correct subdirectory.
|
|
170
|
+
*
|
|
171
|
+
* May also point at a PREBUILT `.tar` / `.tar.gz` / `.tar.zst`
|
|
172
|
+
* file, which is embedded verbatim (no per-build tar+compress —
|
|
173
|
+
* useful when a large payload is slow to compress and rarely
|
|
174
|
+
* changes). Format is taken from the extension; `format`/
|
|
175
|
+
* `compression` are ignored for prebuilt archives.
|
|
170
176
|
*/
|
|
171
177
|
path: string;
|
|
172
178
|
/**
|
|
@@ -177,9 +183,23 @@ export interface SparkBunConfig {
|
|
|
177
183
|
*/
|
|
178
184
|
format?: "zstd" | "gzip";
|
|
179
185
|
/**
|
|
180
|
-
* Compression
|
|
181
|
-
*
|
|
182
|
-
*
|
|
186
|
+
* Compression tier.
|
|
187
|
+
* - "basic" — Bun's built-in CompressionStream (zstd ≈ level 3,
|
|
188
|
+
* gzip ≈ level 6). Fast, streaming, no level control. (default)
|
|
189
|
+
* - number — explicit level via node:zlib streaming transforms.
|
|
190
|
+
* zstd: 1–22 (22 = maximum; large window, smallest output).
|
|
191
|
+
* gzip: 1–9. Higher is smaller but slower to compress.
|
|
192
|
+
* - "none" — no compression; a plain `.tar`. For payloads that
|
|
193
|
+
* are already compressed (video, media) where recompression
|
|
194
|
+
* wastes build time. The manifest records format "none" and
|
|
195
|
+
* the app's extractor must skip its decompression step.
|
|
196
|
+
* @default "basic"
|
|
197
|
+
*/
|
|
198
|
+
compression?: "none" | "basic" | number;
|
|
199
|
+
/**
|
|
200
|
+
* Legacy alias for a numeric `compression` tier. Ignored when
|
|
201
|
+
* `compression` is set.
|
|
202
|
+
* @deprecated Use `compression` instead.
|
|
183
203
|
*/
|
|
184
204
|
level?: number;
|
|
185
205
|
}>;
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming, pure-Bun tar + compression for installer payload archives.
|
|
3
|
+
*
|
|
4
|
+
* The tar layer is hand-rolled ustar, emitted entry-by-entry with file bodies
|
|
5
|
+
* read in 8 MiB slices, so build memory stays bounded regardless of payload
|
|
6
|
+
* size — and there is no system `tar` dependency. (Shelling out to tar was a
|
|
7
|
+
* reliability problem: macOS bsdtar emits xattr pax headers that GNU tar
|
|
8
|
+
* rejects, and GNU tar parses "C:\..." as a remote-host spec.) The output is
|
|
9
|
+
* plain ustar — long paths via the name/prefix split, file modes preserved
|
|
10
|
+
* (executables stay executable), symlinks as typeflag '2' — readable by every
|
|
11
|
+
* tar implementation and by apps' streaming extractors.
|
|
12
|
+
*
|
|
13
|
+
* Compression tiers, chosen per archive (config `compression` field or
|
|
14
|
+
* `--compression` flag):
|
|
15
|
+
* - "basic" — Bun's built-in CompressionStream (zstd default ≈ level 3;
|
|
16
|
+
* gzip default ≈ level 6). Fast, streaming, no level control.
|
|
17
|
+
* The default tier.
|
|
18
|
+
* - <number> — explicit level via node:zlib streaming transforms (zstd 1–22,
|
|
19
|
+
* gzip 1–9). zstd 22 uses a large window; its ratio depends on
|
|
20
|
+
* it. Slower to build, smallest output.
|
|
21
|
+
* - "none" — no compression; a plain `.tar`. For payloads that are already
|
|
22
|
+
* compressed (video, archives) where recompression wastes time.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { readdirSync, statSync, lstatSync, readlinkSync, createWriteStream } from "node:fs";
|
|
26
|
+
import { pipeline } from "node:stream/promises";
|
|
27
|
+
import zlib from "node:zlib";
|
|
28
|
+
import { join, basename } from "node:path";
|
|
29
|
+
|
|
30
|
+
export type ArchiveFormat = "zstd" | "gzip";
|
|
31
|
+
export type ArchiveCompression = "none" | "basic" | number;
|
|
32
|
+
|
|
33
|
+
const READ_CHUNK = 8 * 1024 * 1024; // file read slice size (bounds memory)
|
|
34
|
+
|
|
35
|
+
/** File extension for an archive of the given format + compression. */
|
|
36
|
+
export function archiveExtension(format: ArchiveFormat, compression: ArchiveCompression): string {
|
|
37
|
+
if (compression === "none") return "tar";
|
|
38
|
+
return format === "gzip" ? "tar.gz" : "tar.zst";
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The effective numeric level recorded in the manifest: 0 for "none", the
|
|
43
|
+
* library default for "basic" (zstd 3 / gzip 6 — approximate; CompressionStream
|
|
44
|
+
* exposes no level), or the explicit number.
|
|
45
|
+
*/
|
|
46
|
+
export function effectiveLevel(format: ArchiveFormat, compression: ArchiveCompression): number {
|
|
47
|
+
if (compression === "none") return 0;
|
|
48
|
+
if (compression === "basic") return format === "gzip" ? 6 : 3;
|
|
49
|
+
return compression;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Human label for build logs, e.g. "zstd level 22", "zstd (basic)", "uncompressed". */
|
|
53
|
+
export function compressionLabel(format: ArchiveFormat, compression: ArchiveCompression): string {
|
|
54
|
+
if (compression === "none") return "uncompressed";
|
|
55
|
+
if (compression === "basic") return `${format} (basic)`;
|
|
56
|
+
return `${format} level ${compression}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** A POSIX ustar 512-byte header. Splits long paths across name/prefix. */
|
|
60
|
+
function ustarHeader(
|
|
61
|
+
path: string,
|
|
62
|
+
size: number,
|
|
63
|
+
type: "0" | "2" | "5",
|
|
64
|
+
mode: number,
|
|
65
|
+
linkname = "",
|
|
66
|
+
): Uint8Array {
|
|
67
|
+
const bl = (s: string) => Buffer.byteLength(s);
|
|
68
|
+
let name = path;
|
|
69
|
+
let prefix = "";
|
|
70
|
+
if (bl(name) > 100) {
|
|
71
|
+
// Split at a "/" so the suffix fits the 100-byte name field and the
|
|
72
|
+
// prefix fits the 155-byte prefix field (extractors rejoin them).
|
|
73
|
+
let split = -1;
|
|
74
|
+
for (let i = 0; i < path.length; i++) {
|
|
75
|
+
if (path[i] !== "/") continue;
|
|
76
|
+
if (bl(path.slice(0, i)) <= 155 && bl(path.slice(i + 1)) <= 100) {
|
|
77
|
+
split = i;
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
if (split < 0) throw new Error(`Path too long for ustar: ${path}`);
|
|
82
|
+
prefix = path.slice(0, split);
|
|
83
|
+
name = path.slice(split + 1);
|
|
84
|
+
}
|
|
85
|
+
if (bl(linkname) > 100) throw new Error(`Symlink target too long for ustar: ${linkname}`);
|
|
86
|
+
const h = Buffer.alloc(512);
|
|
87
|
+
h.write(name, 0);
|
|
88
|
+
const putOct = (off: number, len: number, v: number) =>
|
|
89
|
+
h.write(v.toString(8).padStart(len - 1, "0") + "\0", off);
|
|
90
|
+
putOct(100, 8, mode & 0o7777);
|
|
91
|
+
putOct(108, 8, 0); // uid
|
|
92
|
+
putOct(116, 8, 0); // gid
|
|
93
|
+
putOct(124, 12, size);
|
|
94
|
+
putOct(136, 12, 0); // mtime
|
|
95
|
+
h.write(type, 156);
|
|
96
|
+
h.write(linkname, 157);
|
|
97
|
+
h.write("ustar\0", 257);
|
|
98
|
+
h.write("00", 263);
|
|
99
|
+
h.write(prefix, 345);
|
|
100
|
+
h.fill(0x20, 148, 156); // checksum field = spaces while summing
|
|
101
|
+
let sum = 0;
|
|
102
|
+
for (let i = 0; i < 512; i++) sum += h[i]!;
|
|
103
|
+
h.write(sum.toString(8).padStart(6, "0") + "\0 ", 148);
|
|
104
|
+
return new Uint8Array(h.buffer, h.byteOffset, h.length);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
type WalkEntry = {
|
|
108
|
+
abs: string;
|
|
109
|
+
rel: string;
|
|
110
|
+
kind: "dir" | "file" | "symlink";
|
|
111
|
+
size: number;
|
|
112
|
+
mode: number;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
/** Depth-first walk; directories before their contents; "/" separators. */
|
|
116
|
+
function* walk(absDir: string, rel: string): Generator<WalkEntry> {
|
|
117
|
+
const entries = readdirSync(absDir, { withFileTypes: true }).sort((x, y) =>
|
|
118
|
+
x.name.localeCompare(y.name),
|
|
119
|
+
);
|
|
120
|
+
for (const e of entries) {
|
|
121
|
+
if (e.name === ".DS_Store") continue;
|
|
122
|
+
const abs = join(absDir, e.name);
|
|
123
|
+
const r = `${rel}/${e.name}`;
|
|
124
|
+
if (e.isSymbolicLink()) {
|
|
125
|
+
yield { abs, rel: r, kind: "symlink", size: 0, mode: 0o777 };
|
|
126
|
+
} else if (e.isDirectory()) {
|
|
127
|
+
yield { abs, rel: r, kind: "dir", size: 0, mode: lstatSync(abs).mode };
|
|
128
|
+
yield* walk(abs, r);
|
|
129
|
+
} else if (e.isFile()) {
|
|
130
|
+
const st = statSync(abs);
|
|
131
|
+
yield { abs, rel: r, kind: "file", size: st.size, mode: st.mode };
|
|
132
|
+
}
|
|
133
|
+
// Sockets/FIFOs/devices are skipped.
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The tar bytes for the tree rooted at `srcDir`, as a lazy async sequence.
|
|
139
|
+
* The directory's own name becomes the archive root.
|
|
140
|
+
*/
|
|
141
|
+
export async function* tarBytes(srcDir: string): AsyncGenerator<Uint8Array> {
|
|
142
|
+
const rootName = basename(srcDir);
|
|
143
|
+
yield ustarHeader(`${rootName}/`, 0, "5", statSync(srcDir).mode);
|
|
144
|
+
for (const e of walk(srcDir, rootName)) {
|
|
145
|
+
if (e.kind === "dir") {
|
|
146
|
+
yield ustarHeader(`${e.rel}/`, 0, "5", e.mode);
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
if (e.kind === "symlink") {
|
|
150
|
+
yield ustarHeader(e.rel, 0, "2", e.mode, readlinkSync(e.abs));
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
yield ustarHeader(e.rel, e.size, "0", e.mode);
|
|
154
|
+
const f = Bun.file(e.abs);
|
|
155
|
+
for (let off = 0; off < e.size; off += READ_CHUNK) {
|
|
156
|
+
yield new Uint8Array(await f.slice(off, Math.min(off + READ_CHUNK, e.size)).arrayBuffer());
|
|
157
|
+
}
|
|
158
|
+
const pad = (512 - (e.size % 512)) % 512;
|
|
159
|
+
if (pad) yield new Uint8Array(pad);
|
|
160
|
+
}
|
|
161
|
+
yield new Uint8Array(1024); // end-of-archive: two zero blocks
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Scan a prebuilt tar archive (optionally zstd/gzip compressed) and total its
|
|
166
|
+
* file payload — the uncompressedBytes/fileCount progress denominators the
|
|
167
|
+
* manifest needs. Streaming: headers are parsed and bodies skipped, so memory
|
|
168
|
+
* stays bounded.
|
|
169
|
+
*/
|
|
170
|
+
export async function scanTarArchive(
|
|
171
|
+
path: string,
|
|
172
|
+
format: ArchiveFormat | "none",
|
|
173
|
+
): Promise<{ uncompressedBytes: number; fileCount: number }> {
|
|
174
|
+
let stream = Bun.file(path).stream() as unknown as ReadableStream<Uint8Array>;
|
|
175
|
+
if (format !== "none") {
|
|
176
|
+
const ds = new DecompressionStream(
|
|
177
|
+
(format === "gzip" ? "gzip" : "zstd") as CompressionFormat,
|
|
178
|
+
) as unknown as ReadableWritablePair<Uint8Array, Uint8Array>;
|
|
179
|
+
stream = stream.pipeThrough(ds);
|
|
180
|
+
}
|
|
181
|
+
const reader = stream.getReader();
|
|
182
|
+
const td = new TextDecoder();
|
|
183
|
+
let buf: Uint8Array<ArrayBufferLike> = new Uint8Array(0);
|
|
184
|
+
let uncompressedBytes = 0;
|
|
185
|
+
let fileCount = 0;
|
|
186
|
+
let skip = 0; // bytes of body+padding left to discard
|
|
187
|
+
while (true) {
|
|
188
|
+
const { done, value } = await reader.read();
|
|
189
|
+
if (done) break;
|
|
190
|
+
if (buf.length === 0) buf = value;
|
|
191
|
+
else {
|
|
192
|
+
const merged = new Uint8Array(buf.length + value.length);
|
|
193
|
+
merged.set(buf, 0);
|
|
194
|
+
merged.set(value, buf.length);
|
|
195
|
+
buf = merged;
|
|
196
|
+
}
|
|
197
|
+
while (true) {
|
|
198
|
+
if (skip > 0) {
|
|
199
|
+
const n = Math.min(skip, buf.length);
|
|
200
|
+
buf = buf.subarray(n);
|
|
201
|
+
skip -= n;
|
|
202
|
+
if (skip > 0) break;
|
|
203
|
+
}
|
|
204
|
+
if (buf.length < 512) break;
|
|
205
|
+
const h = buf.subarray(0, 512);
|
|
206
|
+
buf = buf.subarray(512);
|
|
207
|
+
let allZero = true;
|
|
208
|
+
for (let i = 0; i < 512; i++) {
|
|
209
|
+
if (h[i] !== 0) { allZero = false; break; }
|
|
210
|
+
}
|
|
211
|
+
if (allZero) continue; // end-of-archive marker block
|
|
212
|
+
const size =
|
|
213
|
+
parseInt(td.decode(h.subarray(124, 136)).replace(/\0.*$/, "").trim() || "0", 8) || 0;
|
|
214
|
+
const typeflag = String.fromCharCode(h[156] || 0);
|
|
215
|
+
// Regular files count toward the payload; metadata entries (long-name
|
|
216
|
+
// 'L'/'K', pax 'x'/'g') and dirs/links don't, but their bodies are
|
|
217
|
+
// still skipped.
|
|
218
|
+
if (typeflag === "0" || typeflag === "\0") {
|
|
219
|
+
uncompressedBytes += size;
|
|
220
|
+
fileCount++;
|
|
221
|
+
}
|
|
222
|
+
skip = size + ((512 - (size % 512)) % 512);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return { uncompressedBytes, fileCount };
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** Wrap an async generator as a web ReadableStream (for CompressionStream). */
|
|
229
|
+
function toReadable(gen: AsyncGenerator<Uint8Array>): ReadableStream<Uint8Array> {
|
|
230
|
+
return new ReadableStream<Uint8Array>({
|
|
231
|
+
async pull(c) {
|
|
232
|
+
const { done, value } = await gen.next();
|
|
233
|
+
if (done) c.close();
|
|
234
|
+
else c.enqueue(value);
|
|
235
|
+
},
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Tar `srcDir` and write it to `destPath`, compressed per the tier. Streaming
|
|
241
|
+
* end to end: nothing close to the payload size is ever in memory.
|
|
242
|
+
*/
|
|
243
|
+
export async function writeTarArchive(
|
|
244
|
+
srcDir: string,
|
|
245
|
+
destPath: string,
|
|
246
|
+
format: ArchiveFormat,
|
|
247
|
+
compression: ArchiveCompression,
|
|
248
|
+
): Promise<void> {
|
|
249
|
+
const tar = tarBytes(srcDir);
|
|
250
|
+
|
|
251
|
+
if (compression === "none") {
|
|
252
|
+
await pipeline(tar, createWriteStream(destPath));
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
if (compression === "basic") {
|
|
257
|
+
// Bun's built-in CompressionStream. "zstd" is valid in Bun but missing
|
|
258
|
+
// from the lib.dom CompressionFormat union (and the stream generics
|
|
259
|
+
// predate Uint8Array<ArrayBuffer>), hence the casts.
|
|
260
|
+
const cs = new CompressionStream(
|
|
261
|
+
(format === "gzip" ? "gzip" : "zstd") as CompressionFormat,
|
|
262
|
+
) as unknown as ReadableWritablePair<Uint8Array, Uint8Array>;
|
|
263
|
+
const compressed = toReadable(tar).pipeThrough(cs);
|
|
264
|
+
const sink = Bun.file(destPath).writer();
|
|
265
|
+
const reader = compressed.getReader();
|
|
266
|
+
try {
|
|
267
|
+
while (true) {
|
|
268
|
+
const { done, value } = await reader.read();
|
|
269
|
+
if (done) break;
|
|
270
|
+
sink.write(value);
|
|
271
|
+
}
|
|
272
|
+
} finally {
|
|
273
|
+
await sink.end();
|
|
274
|
+
}
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// Explicit level via node:zlib streaming transforms. The zstd APIs exist in
|
|
279
|
+
// Bun's node:zlib but not yet in the bundled @types, hence the cast.
|
|
280
|
+
const z = zlib as unknown as {
|
|
281
|
+
createGzip(opts: { level: number }): NodeJS.ReadWriteStream;
|
|
282
|
+
createZstdCompress(opts: { params: Record<number, number> }): NodeJS.ReadWriteStream;
|
|
283
|
+
constants: { ZSTD_c_compressionLevel: number };
|
|
284
|
+
};
|
|
285
|
+
const transform =
|
|
286
|
+
format === "gzip"
|
|
287
|
+
? z.createGzip({ level: compression })
|
|
288
|
+
: z.createZstdCompress({
|
|
289
|
+
params: { [z.constants.ZSTD_c_compressionLevel]: compression },
|
|
290
|
+
});
|
|
291
|
+
await pipeline(tar, transform, createWriteStream(destPath));
|
|
292
|
+
}
|
package/src/cli/index.ts
CHANGED
|
@@ -33,6 +33,15 @@ import {
|
|
|
33
33
|
getMacOSBundleDisplayName,
|
|
34
34
|
} from "../shared/naming";
|
|
35
35
|
import { getTemplate, getTemplateNames } from "./templates/embedded";
|
|
36
|
+
import {
|
|
37
|
+
writeTarArchive,
|
|
38
|
+
scanTarArchive,
|
|
39
|
+
archiveExtension,
|
|
40
|
+
effectiveLevel,
|
|
41
|
+
compressionLabel,
|
|
42
|
+
type ArchiveFormat,
|
|
43
|
+
type ArchiveCompression,
|
|
44
|
+
} from "./archive-stream";
|
|
36
45
|
// MacOS named pipes hang at around 4KB
|
|
37
46
|
// @ts-expect-error - reserved for future use
|
|
38
47
|
const _MAX_CHUNK_SIZE = 1024 * 2;
|
|
@@ -64,6 +73,38 @@ function createTar(tarPath: string, cwd: string, entries: string[]) {
|
|
|
64
73
|
);
|
|
65
74
|
}
|
|
66
75
|
|
|
76
|
+
// Compile an executable via Bun.build({ compile }) with a retry around a known
|
|
77
|
+
// transient failure: on Windows the compile stamps PE version metadata through
|
|
78
|
+
// resource updates (EndUpdateResourceW), and antivirus/indexer scans of the
|
|
79
|
+
// freshly written exe intermittently fail that commit with "Failed to set
|
|
80
|
+
// Windows metadata: FailedToCommit". Clear any stale output first (a scanned
|
|
81
|
+
// leftover exe can hold locks), then retry a few times before giving up.
|
|
82
|
+
async function buildCompiledExeWithRetry(
|
|
83
|
+
options: Parameters<typeof Bun.build>[0],
|
|
84
|
+
outputPath: string,
|
|
85
|
+
): Promise<Awaited<ReturnType<typeof Bun.build>>> {
|
|
86
|
+
try { rmSync(outputPath, { force: true }); } catch {}
|
|
87
|
+
for (let attempt = 1; ; attempt++) {
|
|
88
|
+
try {
|
|
89
|
+
return await Bun.build(options);
|
|
90
|
+
} catch (err: any) {
|
|
91
|
+
// Bun surfaces compile errors as BuildMessage-like objects whose
|
|
92
|
+
// .message can be empty — the text only shows through Bun's inspect
|
|
93
|
+
// formatting, so match against every representation we can get.
|
|
94
|
+
let msg = "";
|
|
95
|
+
try {
|
|
96
|
+
msg = `${err?.message ?? ""} ${String(err)} ${Bun.inspect(err)}`;
|
|
97
|
+
} catch {}
|
|
98
|
+
if (attempt < 4 && /FailedToCommit|Failed to set Windows metadata/i.test(msg)) {
|
|
99
|
+
console.log(` PE metadata commit failed (attempt ${attempt}/4) — retrying in 2s...`);
|
|
100
|
+
await Bun.sleep(2000);
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
67
108
|
// Recursively measure a directory's total file bytes and file count.
|
|
68
109
|
// Used to embed accurate progress denominators in the installer archive manifest.
|
|
69
110
|
function measureDir(dir: string): { bytes: number; fileCount: number } {
|
|
@@ -84,28 +125,6 @@ function measureDir(dir: string): { bytes: number; fileCount: number } {
|
|
|
84
125
|
return { bytes, fileCount };
|
|
85
126
|
}
|
|
86
127
|
|
|
87
|
-
// Stream-compress a file with zstd or gzip at a specific level, using node:zlib
|
|
88
|
-
// transforms so build memory stays bounded regardless of input size. Web
|
|
89
|
-
// CompressionStream("zstd") is intentionally not used — it has no level control,
|
|
90
|
-
// and the high compression ratio depends on level 22's large window.
|
|
91
|
-
async function streamCompressFile(
|
|
92
|
-
srcPath: string,
|
|
93
|
-
destPath: string,
|
|
94
|
-
format: "zstd" | "gzip",
|
|
95
|
-
level: number,
|
|
96
|
-
): Promise<void> {
|
|
97
|
-
const zlib = require("node:zlib");
|
|
98
|
-
const { createReadStream, createWriteStream } = require("node:fs");
|
|
99
|
-
const { pipeline } = require("node:stream/promises");
|
|
100
|
-
const transform =
|
|
101
|
-
format === "gzip"
|
|
102
|
-
? zlib.createGzip({ level })
|
|
103
|
-
: zlib.createZstdCompress({
|
|
104
|
-
params: { [zlib.constants.ZSTD_c_compressionLevel]: level },
|
|
105
|
-
});
|
|
106
|
-
await pipeline(createReadStream(srcPath), transform, createWriteStream(destPath));
|
|
107
|
-
}
|
|
108
|
-
|
|
109
128
|
// this when run as an npm script this will be where the folder where package.json is.
|
|
110
129
|
const projectRoot = process.cwd();
|
|
111
130
|
|
|
@@ -3313,24 +3332,53 @@ usageDescriptions : ""}${urlTypes ? "\n" + urlTypes : ""}${documentTypes ?
|
|
|
3313
3332
|
|
|
3314
3333
|
// Compression defaults for archives supplied via --archive flags.
|
|
3315
3334
|
// (Archives declared in sparkbun.config.ts carry their own per-archive
|
|
3316
|
-
// format/
|
|
3317
|
-
const globalFormat = ((getFlag("--format=") as
|
|
3335
|
+
// format/compression, so each project can mix tight and fast compression.)
|
|
3336
|
+
const globalFormat = ((getFlag("--format=") as ArchiveFormat) || "zstd");
|
|
3318
3337
|
const globalLevelFlag = getFlag("--level=");
|
|
3319
|
-
const
|
|
3338
|
+
const globalCompressionFlag = getFlag("--compression=");
|
|
3339
|
+
|
|
3340
|
+
// Parse a compression tier: "none" | "basic" | numeric level. Validates
|
|
3341
|
+
// the level range for the format (zstd 1–22, gzip 1–9).
|
|
3342
|
+
const parseCompression = (
|
|
3343
|
+
raw: string | number,
|
|
3344
|
+
fmt: ArchiveFormat,
|
|
3345
|
+
source: string,
|
|
3346
|
+
): ArchiveCompression => {
|
|
3347
|
+
if (raw === "none" || raw === "basic") return raw;
|
|
3348
|
+
const level = Number(raw);
|
|
3349
|
+
const max = fmt === "gzip" ? 9 : 22;
|
|
3350
|
+
if (!Number.isInteger(level) || level < 1 || level > max) {
|
|
3351
|
+
console.error(
|
|
3352
|
+
`Invalid compression for ${source}: "${raw}". Expected "none", "basic", or a ${fmt} level 1-${max}.`,
|
|
3353
|
+
);
|
|
3354
|
+
process.exit(1);
|
|
3355
|
+
}
|
|
3356
|
+
return level;
|
|
3357
|
+
};
|
|
3358
|
+
|
|
3359
|
+
// Compression tier for archives that don't specify their own: the
|
|
3360
|
+
// --compression / legacy --level flags, else "basic" (Bun's built-in
|
|
3361
|
+
// CompressionStream — fast, streaming, no level control).
|
|
3362
|
+
const defaultCompressionFor = (fmt: ArchiveFormat, source: string): ArchiveCompression =>
|
|
3363
|
+
globalCompressionFlag
|
|
3364
|
+
? parseCompression(globalCompressionFlag, fmt, source)
|
|
3365
|
+
: globalLevelFlag
|
|
3366
|
+
? parseCompression(globalLevelFlag, fmt, source)
|
|
3367
|
+
: "basic";
|
|
3320
3368
|
|
|
3321
3369
|
type ArchiveSpec = {
|
|
3322
3370
|
name: string;
|
|
3323
3371
|
path: string;
|
|
3324
|
-
format:
|
|
3325
|
-
|
|
3372
|
+
format: ArchiveFormat;
|
|
3373
|
+
compression: ArchiveCompression;
|
|
3326
3374
|
};
|
|
3327
3375
|
const archives: ArchiveSpec[] = [];
|
|
3328
3376
|
|
|
3329
3377
|
const addArchive = (
|
|
3330
3378
|
name: string,
|
|
3331
3379
|
rawPath: string,
|
|
3332
|
-
fmt:
|
|
3333
|
-
|
|
3380
|
+
fmt: ArchiveFormat,
|
|
3381
|
+
compression: ArchiveCompression,
|
|
3334
3382
|
) => {
|
|
3335
3383
|
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
|
|
3336
3384
|
console.error(`Invalid archive name: "${name}". Must be a valid identifier ([a-zA-Z_][a-zA-Z0-9_]*).`);
|
|
@@ -3345,22 +3393,35 @@ usageDescriptions : ""}${urlTypes ? "\n" + urlTypes : ""}${documentTypes ?
|
|
|
3345
3393
|
console.error(`Archive path does not exist: ${resolvedPath}`);
|
|
3346
3394
|
process.exit(1);
|
|
3347
3395
|
}
|
|
3348
|
-
archives.push({ name, path: resolvedPath, format: fmt,
|
|
3396
|
+
archives.push({ name, path: resolvedPath, format: fmt, compression });
|
|
3349
3397
|
};
|
|
3350
3398
|
|
|
3351
|
-
// Archives declared in sparkbun.config.ts (build.installer.archives)
|
|
3399
|
+
// Archives declared in sparkbun.config.ts (build.installer.archives).
|
|
3400
|
+
// `compression` ("none" | "basic" | level) is the preferred knob; the
|
|
3401
|
+
// legacy `level` field still works as a numeric tier.
|
|
3352
3402
|
const configArchives = (config.build as any)?.installer?.archives as
|
|
3353
|
-
| Array<{
|
|
3403
|
+
| Array<{
|
|
3404
|
+
name: string;
|
|
3405
|
+
path: string;
|
|
3406
|
+
format?: ArchiveFormat;
|
|
3407
|
+
compression?: "none" | "basic" | number;
|
|
3408
|
+
level?: number;
|
|
3409
|
+
}>
|
|
3354
3410
|
| undefined;
|
|
3355
3411
|
if (configArchives) {
|
|
3356
3412
|
for (const a of configArchives) {
|
|
3357
3413
|
const fmt = a.format || globalFormat;
|
|
3358
|
-
const
|
|
3359
|
-
|
|
3414
|
+
const compression =
|
|
3415
|
+
a.compression !== undefined
|
|
3416
|
+
? parseCompression(a.compression, fmt, `archive "${a.name}"`)
|
|
3417
|
+
: a.level !== undefined
|
|
3418
|
+
? parseCompression(a.level, fmt, `archive "${a.name}"`)
|
|
3419
|
+
: defaultCompressionFor(fmt, `archive "${a.name}"`);
|
|
3420
|
+
addArchive(a.name, a.path, fmt, compression);
|
|
3360
3421
|
}
|
|
3361
3422
|
}
|
|
3362
3423
|
|
|
3363
|
-
// Archives passed via --archive name=path (
|
|
3424
|
+
// Archives passed via --archive name=path (format/compression from global flags)
|
|
3364
3425
|
for (let i = 0; i < process.argv.length; i++) {
|
|
3365
3426
|
if (process.argv[i] === "--archive" && process.argv[i + 1]) {
|
|
3366
3427
|
const arg = process.argv[i + 1]!;
|
|
@@ -3369,8 +3430,13 @@ usageDescriptions : ""}${urlTypes ? "\n" + urlTypes : ""}${documentTypes ?
|
|
|
3369
3430
|
console.error(`Invalid --archive format: "${arg}". Expected: name=path`);
|
|
3370
3431
|
process.exit(1);
|
|
3371
3432
|
}
|
|
3372
|
-
const
|
|
3373
|
-
addArchive(
|
|
3433
|
+
const name = arg.substring(0, eqIdx);
|
|
3434
|
+
addArchive(
|
|
3435
|
+
name,
|
|
3436
|
+
arg.substring(eqIdx + 1),
|
|
3437
|
+
globalFormat,
|
|
3438
|
+
defaultCompressionFor(globalFormat, `--archive ${name}`),
|
|
3439
|
+
);
|
|
3374
3440
|
}
|
|
3375
3441
|
}
|
|
3376
3442
|
|
|
@@ -3438,7 +3504,13 @@ usageDescriptions : ""}${urlTypes ? "\n" + urlTypes : ""}${documentTypes ?
|
|
|
3438
3504
|
|
|
3439
3505
|
type ArchiveInfo = {
|
|
3440
3506
|
file: string;
|
|
3441
|
-
|
|
3507
|
+
// "none" = plain .tar, no decompression step at extract time.
|
|
3508
|
+
format: "zstd" | "gzip" | "none";
|
|
3509
|
+
// The tier as configured ("prebuilt" = the config pointed at an
|
|
3510
|
+
// already-built archive file, embedded verbatim); `level` keeps the
|
|
3511
|
+
// effective numeric level for older consumers (0 = uncompressed or
|
|
3512
|
+
// unknown; "basic" ≈ zstd 3 / gzip 6).
|
|
3513
|
+
compression: "none" | "basic" | number | "prebuilt";
|
|
3442
3514
|
level: number;
|
|
3443
3515
|
uncompressedBytes: number;
|
|
3444
3516
|
fileCount: number;
|
|
@@ -3447,51 +3519,74 @@ usageDescriptions : ""}${urlTypes ? "\n" + urlTypes : ""}${documentTypes ?
|
|
|
3447
3519
|
const archiveManifest: Record<string, ArchiveInfo> = {};
|
|
3448
3520
|
|
|
3449
3521
|
if (archives.length > 0) {
|
|
3450
|
-
console.log("Creating archives (tar +
|
|
3522
|
+
console.log("Creating archives (streaming tar + compression)...");
|
|
3451
3523
|
|
|
3452
3524
|
for (const spec of archives) {
|
|
3453
|
-
|
|
3525
|
+
// A FILE path is a prebuilt archive: embed it verbatim instead of
|
|
3526
|
+
// tarring + compressing a directory. Lets projects with large,
|
|
3527
|
+
// slow-to-compress payloads (e.g. zstd-22 game data) build them
|
|
3528
|
+
// once and skip the per-build recompression entirely. Format comes
|
|
3529
|
+
// from the extension; the manifest's progress denominators come
|
|
3530
|
+
// from a streaming scan of the archive.
|
|
3531
|
+
if (statSync(spec.path).isFile()) {
|
|
3532
|
+
const m = spec.path.match(/\.tar(\.zst|\.gz)?$/i);
|
|
3533
|
+
if (!m) {
|
|
3534
|
+
console.error(
|
|
3535
|
+
`Archive "${spec.name}": file paths must point at a prebuilt .tar/.tar.gz/.tar.zst (got ${spec.path})`,
|
|
3536
|
+
);
|
|
3537
|
+
process.exit(1);
|
|
3538
|
+
}
|
|
3539
|
+
const fmt: "zstd" | "gzip" | "none" =
|
|
3540
|
+
m[1]?.toLowerCase() === ".zst" ? "zstd" : m[1]?.toLowerCase() === ".gz" ? "gzip" : "none";
|
|
3541
|
+
const preExt = fmt === "none" ? "tar" : fmt === "gzip" ? "tar.gz" : "tar.zst";
|
|
3542
|
+
const preFileName = `archive-${spec.name}.${preExt}`;
|
|
3543
|
+
console.log(` Embedding prebuilt ${spec.name}: ${spec.path}`);
|
|
3544
|
+
cpSync(spec.path, join(stagingDir, preFileName));
|
|
3545
|
+
const { uncompressedBytes, fileCount } = await scanTarArchive(spec.path, fmt);
|
|
3546
|
+
const compressedBytes = statSync(spec.path).size;
|
|
3547
|
+
console.log(
|
|
3548
|
+
` ${spec.name}: ${(uncompressedBytes / 1024 / 1024).toFixed(0)} MB -> ` +
|
|
3549
|
+
`${(compressedBytes / 1024 / 1024).toFixed(0)} MB (prebuilt ${preExt}, ${fileCount} files)`,
|
|
3550
|
+
);
|
|
3551
|
+
archiveManifest[spec.name] = {
|
|
3552
|
+
file: preFileName,
|
|
3553
|
+
format: fmt,
|
|
3554
|
+
compression: "prebuilt",
|
|
3555
|
+
level: 0,
|
|
3556
|
+
uncompressedBytes,
|
|
3557
|
+
fileCount,
|
|
3558
|
+
compressedBytes,
|
|
3559
|
+
};
|
|
3560
|
+
continue;
|
|
3561
|
+
}
|
|
3562
|
+
|
|
3563
|
+
const ext = archiveExtension(spec.format, spec.compression);
|
|
3454
3564
|
const archiveFileName = `archive-${spec.name}.${ext}`;
|
|
3455
3565
|
const archiveDestPath = join(stagingDir, archiveFileName);
|
|
3456
|
-
const tarTmpPath = join(stagingDir, `archive-${spec.name}.tar`);
|
|
3457
3566
|
|
|
3458
3567
|
console.log(` Archiving ${spec.name}: ${spec.path}`);
|
|
3459
3568
|
|
|
3460
|
-
//
|
|
3461
|
-
//
|
|
3462
|
-
//
|
|
3463
|
-
//
|
|
3464
|
-
//
|
|
3465
|
-
|
|
3466
|
-
// The on-disk format may be GNU (Windows/Linux — long paths via
|
|
3467
|
-
// "././@LongLink") or pax (macOS bsdtar — long paths via extended
|
|
3468
|
-
// headers). The installer's streaming extractor understands both, plus
|
|
3469
|
-
// plain ustar, so the archive is portable regardless of build host.
|
|
3470
|
-
//
|
|
3471
|
-
// (Bun.Archive is intentionally not used here: its lazy Bun.file()
|
|
3472
|
-
// inputs write empty file bodies, and eager byte inputs would require
|
|
3473
|
-
// loading the entire payload into memory.)
|
|
3474
|
-
const parent = dirname(spec.path);
|
|
3475
|
-
const baseName = basename(spec.path);
|
|
3476
|
-
createTar(tarTmpPath, parent, [baseName]);
|
|
3569
|
+
// Hand-rolled streaming tar piped straight into the compressor (see
|
|
3570
|
+
// archive-stream.ts): plain ustar with the source folder name kept
|
|
3571
|
+
// as the archive root (e.g. "Dune 2000/..."), file modes preserved,
|
|
3572
|
+
// bounded build memory, and no system tar dependency — the on-disk
|
|
3573
|
+
// format is identical regardless of build host.
|
|
3574
|
+
await writeTarArchive(spec.path, archiveDestPath, spec.format, spec.compression);
|
|
3477
3575
|
|
|
3478
3576
|
// Measure uncompressed bytes + file count for progress denominators.
|
|
3479
3577
|
const { bytes: uncompressedBytes, fileCount } = measureDir(spec.path);
|
|
3480
3578
|
|
|
3481
|
-
// Stream-compress the tar at the requested level (bounded build memory).
|
|
3482
|
-
await streamCompressFile(tarTmpPath, archiveDestPath, spec.format, spec.level);
|
|
3483
|
-
unlinkSync(tarTmpPath);
|
|
3484
|
-
|
|
3485
3579
|
const compressedBytes = statSync(archiveDestPath).size;
|
|
3486
3580
|
console.log(
|
|
3487
3581
|
` ${spec.name}: ${(uncompressedBytes / 1024 / 1024).toFixed(0)} MB -> ` +
|
|
3488
3582
|
`${(compressedBytes / 1024 / 1024).toFixed(0)} MB ` +
|
|
3489
|
-
`(${spec.format
|
|
3583
|
+
`(${compressionLabel(spec.format, spec.compression)}, ${fileCount} files)`,
|
|
3490
3584
|
);
|
|
3491
3585
|
archiveManifest[spec.name] = {
|
|
3492
3586
|
file: archiveFileName,
|
|
3493
|
-
format: spec.format,
|
|
3494
|
-
|
|
3587
|
+
format: spec.compression === "none" ? "none" : spec.format,
|
|
3588
|
+
compression: spec.compression,
|
|
3589
|
+
level: effectiveLevel(spec.format, spec.compression),
|
|
3495
3590
|
uncompressedBytes,
|
|
3496
3591
|
fileCount,
|
|
3497
3592
|
compressedBytes,
|
|
@@ -3688,6 +3783,8 @@ ${archiveExports.join("\n")}
|
|
|
3688
3783
|
|
|
3689
3784
|
console.log("Compiling installer executable...");
|
|
3690
3785
|
|
|
3786
|
+
// (Compiled via buildCompiledExeWithRetry below — PE metadata commit
|
|
3787
|
+
// failures from AV scans are transient and retried.)
|
|
3691
3788
|
const installerCompileOptions: any = {
|
|
3692
3789
|
target: `bun-${targetOSName}-${targetARCH}`,
|
|
3693
3790
|
outfile: outputPath,
|
|
@@ -3727,10 +3824,13 @@ ${archiveExports.join("\n")}
|
|
|
3727
3824
|
};
|
|
3728
3825
|
}
|
|
3729
3826
|
|
|
3730
|
-
const installerBuild = await
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3827
|
+
const installerBuild = await buildCompiledExeWithRetry(
|
|
3828
|
+
{
|
|
3829
|
+
entrypoints: [join(stagingDir, "installer.ts")],
|
|
3830
|
+
compile: installerCompileOptions,
|
|
3831
|
+
},
|
|
3832
|
+
outputPath,
|
|
3833
|
+
);
|
|
3734
3834
|
if (!installerBuild.success) {
|
|
3735
3835
|
console.error("Installer compilation failed:", installerBuild.logs);
|
|
3736
3836
|
throw new Error("Installer compilation failed");
|
|
@@ -4437,10 +4537,13 @@ ${archiveExports.join("\n")}
|
|
|
4437
4537
|
}
|
|
4438
4538
|
|
|
4439
4539
|
console.log("Compiling installer...");
|
|
4440
|
-
const installerBuild = await
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4540
|
+
const installerBuild = await buildCompiledExeWithRetry(
|
|
4541
|
+
{
|
|
4542
|
+
entrypoints: [join(stagingDir, "installer.ts")],
|
|
4543
|
+
compile: installerCompileOptions,
|
|
4544
|
+
},
|
|
4545
|
+
outputPath,
|
|
4546
|
+
);
|
|
4444
4547
|
if (!installerBuild.success) {
|
|
4445
4548
|
console.error("Installer compilation failed:", installerBuild.logs);
|
|
4446
4549
|
throw new Error("Installer compilation failed");
|