steam-manifest-tools 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib.js +119 -44
- package/package.json +1 -1
package/lib.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
const crypto = require("crypto");
|
|
2
|
-
const fs = require("fs");
|
|
3
|
-
const fsPromises = require("fs/promises");
|
|
4
|
-
const path = require("path");
|
|
1
|
+
const crypto = require("node:crypto");
|
|
2
|
+
const fs = require("node:fs");
|
|
3
|
+
const fsPromises = require("node:fs/promises");
|
|
4
|
+
const path = require("node:path");
|
|
5
|
+
const worker_threads = require("node:worker_threads");
|
|
5
6
|
const AdmZip = require("adm-zip");
|
|
6
7
|
const CdnCompression = require("steam-user/components/cdn_compression");
|
|
7
8
|
const ContentManifest = require("steam-user/components/content_manifest");
|
|
@@ -30,6 +31,85 @@ const DEFAULT_HOSTS = [
|
|
|
30
31
|
"https://google2.cdn.steampipe.steamcontent.com",
|
|
31
32
|
];
|
|
32
33
|
|
|
34
|
+
const workers = [];
|
|
35
|
+
|
|
36
|
+
const getAvailableWorker = () => {
|
|
37
|
+
const w = workers.find(x => !x.busy);
|
|
38
|
+
if (w) {
|
|
39
|
+
//console.log(`Found an idle worker`);
|
|
40
|
+
w.busy = true;
|
|
41
|
+
return Promise.resolve(w);
|
|
42
|
+
}
|
|
43
|
+
if (workers.length < 8) {
|
|
44
|
+
//console.log(`Creating a new worker`);
|
|
45
|
+
const w = {
|
|
46
|
+
inst: new worker_threads.Worker(__filename),
|
|
47
|
+
busy: true,
|
|
48
|
+
}
|
|
49
|
+
w.inst.on("message", (x) => {
|
|
50
|
+
w.busy = false;
|
|
51
|
+
w.resolve(x);
|
|
52
|
+
});
|
|
53
|
+
w.inst.on("error", (x) => {
|
|
54
|
+
w.busy = false;
|
|
55
|
+
w.reject(x);
|
|
56
|
+
});
|
|
57
|
+
w.inst.on("exit", (code) => {
|
|
58
|
+
console.log(`Worker exited with code ${code}`);
|
|
59
|
+
workers.splice(workers.indexOf(w), 1);
|
|
60
|
+
});
|
|
61
|
+
workers.push(w);
|
|
62
|
+
return Promise.resolve(w);
|
|
63
|
+
}
|
|
64
|
+
//console.log("All workers busy, waiting");
|
|
65
|
+
return new Promise(resolve => {
|
|
66
|
+
let i;
|
|
67
|
+
i = setInterval(() => {
|
|
68
|
+
const w = workers.find(x => !x.busy);
|
|
69
|
+
if (w) {
|
|
70
|
+
w.busy = true;
|
|
71
|
+
clearInterval(i);
|
|
72
|
+
resolve(w);
|
|
73
|
+
}
|
|
74
|
+
}, 4);
|
|
75
|
+
});
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const doJobInWorker = (job) => {
|
|
79
|
+
return new Promise((resolve, reject) => {
|
|
80
|
+
getAvailableWorker().then((w) => {
|
|
81
|
+
w.resolve = resolve;
|
|
82
|
+
w.reject = reject;
|
|
83
|
+
w.inst.postMessage(job);
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
if (!worker_threads.isMainThread) {
|
|
89
|
+
worker_threads.parentPort.on("message", (job) => {
|
|
90
|
+
if ("getChunk" in job) {
|
|
91
|
+
(async () => {
|
|
92
|
+
const [ depotId, depotKey, hash ] = job.getChunk;
|
|
93
|
+
try {
|
|
94
|
+
let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
|
|
95
|
+
data = SteamCrypto.symmetricDecrypt(data, depotKey);
|
|
96
|
+
data = await CdnCompression.unzip(data);
|
|
97
|
+
if (sha1(data) != hash) {
|
|
98
|
+
throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
|
|
99
|
+
}
|
|
100
|
+
worker_threads.parentPort.postMessage(data);
|
|
101
|
+
}
|
|
102
|
+
catch (e) {
|
|
103
|
+
if (e.code == "ENOENT") {
|
|
104
|
+
throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
|
|
105
|
+
}
|
|
106
|
+
throw e;
|
|
107
|
+
}
|
|
108
|
+
})();
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
33
113
|
const sha1 = (data) => crypto.createHash("sha1").update(data).digest("hex");
|
|
34
114
|
|
|
35
115
|
const sha1file = async (file) => {
|
|
@@ -74,22 +154,10 @@ const getFileContents = async (file) => {
|
|
|
74
154
|
}
|
|
75
155
|
};
|
|
76
156
|
|
|
77
|
-
const getChunk =
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
data = await CdnCompression.unzip(data);
|
|
82
|
-
if (sha1(data) != hash) {
|
|
83
|
-
throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
|
|
84
|
-
}
|
|
85
|
-
return data;
|
|
86
|
-
}
|
|
87
|
-
catch (e) {
|
|
88
|
-
if (e.code == "ENOENT") {
|
|
89
|
-
throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
|
|
90
|
-
}
|
|
91
|
-
throw e;
|
|
92
|
-
}
|
|
157
|
+
const getChunk = (depotId, depotKey, hash) => {
|
|
158
|
+
return doJobInWorker({
|
|
159
|
+
getChunk: [ depotId, depotKey, hash ]
|
|
160
|
+
});
|
|
93
161
|
};
|
|
94
162
|
|
|
95
163
|
const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
|
|
@@ -252,34 +320,37 @@ module.exports = {
|
|
|
252
320
|
ContentManifest.decryptFilenames(manifest, depotKey);
|
|
253
321
|
installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
|
|
254
322
|
await fsPromises.mkdir(installDir, { recursive: true });
|
|
255
|
-
const promises = [];
|
|
256
323
|
for (const file of manifest.files) {
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
324
|
+
const filename = file.filename.replace(/\\/g, "/");
|
|
325
|
+
if (file.flags & 64) {
|
|
326
|
+
await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
const exists = fs.existsSync(path.join(installDir, filename));
|
|
330
|
+
if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
|
|
331
|
+
for (const chunk of file.chunks) {
|
|
332
|
+
chunk.offset = parseInt(chunk.offset);
|
|
262
333
|
}
|
|
263
|
-
|
|
264
|
-
if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
|
|
265
|
-
for (const chunk of file.chunks) {
|
|
266
|
-
chunk.offset = parseInt(chunk.offset);
|
|
267
|
-
}
|
|
268
|
-
file.chunks.sort((a, b) => a.offset - b.offset);
|
|
334
|
+
file.chunks.sort((a, b) => a.offset - b.offset);
|
|
269
335
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
336
|
+
await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
|
|
337
|
+
const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
|
|
338
|
+
for (let i = 0; i != file.chunks.length; ) {
|
|
339
|
+
const j = Math.min(i + 100, file.chunks.length);
|
|
340
|
+
const promises = [];
|
|
341
|
+
for (; i != j; ++i) {
|
|
342
|
+
promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
|
|
274
343
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
onFileWritten(filename, exists);
|
|
344
|
+
for (const p of promises) {
|
|
345
|
+
await writeStream.write(await p);
|
|
278
346
|
}
|
|
279
347
|
}
|
|
280
|
-
|
|
348
|
+
await writeStream.close();
|
|
349
|
+
if (onFileWritten) {
|
|
350
|
+
onFileWritten(filename, exists);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
281
353
|
}
|
|
282
|
-
await Promise.all(promises);
|
|
283
354
|
},
|
|
284
355
|
downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
|
|
285
356
|
const depotId = manifest.depot_id;
|
|
@@ -305,11 +376,15 @@ module.exports = {
|
|
|
305
376
|
const ioLoop = async () => {
|
|
306
377
|
ioLoopRunning = true;
|
|
307
378
|
while (ioQueue.length > 0) {
|
|
308
|
-
const
|
|
309
|
-
|
|
379
|
+
const completedChunks = {};
|
|
380
|
+
for (let i = 0; ioQueue.length > 0 && i != 100; ++i) {
|
|
381
|
+
const chunkSha = ioQueue.shift();
|
|
382
|
+
completedChunks[chunkSha] = getChunk(depotId, depotKey, chunkSha);
|
|
383
|
+
}
|
|
310
384
|
for (const file of manifest.files) {
|
|
311
385
|
for (const chunk of file.chunks) {
|
|
312
|
-
if (chunk.sha
|
|
386
|
+
if (chunk.sha in completedChunks) {
|
|
387
|
+
const data = await completedChunks[chunk.sha];
|
|
313
388
|
await writeStreams[file.filename].write(data, 0, data.byteLength, parseInt(chunk.offset));
|
|
314
389
|
}
|
|
315
390
|
}
|