steam-manifest-tools 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib.js +70 -55
- package/mango.js +33 -25
- package/package.json +2 -2
package/lib.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
const crypto = require("node:crypto");
|
|
2
2
|
const fs = require("node:fs");
|
|
3
3
|
const fsPromises = require("node:fs/promises");
|
|
4
|
+
const os = require("node:os");
|
|
4
5
|
const path = require("node:path");
|
|
5
6
|
const worker_threads = require("node:worker_threads");
|
|
6
7
|
const AdmZip = require("adm-zip");
|
|
7
|
-
const
|
|
8
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
8
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
9
9
|
const SteamCrypto = require("@doctormckay/steam-crypto");
|
|
10
10
|
|
|
11
11
|
const DEFAULT_HOSTS = [
|
|
@@ -31,6 +31,8 @@ const DEFAULT_HOSTS = [
|
|
|
31
31
|
"https://google2.cdn.steampipe.steamcontent.com",
|
|
32
32
|
];
|
|
33
33
|
|
|
34
|
+
const MAX_WORKERS = Math.ceil(os.cpus().length / 2);
|
|
35
|
+
|
|
34
36
|
const workers = [];
|
|
35
37
|
|
|
36
38
|
const getAvailableWorker = () => {
|
|
@@ -40,7 +42,7 @@ const getAvailableWorker = () => {
|
|
|
40
42
|
w.busy = true;
|
|
41
43
|
return Promise.resolve(w);
|
|
42
44
|
}
|
|
43
|
-
if (workers.length <
|
|
45
|
+
if (workers.length < MAX_WORKERS) {
|
|
44
46
|
//console.log(`Creating a new worker`);
|
|
45
47
|
const w = {
|
|
46
48
|
inst: new worker_threads.Worker(__filename),
|
|
@@ -71,7 +73,7 @@ const getAvailableWorker = () => {
|
|
|
71
73
|
clearInterval(i);
|
|
72
74
|
resolve(w);
|
|
73
75
|
}
|
|
74
|
-
},
|
|
76
|
+
}, 15);
|
|
75
77
|
});
|
|
76
78
|
};
|
|
77
79
|
|
|
@@ -86,16 +88,18 @@ const doJobInWorker = (job) => {
|
|
|
86
88
|
};
|
|
87
89
|
|
|
88
90
|
if (!worker_threads.isMainThread) {
|
|
91
|
+
const CdnCompression = require("lean-and-mean-steam-user/components/cdn_compression");
|
|
92
|
+
|
|
89
93
|
worker_threads.parentPort.on("message", (job) => {
|
|
90
94
|
if ("getChunk" in job) {
|
|
91
95
|
(async () => {
|
|
92
96
|
const [ depotId, depotKey, hash ] = job.getChunk;
|
|
93
97
|
try {
|
|
94
|
-
let data =
|
|
98
|
+
let data = fs.readFileSync(`depot/${depotId}/chunk/${hash}`);
|
|
95
99
|
data = SteamCrypto.symmetricDecrypt(data, depotKey);
|
|
96
100
|
data = await CdnCompression.unzip(data);
|
|
97
101
|
if (sha1(data) != hash) {
|
|
98
|
-
throw new Error(`
|
|
102
|
+
throw new Error(`Hash mismatch`);
|
|
99
103
|
}
|
|
100
104
|
worker_threads.parentPort.postMessage(data);
|
|
101
105
|
}
|
|
@@ -103,7 +107,7 @@ if (!worker_threads.isMainThread) {
|
|
|
103
107
|
if (e.code == "ENOENT") {
|
|
104
108
|
throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
|
|
105
109
|
}
|
|
106
|
-
throw e;
|
|
110
|
+
throw new Error(`depot/${depotId}/chunk/${hash} seems to have a problem: ${e.message}`);
|
|
107
111
|
}
|
|
108
112
|
})();
|
|
109
113
|
}
|
|
@@ -129,7 +133,7 @@ const compress = (buf) => {
|
|
|
129
133
|
return zip.toBuffer();
|
|
130
134
|
};
|
|
131
135
|
|
|
132
|
-
//
|
|
136
|
+
// deprecated
|
|
133
137
|
const getFiles = async (dir) => {
|
|
134
138
|
const dirents = await fsPromises.readdir(dir, { withFileTypes: true });
|
|
135
139
|
const files = await Promise.all(dirents.map((dirent) => {
|
|
@@ -139,6 +143,7 @@ const getFiles = async (dir) => {
|
|
|
139
143
|
return Array.prototype.concat(...files);
|
|
140
144
|
};
|
|
141
145
|
|
|
146
|
+
// deprecated
|
|
142
147
|
const getFileContents = async (file) => {
|
|
143
148
|
while (true) {
|
|
144
149
|
try {
|
|
@@ -227,6 +232,43 @@ const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishD
|
|
|
227
232
|
});
|
|
228
233
|
};
|
|
229
234
|
|
|
235
|
+
const install = async (manifest, depotKey, installDir, onFileWritten) => {
|
|
236
|
+
ContentManifest.decryptFilenames(manifest, depotKey);
|
|
237
|
+
installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
|
|
238
|
+
await fsPromises.mkdir(installDir, { recursive: true });
|
|
239
|
+
for (const file of manifest.files) {
|
|
240
|
+
const filename = file.filename.replace(/\\/g, "/");
|
|
241
|
+
if (file.flags & 64) {
|
|
242
|
+
await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
const exists = fs.existsSync(path.join(installDir, filename));
|
|
246
|
+
if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
|
|
247
|
+
for (const chunk of file.chunks) {
|
|
248
|
+
chunk.offset = parseInt(chunk.offset);
|
|
249
|
+
}
|
|
250
|
+
file.chunks.sort((a, b) => a.offset - b.offset);
|
|
251
|
+
|
|
252
|
+
await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
|
|
253
|
+
const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
|
|
254
|
+
for (let i = 0; i != file.chunks.length; ) {
|
|
255
|
+
const j = Math.min(i + 100, file.chunks.length);
|
|
256
|
+
const promises = [];
|
|
257
|
+
for (; i != j; ++i) {
|
|
258
|
+
promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
|
|
259
|
+
}
|
|
260
|
+
for (const p of promises) {
|
|
261
|
+
await writeStream.write(await p);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
await writeStream.close();
|
|
265
|
+
if (onFileWritten) {
|
|
266
|
+
onFileWritten(filename, exists);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
|
|
230
272
|
module.exports = {
|
|
231
273
|
DEFAULT_HOSTS,
|
|
232
274
|
sha1,
|
|
@@ -316,42 +358,7 @@ module.exports = {
|
|
|
316
358
|
}
|
|
317
359
|
},
|
|
318
360
|
downloadChunks,
|
|
319
|
-
install
|
|
320
|
-
ContentManifest.decryptFilenames(manifest, depotKey);
|
|
321
|
-
installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
|
|
322
|
-
await fsPromises.mkdir(installDir, { recursive: true });
|
|
323
|
-
for (const file of manifest.files) {
|
|
324
|
-
const filename = file.filename.replace(/\\/g, "/");
|
|
325
|
-
if (file.flags & 64) {
|
|
326
|
-
await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
|
|
327
|
-
continue;
|
|
328
|
-
}
|
|
329
|
-
const exists = fs.existsSync(path.join(installDir, filename));
|
|
330
|
-
if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
|
|
331
|
-
for (const chunk of file.chunks) {
|
|
332
|
-
chunk.offset = parseInt(chunk.offset);
|
|
333
|
-
}
|
|
334
|
-
file.chunks.sort((a, b) => a.offset - b.offset);
|
|
335
|
-
|
|
336
|
-
await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
|
|
337
|
-
const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
|
|
338
|
-
for (let i = 0; i != file.chunks.length; ) {
|
|
339
|
-
const j = Math.min(i + 100, file.chunks.length);
|
|
340
|
-
const promises = [];
|
|
341
|
-
for (; i != j; ++i) {
|
|
342
|
-
promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
|
|
343
|
-
}
|
|
344
|
-
for (const p of promises) {
|
|
345
|
-
await writeStream.write(await p);
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
await writeStream.close();
|
|
349
|
-
if (onFileWritten) {
|
|
350
|
-
onFileWritten(filename, exists);
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
},
|
|
361
|
+
install,
|
|
355
362
|
downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
|
|
356
363
|
const depotId = manifest.depot_id;
|
|
357
364
|
|
|
@@ -399,11 +406,17 @@ module.exports = {
|
|
|
399
406
|
}
|
|
400
407
|
};
|
|
401
408
|
|
|
409
|
+
let just_do_an_install = false;
|
|
402
410
|
await downloadChunks(
|
|
403
411
|
manifest,
|
|
404
412
|
(num_chunks, alreadyDownloaded) => {
|
|
405
|
-
|
|
406
|
-
|
|
413
|
+
if (num_chunks == 0) {
|
|
414
|
+
just_do_an_install = true;
|
|
415
|
+
}
|
|
416
|
+
else {
|
|
417
|
+
for (const path of alreadyDownloaded) {
|
|
418
|
+
ioAddPath(path);
|
|
419
|
+
}
|
|
407
420
|
}
|
|
408
421
|
if (onStartDownloading) {
|
|
409
422
|
onStartDownloading(num_chunks);
|
|
@@ -429,31 +442,33 @@ module.exports = {
|
|
|
429
442
|
for (const writeStream of Object.values(writeStreams)) {
|
|
430
443
|
await writeStream.close();
|
|
431
444
|
}
|
|
445
|
+
|
|
446
|
+
if (just_do_an_install) {
|
|
447
|
+
await install(manifest, depotKey, installDir);
|
|
448
|
+
}
|
|
432
449
|
},
|
|
433
450
|
verifyChunks: async (depotId, depotKey, onDeletedFile) => {
|
|
434
|
-
const files = await
|
|
451
|
+
const files = await fsPromises.readdir(`depot/${depotId}/chunk`);
|
|
435
452
|
let promises = [];
|
|
436
453
|
for (const file of files) {
|
|
437
454
|
const hash = file.substr(file.length - 40);
|
|
438
|
-
if (promises.length >
|
|
455
|
+
if (promises.length > 100) {
|
|
439
456
|
await Promise.all(promises);
|
|
440
457
|
promises = [];
|
|
441
458
|
}
|
|
442
|
-
promises.push(
|
|
459
|
+
promises.push((async () => {
|
|
443
460
|
try {
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
if (sha1(data) == hash) {
|
|
447
|
-
return;
|
|
448
|
-
}
|
|
461
|
+
await getChunk(depotId, depotKey, hash);
|
|
462
|
+
return;
|
|
449
463
|
}
|
|
450
464
|
catch (e) {
|
|
465
|
+
//console.log(e);
|
|
451
466
|
}
|
|
452
467
|
await fsPromises.unlink(file);
|
|
453
468
|
if (onDeletedFile) {
|
|
454
469
|
onDeletedFile(file, hash);
|
|
455
470
|
}
|
|
456
|
-
}));
|
|
471
|
+
})());
|
|
457
472
|
}
|
|
458
473
|
await Promise.all(promises);
|
|
459
474
|
},
|
package/mango.js
CHANGED
|
@@ -2,8 +2,13 @@ const parseArguments = (base, bools = []) => {
|
|
|
2
2
|
const res = [];
|
|
3
3
|
let j = 0;
|
|
4
4
|
for (let i = base; i != process.argv.length; ++i) {
|
|
5
|
-
if (
|
|
6
|
-
|
|
5
|
+
if (process.argv[i].startsWith("--")) {
|
|
6
|
+
if (bools.indexOf(process.argv[i]) != -1) {
|
|
7
|
+
res[process.argv[i]] = true;
|
|
8
|
+
}
|
|
9
|
+
else {
|
|
10
|
+
console.log(`Ignoring unknown argument: ${process.argv[i]}`);
|
|
11
|
+
}
|
|
7
12
|
}
|
|
8
13
|
else {
|
|
9
14
|
res[j++] = process.argv[i];
|
|
@@ -20,8 +25,7 @@ switch (tool) {
|
|
|
20
25
|
} break;
|
|
21
26
|
|
|
22
27
|
case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
|
|
23
|
-
const depotId =
|
|
24
|
-
const manifestId = process.argv[4];
|
|
28
|
+
const [depotId, manifestId] = parseArguments(3);
|
|
25
29
|
if (!depotId || !manifestId) {
|
|
26
30
|
console.log("Syntax: mango download-manifest <depot id> <manifest id>");
|
|
27
31
|
process.exit(1);
|
|
@@ -43,7 +47,7 @@ switch (tool) {
|
|
|
43
47
|
process.exit(1);
|
|
44
48
|
}
|
|
45
49
|
|
|
46
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
50
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
47
51
|
const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
|
|
48
52
|
|
|
49
53
|
const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
|
|
@@ -63,15 +67,13 @@ switch (tool) {
|
|
|
63
67
|
} break;
|
|
64
68
|
|
|
65
69
|
case "populate-chunks": {
|
|
66
|
-
const depotId =
|
|
67
|
-
const manifestId = process.argv[4];
|
|
68
|
-
const installDir = process.argv[5];
|
|
70
|
+
const [depotId, manifestId, installDir] = parseArguments(3);
|
|
69
71
|
if (!depotId || !manifestId || !installDir) {
|
|
70
72
|
console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir>");
|
|
71
73
|
process.exit(1);
|
|
72
74
|
}
|
|
73
75
|
|
|
74
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
76
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
75
77
|
const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
|
|
76
78
|
|
|
77
79
|
(async () => {
|
|
@@ -177,15 +179,13 @@ switch (tool) {
|
|
|
177
179
|
} break;
|
|
178
180
|
|
|
179
181
|
case "install": {
|
|
180
|
-
const depotId =
|
|
181
|
-
const manifestId = process.argv[4];
|
|
182
|
-
const installDir = process.argv[5];
|
|
182
|
+
const [depotId, manifestId, installDir] = parseArguments(3);
|
|
183
183
|
if (!depotId || !manifestId) {
|
|
184
184
|
console.log("Syntax: mango install <depot id> <manifest id> [install dir]");
|
|
185
185
|
process.exit(1);
|
|
186
186
|
}
|
|
187
187
|
|
|
188
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
188
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
189
189
|
const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
|
|
190
190
|
|
|
191
191
|
(async () => {
|
|
@@ -208,7 +208,7 @@ switch (tool) {
|
|
|
208
208
|
process.exit(1);
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
211
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
212
212
|
const { DEFAULT_HOSTS, fetchManifest, fetchDepotKey, downloadAndInstall } = require("./lib.js");
|
|
213
213
|
|
|
214
214
|
const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
|
|
@@ -220,12 +220,23 @@ switch (tool) {
|
|
|
220
220
|
let depotKey = await fetchDepotKey(manifest.depot_id);
|
|
221
221
|
depotKey = Buffer.from(depotKey, "hex");
|
|
222
222
|
|
|
223
|
+
let remaining_chunks;
|
|
223
224
|
await downloadAndInstall(
|
|
224
225
|
manifest,
|
|
225
226
|
depotKey,
|
|
226
|
-
|
|
227
|
+
(num_chunks) => {
|
|
228
|
+
remaining_chunks = num_chunks;
|
|
229
|
+
if (remaining_chunks == 0) {
|
|
230
|
+
console.log("Done with downloading. Installing may still take a bit.");
|
|
231
|
+
}
|
|
232
|
+
},
|
|
227
233
|
(path, host) => { console.log(`${path}: Downloading from ${host}`); },
|
|
228
|
-
(path, status, host) => {
|
|
234
|
+
(path, status, host) => {
|
|
235
|
+
console.log(`${path}: Got ${status/*} from ${host*/}`);
|
|
236
|
+
if (status == 200 && --remaining_chunks == 0) {
|
|
237
|
+
console.log("Done with downloading. Installing may still take a bit.");
|
|
238
|
+
}
|
|
239
|
+
},
|
|
229
240
|
(path, err) => { console.log(`${path}: `, err); },
|
|
230
241
|
hosts,
|
|
231
242
|
installDir
|
|
@@ -244,7 +255,7 @@ switch (tool) {
|
|
|
244
255
|
}
|
|
245
256
|
|
|
246
257
|
const fs = require("fs");
|
|
247
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
258
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
248
259
|
const { fetchManifest, fetchDepotKey } = require("./lib.js");
|
|
249
260
|
|
|
250
261
|
(async () => {
|
|
@@ -263,8 +274,7 @@ switch (tool) {
|
|
|
263
274
|
} break;
|
|
264
275
|
|
|
265
276
|
case "to-hashdeep-auditfile": {
|
|
266
|
-
const depotId =
|
|
267
|
-
const manifestId = process.argv[4];
|
|
277
|
+
const [depotId, manifestId] = parseArguments(3);
|
|
268
278
|
if (!depotId || !manifestId) {
|
|
269
279
|
console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id>");
|
|
270
280
|
process.exit(1);
|
|
@@ -272,7 +282,7 @@ switch (tool) {
|
|
|
272
282
|
|
|
273
283
|
const fs = require("fs");
|
|
274
284
|
const path = require("path");
|
|
275
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
285
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
276
286
|
const { fetchManifest, fetchDepotKey } = require("./lib.js");
|
|
277
287
|
|
|
278
288
|
(async () => {
|
|
@@ -301,15 +311,13 @@ switch (tool) {
|
|
|
301
311
|
} break;
|
|
302
312
|
|
|
303
313
|
case "to-torrent": {
|
|
304
|
-
const depotId =
|
|
305
|
-
const manifestId = process.argv[4];
|
|
306
|
-
const fileHash = process.argv[5];
|
|
314
|
+
const [depotId, manifestId, fileHash] = parseArguments(3);
|
|
307
315
|
if (!depotId || !manifestId || !fileHash) {
|
|
308
316
|
console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
|
|
309
317
|
process.exit(1);
|
|
310
318
|
}
|
|
311
319
|
|
|
312
|
-
const ContentManifest = require("steam-user/components/content_manifest");
|
|
320
|
+
const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
|
|
313
321
|
|
|
314
322
|
function toPieces(hexArray) {
|
|
315
323
|
const bufs = hexArray.map(h => {
|
|
@@ -357,7 +365,7 @@ switch (tool) {
|
|
|
357
365
|
} break;
|
|
358
366
|
|
|
359
367
|
case "verify-chunks": {
|
|
360
|
-
const depotId =
|
|
368
|
+
const [depotId] = parseArguments(3);
|
|
361
369
|
if (!depotId) {
|
|
362
370
|
console.log("Syntax: mango verify-chunks <depot id>");
|
|
363
371
|
process.exit(1);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "steam-manifest-tools",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"main": "lib.js",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
"dependencies": {
|
|
13
13
|
"@doctormckay/steam-crypto": "^1.2.0",
|
|
14
14
|
"adm-zip": "^0.5.16",
|
|
15
|
-
"steam-user": "^
|
|
15
|
+
"lean-and-mean-steam-user": "^0.1.0"
|
|
16
16
|
},
|
|
17
17
|
"devDependencies": {
|
|
18
18
|
"express": "^5.1.0",
|