steam-manifest-tools 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/lib.js +160 -70
  2. package/mango.js +33 -25
  3. package/package.json +2 -2
package/lib.js CHANGED
@@ -1,10 +1,11 @@
1
- const crypto = require("crypto");
2
- const fs = require("fs");
3
- const fsPromises = require("fs/promises");
4
- const path = require("path");
1
+ const crypto = require("node:crypto");
2
+ const fs = require("node:fs");
3
+ const fsPromises = require("node:fs/promises");
4
+ const os = require("node:os");
5
+ const path = require("node:path");
6
+ const worker_threads = require("node:worker_threads");
5
7
  const AdmZip = require("adm-zip");
6
- const CdnCompression = require("steam-user/components/cdn_compression");
7
- const ContentManifest = require("steam-user/components/content_manifest");
8
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
8
9
  const SteamCrypto = require("@doctormckay/steam-crypto");
9
10
 
10
11
  const DEFAULT_HOSTS = [
@@ -30,6 +31,89 @@ const DEFAULT_HOSTS = [
30
31
  "https://google2.cdn.steampipe.steamcontent.com",
31
32
  ];
32
33
 
34
+ const MAX_WORKERS = Math.ceil(os.cpus().length / 2);
35
+
36
+ const workers = [];
37
+
38
+ const getAvailableWorker = () => {
39
+ const w = workers.find(x => !x.busy);
40
+ if (w) {
41
+ //console.log(`Found an idle worker`);
42
+ w.busy = true;
43
+ return Promise.resolve(w);
44
+ }
45
+ if (workers.length < MAX_WORKERS) {
46
+ //console.log(`Creating a new worker`);
47
+ const w = {
48
+ inst: new worker_threads.Worker(__filename),
49
+ busy: true,
50
+ }
51
+ w.inst.on("message", (x) => {
52
+ w.busy = false;
53
+ w.resolve(x);
54
+ });
55
+ w.inst.on("error", (x) => {
56
+ w.busy = false;
57
+ w.reject(x);
58
+ });
59
+ w.inst.on("exit", (code) => {
60
+ console.log(`Worker exited with code ${code}`);
61
+ workers.splice(workers.indexOf(w), 1);
62
+ });
63
+ workers.push(w);
64
+ return Promise.resolve(w);
65
+ }
66
+ //console.log("All workers busy, waiting");
67
+ return new Promise(resolve => {
68
+ let i;
69
+ i = setInterval(() => {
70
+ const w = workers.find(x => !x.busy);
71
+ if (w) {
72
+ w.busy = true;
73
+ clearInterval(i);
74
+ resolve(w);
75
+ }
76
+ }, 15);
77
+ });
78
+ };
79
+
80
+ const doJobInWorker = (job) => {
81
+ return new Promise((resolve, reject) => {
82
+ getAvailableWorker().then((w) => {
83
+ w.resolve = resolve;
84
+ w.reject = reject;
85
+ w.inst.postMessage(job);
86
+ });
87
+ });
88
+ };
89
+
90
+ if (!worker_threads.isMainThread) {
91
+ const CdnCompression = require("lean-and-mean-steam-user/components/cdn_compression");
92
+
93
+ worker_threads.parentPort.on("message", (job) => {
94
+ if ("getChunk" in job) {
95
+ (async () => {
96
+ const [ depotId, depotKey, hash ] = job.getChunk;
97
+ try {
98
+ let data = fs.readFileSync(`depot/${depotId}/chunk/${hash}`);
99
+ data = SteamCrypto.symmetricDecrypt(data, depotKey);
100
+ data = await CdnCompression.unzip(data);
101
+ if (sha1(data) != hash) {
102
+ throw new Error(`Hash mismatch`);
103
+ }
104
+ worker_threads.parentPort.postMessage(data);
105
+ }
106
+ catch (e) {
107
+ if (e.code == "ENOENT") {
108
+ throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
109
+ }
110
+ throw new Error(`depot/${depotId}/chunk/${hash} seems to have a problem: ${e.message}`);
111
+ }
112
+ })();
113
+ }
114
+ });
115
+ }
116
+
33
117
  const sha1 = (data) => crypto.createHash("sha1").update(data).digest("hex");
34
118
 
35
119
  const sha1file = async (file) => {
@@ -49,7 +133,7 @@ const compress = (buf) => {
49
133
  return zip.toBuffer();
50
134
  };
51
135
 
52
- // https://stackoverflow.com/a/45130990
136
+ // deprecated
53
137
  const getFiles = async (dir) => {
54
138
  const dirents = await fsPromises.readdir(dir, { withFileTypes: true });
55
139
  const files = await Promise.all(dirents.map((dirent) => {
@@ -59,6 +143,7 @@ const getFiles = async (dir) => {
59
143
  return Array.prototype.concat(...files);
60
144
  };
61
145
 
146
+ // deprecated
62
147
  const getFileContents = async (file) => {
63
148
  while (true) {
64
149
  try {
@@ -74,22 +159,10 @@ const getFileContents = async (file) => {
74
159
  }
75
160
  };
76
161
 
77
- const getChunk = async (depotId, depotKey, hash) => {
78
- try {
79
- let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
80
- data = SteamCrypto.symmetricDecrypt(data, depotKey);
81
- data = await CdnCompression.unzip(data);
82
- if (sha1(data) != hash) {
83
- throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
84
- }
85
- return data;
86
- }
87
- catch (e) {
88
- if (e.code == "ENOENT") {
89
- throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
90
- }
91
- throw e;
92
- }
162
+ const getChunk = (depotId, depotKey, hash) => {
163
+ return doJobInWorker({
164
+ getChunk: [ depotId, depotKey, hash ]
165
+ });
93
166
  };
94
167
 
95
168
  const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
@@ -159,6 +232,43 @@ const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishD
159
232
  });
160
233
  };
161
234
 
235
+ const install = async (manifest, depotKey, installDir, onFileWritten) => {
236
+ ContentManifest.decryptFilenames(manifest, depotKey);
237
+ installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
238
+ await fsPromises.mkdir(installDir, { recursive: true });
239
+ for (const file of manifest.files) {
240
+ const filename = file.filename.replace(/\\/g, "/");
241
+ if (file.flags & 64) {
242
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
243
+ continue;
244
+ }
245
+ const exists = fs.existsSync(path.join(installDir, filename));
246
+ if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
247
+ for (const chunk of file.chunks) {
248
+ chunk.offset = parseInt(chunk.offset);
249
+ }
250
+ file.chunks.sort((a, b) => a.offset - b.offset);
251
+
252
+ await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
253
+ const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
254
+ for (let i = 0; i != file.chunks.length; ) {
255
+ const j = Math.min(i + 100, file.chunks.length);
256
+ const promises = [];
257
+ for (; i != j; ++i) {
258
+ promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
259
+ }
260
+ for (const p of promises) {
261
+ await writeStream.write(await p);
262
+ }
263
+ }
264
+ await writeStream.close();
265
+ if (onFileWritten) {
266
+ onFileWritten(filename, exists);
267
+ }
268
+ }
269
+ }
270
+ };
271
+
162
272
  module.exports = {
163
273
  DEFAULT_HOSTS,
164
274
  sha1,
@@ -248,39 +358,7 @@ module.exports = {
248
358
  }
249
359
  },
250
360
  downloadChunks,
251
- install: async (manifest, depotKey, installDir, onFileWritten) => {
252
- ContentManifest.decryptFilenames(manifest, depotKey);
253
- installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
254
- await fsPromises.mkdir(installDir, { recursive: true });
255
- const promises = [];
256
- for (const file of manifest.files) {
257
- promises.push((async () => {
258
- const filename = file.filename.replace(/\\/g, "/");
259
- if (file.flags & 64) {
260
- await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
261
- return;
262
- }
263
- const exists = fs.existsSync(path.join(installDir, filename));
264
- if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
265
- for (const chunk of file.chunks) {
266
- chunk.offset = parseInt(chunk.offset);
267
- }
268
- file.chunks.sort((a, b) => a.offset - b.offset);
269
-
270
- await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
271
- const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
272
- for (const chunk of file.chunks) {
273
- await writeStream.write(await getChunk(manifest.depot_id, depotKey, chunk.sha));
274
- }
275
- await writeStream.close();
276
- if (onFileWritten) {
277
- onFileWritten(filename, exists);
278
- }
279
- }
280
- })());
281
- }
282
- await Promise.all(promises);
283
- },
361
+ install,
284
362
  downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
285
363
  const depotId = manifest.depot_id;
286
364
 
@@ -305,11 +383,15 @@ module.exports = {
305
383
  const ioLoop = async () => {
306
384
  ioLoopRunning = true;
307
385
  while (ioQueue.length > 0) {
308
- const chunkSha = ioQueue.shift();
309
- const data = await getChunk(depotId, depotKey, chunkSha);
386
+ const completedChunks = {};
387
+ for (let i = 0; ioQueue.length > 0 && i != 100; ++i) {
388
+ const chunkSha = ioQueue.shift();
389
+ completedChunks[chunkSha] = getChunk(depotId, depotKey, chunkSha);
390
+ }
310
391
  for (const file of manifest.files) {
311
392
  for (const chunk of file.chunks) {
312
- if (chunk.sha == chunkSha) {
393
+ if (chunk.sha in completedChunks) {
394
+ const data = await completedChunks[chunk.sha];
313
395
  await writeStreams[file.filename].write(data, 0, data.byteLength, parseInt(chunk.offset));
314
396
  }
315
397
  }
@@ -324,11 +406,17 @@ module.exports = {
324
406
  }
325
407
  };
326
408
 
409
+ let just_do_an_install = false;
327
410
  await downloadChunks(
328
411
  manifest,
329
412
  (num_chunks, alreadyDownloaded) => {
330
- for (const path of alreadyDownloaded) {
331
- ioAddPath(path);
413
+ if (num_chunks == 0) {
414
+ just_do_an_install = true;
415
+ }
416
+ else {
417
+ for (const path of alreadyDownloaded) {
418
+ ioAddPath(path);
419
+ }
332
420
  }
333
421
  if (onStartDownloading) {
334
422
  onStartDownloading(num_chunks);
@@ -354,31 +442,33 @@ module.exports = {
354
442
  for (const writeStream of Object.values(writeStreams)) {
355
443
  await writeStream.close();
356
444
  }
445
+
446
+ if (just_do_an_install) {
447
+ await install(manifest, depotKey, installDir);
448
+ }
357
449
  },
358
450
  verifyChunks: async (depotId, depotKey, onDeletedFile) => {
359
- const files = await getFiles(`depot/${depotId}/chunk`);
451
+ const files = await fsPromises.readdir(`depot/${depotId}/chunk`);
360
452
  let promises = [];
361
453
  for (const file of files) {
362
454
  const hash = file.substr(file.length - 40);
363
- if (promises.length > 40) {
455
+ if (promises.length > 100) {
364
456
  await Promise.all(promises);
365
457
  promises = [];
366
458
  }
367
- promises.push(getFileContents(file).then(async data => {
459
+ promises.push((async () => {
368
460
  try {
369
- data = SteamCrypto.symmetricDecrypt(data, depotKey);
370
- data = await CdnCompression.unzip(data);
371
- if (sha1(data) == hash) {
372
- return;
373
- }
461
+ await getChunk(depotId, depotKey, hash);
462
+ return;
374
463
  }
375
464
  catch (e) {
465
+ //console.log(e);
376
466
  }
377
467
  await fsPromises.unlink(file);
378
468
  if (onDeletedFile) {
379
469
  onDeletedFile(file, hash);
380
470
  }
381
- }));
471
+ })());
382
472
  }
383
473
  await Promise.all(promises);
384
474
  },
package/mango.js CHANGED
@@ -2,8 +2,13 @@ const parseArguments = (base, bools = []) => {
2
2
  const res = [];
3
3
  let j = 0;
4
4
  for (let i = base; i != process.argv.length; ++i) {
5
- if (bools.indexOf(process.argv[i]) != -1) {
6
- res[process.argv[i]] = true;
5
+ if (process.argv[i].startsWith("--")) {
6
+ if (bools.indexOf(process.argv[i]) != -1) {
7
+ res[process.argv[i]] = true;
8
+ }
9
+ else {
10
+ console.log(`Ignoring unknown argument: ${process.argv[i]}`);
11
+ }
7
12
  }
8
13
  else {
9
14
  res[j++] = process.argv[i];
@@ -20,8 +25,7 @@ switch (tool) {
20
25
  } break;
21
26
 
22
27
  case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
23
- const depotId = process.argv[3];
24
- const manifestId = process.argv[4];
28
+ const [depotId, manifestId] = parseArguments(3);
25
29
  if (!depotId || !manifestId) {
26
30
  console.log("Syntax: mango download-manifest <depot id> <manifest id>");
27
31
  process.exit(1);
@@ -43,7 +47,7 @@ switch (tool) {
43
47
  process.exit(1);
44
48
  }
45
49
 
46
- const ContentManifest = require("steam-user/components/content_manifest");
50
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
47
51
  const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
48
52
 
49
53
  const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
@@ -63,15 +67,13 @@ switch (tool) {
63
67
  } break;
64
68
 
65
69
  case "populate-chunks": {
66
- const depotId = process.argv[3];
67
- const manifestId = process.argv[4];
68
- const installDir = process.argv[5];
70
+ const [depotId, manifestId, installDir] = parseArguments(3);
69
71
  if (!depotId || !manifestId || !installDir) {
70
72
  console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir>");
71
73
  process.exit(1);
72
74
  }
73
75
 
74
- const ContentManifest = require("steam-user/components/content_manifest");
76
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
75
77
  const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
76
78
 
77
79
  (async () => {
@@ -177,15 +179,13 @@ switch (tool) {
177
179
  } break;
178
180
 
179
181
  case "install": {
180
- const depotId = process.argv[3];
181
- const manifestId = process.argv[4];
182
- const installDir = process.argv[5];
182
+ const [depotId, manifestId, installDir] = parseArguments(3);
183
183
  if (!depotId || !manifestId) {
184
184
  console.log("Syntax: mango install <depot id> <manifest id> [install dir]");
185
185
  process.exit(1);
186
186
  }
187
187
 
188
- const ContentManifest = require("steam-user/components/content_manifest");
188
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
189
189
  const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
190
190
 
191
191
  (async () => {
@@ -208,7 +208,7 @@ switch (tool) {
208
208
  process.exit(1);
209
209
  }
210
210
 
211
- const ContentManifest = require("steam-user/components/content_manifest");
211
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
212
212
  const { DEFAULT_HOSTS, fetchManifest, fetchDepotKey, downloadAndInstall } = require("./lib.js");
213
213
 
214
214
  const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
@@ -220,12 +220,23 @@ switch (tool) {
220
220
  let depotKey = await fetchDepotKey(manifest.depot_id);
221
221
  depotKey = Buffer.from(depotKey, "hex");
222
222
 
223
+ let remaining_chunks;
223
224
  await downloadAndInstall(
224
225
  manifest,
225
226
  depotKey,
226
- undefined /*(num_chunks) => {}*/,
227
+ (num_chunks) => {
228
+ remaining_chunks = num_chunks;
229
+ if (remaining_chunks == 0) {
230
+ console.log("Done with downloading. Installing may still take a bit.");
231
+ }
232
+ },
227
233
  (path, host) => { console.log(`${path}: Downloading from ${host}`); },
228
- (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
234
+ (path, status, host) => {
235
+ console.log(`${path}: Got ${status/*} from ${host*/}`);
236
+ if (status == 200 && --remaining_chunks == 0) {
237
+ console.log("Done with downloading. Installing may still take a bit.");
238
+ }
239
+ },
229
240
  (path, err) => { console.log(`${path}: `, err); },
230
241
  hosts,
231
242
  installDir
@@ -244,7 +255,7 @@ switch (tool) {
244
255
  }
245
256
 
246
257
  const fs = require("fs");
247
- const ContentManifest = require("steam-user/components/content_manifest");
258
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
248
259
  const { fetchManifest, fetchDepotKey } = require("./lib.js");
249
260
 
250
261
  (async () => {
@@ -263,8 +274,7 @@ switch (tool) {
263
274
  } break;
264
275
 
265
276
  case "to-hashdeep-auditfile": {
266
- const depotId = process.argv[3];
267
- const manifestId = process.argv[4];
277
+ const [depotId, manifestId] = parseArguments(3);
268
278
  if (!depotId || !manifestId) {
269
279
  console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id>");
270
280
  process.exit(1);
@@ -272,7 +282,7 @@ switch (tool) {
272
282
 
273
283
  const fs = require("fs");
274
284
  const path = require("path");
275
- const ContentManifest = require("steam-user/components/content_manifest");
285
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
276
286
  const { fetchManifest, fetchDepotKey } = require("./lib.js");
277
287
 
278
288
  (async () => {
@@ -301,15 +311,13 @@ switch (tool) {
301
311
  } break;
302
312
 
303
313
  case "to-torrent": {
304
- const depotId = process.argv[3];
305
- const manifestId = process.argv[4];
306
- const fileHash = process.argv[5];
314
+ const [depotId, manifestId, fileHash] = parseArguments(3);
307
315
  if (!depotId || !manifestId || !fileHash) {
308
316
  console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
309
317
  process.exit(1);
310
318
  }
311
319
 
312
- const ContentManifest = require("steam-user/components/content_manifest");
320
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
313
321
 
314
322
  function toPieces(hexArray) {
315
323
  const bufs = hexArray.map(h => {
@@ -357,7 +365,7 @@ switch (tool) {
357
365
  } break;
358
366
 
359
367
  case "verify-chunks": {
360
- const depotId = process.argv[3];
368
+ const [depotId] = parseArguments(3);
361
369
  if (!depotId) {
362
370
  console.log("Syntax: mango verify-chunks <depot id>");
363
371
  process.exit(1);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "steam-manifest-tools",
3
- "version": "0.1.6",
3
+ "version": "0.1.8",
4
4
  "main": "lib.js",
5
5
  "repository": {
6
6
  "type": "git",
@@ -12,7 +12,7 @@
12
12
  "dependencies": {
13
13
  "@doctormckay/steam-crypto": "^1.2.0",
14
14
  "adm-zip": "^0.5.16",
15
- "steam-user": "^5.2.3"
15
+ "lean-and-mean-steam-user": "^0.1.0"
16
16
  },
17
17
  "devDependencies": {
18
18
  "express": "^5.1.0",