steam-manifest-tools 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/lib.js +73 -58
  2. package/mango.js +41 -25
  3. package/package.json +2 -2
package/lib.js CHANGED
@@ -1,11 +1,11 @@
1
1
  const crypto = require("node:crypto");
2
2
  const fs = require("node:fs");
3
3
  const fsPromises = require("node:fs/promises");
4
+ const os = require("node:os");
4
5
  const path = require("node:path");
5
6
  const worker_threads = require("node:worker_threads");
6
7
  const AdmZip = require("adm-zip");
7
- const CdnCompression = require("steam-user/components/cdn_compression");
8
- const ContentManifest = require("steam-user/components/content_manifest");
8
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
9
9
  const SteamCrypto = require("@doctormckay/steam-crypto");
10
10
 
11
11
  const DEFAULT_HOSTS = [
@@ -31,6 +31,8 @@ const DEFAULT_HOSTS = [
31
31
  "https://google2.cdn.steampipe.steamcontent.com",
32
32
  ];
33
33
 
34
+ const MAX_WORKERS = Math.ceil(os.cpus().length / 2);
35
+
34
36
  const workers = [];
35
37
 
36
38
  const getAvailableWorker = () => {
@@ -40,7 +42,7 @@ const getAvailableWorker = () => {
40
42
  w.busy = true;
41
43
  return Promise.resolve(w);
42
44
  }
43
- if (workers.length < 8) {
45
+ if (workers.length < MAX_WORKERS) {
44
46
  //console.log(`Creating a new worker`);
45
47
  const w = {
46
48
  inst: new worker_threads.Worker(__filename),
@@ -55,7 +57,7 @@ const getAvailableWorker = () => {
55
57
  w.reject(x);
56
58
  });
57
59
  w.inst.on("exit", (code) => {
58
- console.log(`Worker exited with code ${code}`);
60
+ //console.log(`Worker exited with code ${code}`);
59
61
  workers.splice(workers.indexOf(w), 1);
60
62
  });
61
63
  workers.push(w);
@@ -71,7 +73,7 @@ const getAvailableWorker = () => {
71
73
  clearInterval(i);
72
74
  resolve(w);
73
75
  }
74
- }, 4);
76
+ }, 15);
75
77
  });
76
78
  };
77
79
 
@@ -86,16 +88,18 @@ const doJobInWorker = (job) => {
86
88
  };
87
89
 
88
90
  if (!worker_threads.isMainThread) {
91
+ const CdnCompression = require("lean-and-mean-steam-user/components/cdn_compression");
92
+
89
93
  worker_threads.parentPort.on("message", (job) => {
90
94
  if ("getChunk" in job) {
91
95
  (async () => {
92
96
  const [ depotId, depotKey, hash ] = job.getChunk;
93
97
  try {
94
- let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
98
+ let data = fs.readFileSync(`depot/${depotId}/chunk/${hash}`);
95
99
  data = SteamCrypto.symmetricDecrypt(data, depotKey);
96
100
  data = await CdnCompression.unzip(data);
97
101
  if (sha1(data) != hash) {
98
- throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
102
+ throw new Error(`Hash mismatch`);
99
103
  }
100
104
  worker_threads.parentPort.postMessage(data);
101
105
  }
@@ -103,7 +107,7 @@ if (!worker_threads.isMainThread) {
103
107
  if (e.code == "ENOENT") {
104
108
  throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
105
109
  }
106
- throw e;
110
+ throw new Error(`depot/${depotId}/chunk/${hash} seems to have a problem: ${e.message}`);
107
111
  }
108
112
  })();
109
113
  }
@@ -129,7 +133,7 @@ const compress = (buf) => {
129
133
  return zip.toBuffer();
130
134
  };
131
135
 
132
- // https://stackoverflow.com/a/45130990
136
+ // deprecated
133
137
  const getFiles = async (dir) => {
134
138
  const dirents = await fsPromises.readdir(dir, { withFileTypes: true });
135
139
  const files = await Promise.all(dirents.map((dirent) => {
@@ -139,6 +143,7 @@ const getFiles = async (dir) => {
139
143
  return Array.prototype.concat(...files);
140
144
  };
141
145
 
146
+ // deprecated
142
147
  const getFileContents = async (file) => {
143
148
  while (true) {
144
149
  try {
@@ -227,6 +232,43 @@ const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishD
227
232
  });
228
233
  };
229
234
 
235
+ const install = async (manifest, depotKey, installDir, onFileWritten) => {
236
+ ContentManifest.decryptFilenames(manifest, depotKey);
237
+ installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
238
+ await fsPromises.mkdir(installDir, { recursive: true });
239
+ for (const file of manifest.files) {
240
+ const filename = file.filename.replace(/\\/g, "/");
241
+ if (file.flags & 64) {
242
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
243
+ continue;
244
+ }
245
+ const exists = fs.existsSync(path.join(installDir, filename));
246
+ if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
247
+ for (const chunk of file.chunks) {
248
+ chunk.offset = parseInt(chunk.offset);
249
+ }
250
+ file.chunks.sort((a, b) => a.offset - b.offset);
251
+
252
+ await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
253
+ const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
254
+ for (let i = 0; i != file.chunks.length; ) {
255
+ const j = Math.min(i + 100, file.chunks.length);
256
+ const promises = [];
257
+ for (; i != j; ++i) {
258
+ promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
259
+ }
260
+ for (const p of promises) {
261
+ await writeStream.write(await p);
262
+ }
263
+ }
264
+ await writeStream.close();
265
+ if (onFileWritten) {
266
+ onFileWritten(filename, exists);
267
+ }
268
+ }
269
+ }
270
+ };
271
+
230
272
  module.exports = {
231
273
  DEFAULT_HOSTS,
232
274
  sha1,
@@ -316,42 +358,7 @@ module.exports = {
316
358
  }
317
359
  },
318
360
  downloadChunks,
319
- install: async (manifest, depotKey, installDir, onFileWritten) => {
320
- ContentManifest.decryptFilenames(manifest, depotKey);
321
- installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
322
- await fsPromises.mkdir(installDir, { recursive: true });
323
- for (const file of manifest.files) {
324
- const filename = file.filename.replace(/\\/g, "/");
325
- if (file.flags & 64) {
326
- await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
327
- continue;
328
- }
329
- const exists = fs.existsSync(path.join(installDir, filename));
330
- if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
331
- for (const chunk of file.chunks) {
332
- chunk.offset = parseInt(chunk.offset);
333
- }
334
- file.chunks.sort((a, b) => a.offset - b.offset);
335
-
336
- await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
337
- const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
338
- for (let i = 0; i != file.chunks.length; ) {
339
- const j = Math.min(i + 100, file.chunks.length);
340
- const promises = [];
341
- for (; i != j; ++i) {
342
- promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
343
- }
344
- for (const p of promises) {
345
- await writeStream.write(await p);
346
- }
347
- }
348
- await writeStream.close();
349
- if (onFileWritten) {
350
- onFileWritten(filename, exists);
351
- }
352
- }
353
- }
354
- },
361
+ install,
355
362
  downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
356
363
  const depotId = manifest.depot_id;
357
364
 
@@ -399,11 +406,17 @@ module.exports = {
399
406
  }
400
407
  };
401
408
 
409
+ let just_do_an_install = false;
402
410
  await downloadChunks(
403
411
  manifest,
404
412
  (num_chunks, alreadyDownloaded) => {
405
- for (const path of alreadyDownloaded) {
406
- ioAddPath(path);
413
+ if (num_chunks == 0) {
414
+ just_do_an_install = true;
415
+ }
416
+ else {
417
+ for (const path of alreadyDownloaded) {
418
+ ioAddPath(path);
419
+ }
407
420
  }
408
421
  if (onStartDownloading) {
409
422
  onStartDownloading(num_chunks);
@@ -429,31 +442,33 @@ module.exports = {
429
442
  for (const writeStream of Object.values(writeStreams)) {
430
443
  await writeStream.close();
431
444
  }
445
+
446
+ if (just_do_an_install) {
447
+ await install(manifest, depotKey, installDir);
448
+ }
432
449
  },
433
450
  verifyChunks: async (depotId, depotKey, onDeletedFile) => {
434
- const files = await getFiles(`depot/${depotId}/chunk`);
451
+ const files = await fsPromises.readdir(`depot/${depotId}/chunk`);
435
452
  let promises = [];
436
- for (const file of files) {
437
- const hash = file.substr(file.length - 40);
438
- if (promises.length > 40) {
453
+ for (const hash of files) {
454
+ if (promises.length > 100) {
439
455
  await Promise.all(promises);
440
456
  promises = [];
441
457
  }
442
- promises.push(getFileContents(file).then(async data => {
458
+ promises.push((async () => {
443
459
  try {
444
- data = SteamCrypto.symmetricDecrypt(data, depotKey);
445
- data = await CdnCompression.unzip(data);
446
- if (sha1(data) == hash) {
447
- return;
448
- }
460
+ await getChunk(depotId, depotKey, hash);
461
+ return;
449
462
  }
450
463
  catch (e) {
464
+ //console.log(e);
451
465
  }
466
+ const file = path.resolve(`depot/${depotId}/chunk`, hash);
452
467
  await fsPromises.unlink(file);
453
468
  if (onDeletedFile) {
454
469
  onDeletedFile(file, hash);
455
470
  }
456
- }));
471
+ })());
457
472
  }
458
473
  await Promise.all(promises);
459
474
  },
package/mango.js CHANGED
@@ -2,8 +2,13 @@ const parseArguments = (base, bools = []) => {
2
2
  const res = [];
3
3
  let j = 0;
4
4
  for (let i = base; i != process.argv.length; ++i) {
5
- if (bools.indexOf(process.argv[i]) != -1) {
6
- res[process.argv[i]] = true;
5
+ if (process.argv[i].startsWith("--")) {
6
+ if (bools.indexOf(process.argv[i]) != -1) {
7
+ res[process.argv[i]] = true;
8
+ }
9
+ else {
10
+ console.log(`Ignoring unknown argument: ${process.argv[i]}`);
11
+ }
7
12
  }
8
13
  else {
9
14
  res[j++] = process.argv[i];
@@ -20,8 +25,7 @@ switch (tool) {
20
25
  } break;
21
26
 
22
27
  case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
23
- const depotId = process.argv[3];
24
- const manifestId = process.argv[4];
28
+ const [depotId, manifestId] = parseArguments(3);
25
29
  if (!depotId || !manifestId) {
26
30
  console.log("Syntax: mango download-manifest <depot id> <manifest id>");
27
31
  process.exit(1);
@@ -30,6 +34,7 @@ switch (tool) {
30
34
  const { fetchManifest } = require("./lib.js");
31
35
 
32
36
  (async () => {
37
+ console.log(`Fetching manifest...`);
33
38
  await fetchManifest(depotId, manifestId);
34
39
  console.log(`Saved in manifests/${depotId}_${manifestId}.manifest`);
35
40
  })();
@@ -43,12 +48,13 @@ switch (tool) {
43
48
  process.exit(1);
44
49
  }
45
50
 
46
- const ContentManifest = require("steam-user/components/content_manifest");
51
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
47
52
  const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
48
53
 
49
54
  const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
50
55
 
51
56
  (async () => {
57
+ console.log(`Fetching manifest...`);
52
58
  const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
53
59
  await downloadChunks(
54
60
  manifest,
@@ -63,18 +69,17 @@ switch (tool) {
63
69
  } break;
64
70
 
65
71
  case "populate-chunks": {
66
- const depotId = process.argv[3];
67
- const manifestId = process.argv[4];
68
- const installDir = process.argv[5];
72
+ const [depotId, manifestId, installDir] = parseArguments(3);
69
73
  if (!depotId || !manifestId || !installDir) {
70
74
  console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir>");
71
75
  process.exit(1);
72
76
  }
73
77
 
74
- const ContentManifest = require("steam-user/components/content_manifest");
78
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
75
79
  const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
76
80
 
77
81
  (async () => {
82
+ console.log(`Fetching manifest...`);
78
83
  const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
79
84
  console.log(`Fetching depot key...`);
80
85
  let depotKey = await fetchDepotKey(manifest.depot_id);
@@ -177,18 +182,17 @@ switch (tool) {
177
182
  } break;
178
183
 
179
184
  case "install": {
180
- const depotId = process.argv[3];
181
- const manifestId = process.argv[4];
182
- const installDir = process.argv[5];
185
+ const [depotId, manifestId, installDir] = parseArguments(3);
183
186
  if (!depotId || !manifestId) {
184
187
  console.log("Syntax: mango install <depot id> <manifest id> [install dir]");
185
188
  process.exit(1);
186
189
  }
187
190
 
188
- const ContentManifest = require("steam-user/components/content_manifest");
191
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
189
192
  const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
190
193
 
191
194
  (async () => {
195
+ console.log(`Fetching manifest...`);
192
196
  const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
193
197
  console.log(`Fetching depot key...`);
194
198
  let depotKey = await fetchDepotKey(manifest.depot_id);
@@ -208,24 +212,36 @@ switch (tool) {
208
212
  process.exit(1);
209
213
  }
210
214
 
211
- const ContentManifest = require("steam-user/components/content_manifest");
215
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
212
216
  const { DEFAULT_HOSTS, fetchManifest, fetchDepotKey, downloadAndInstall } = require("./lib.js");
213
217
 
214
218
  const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
215
219
 
216
220
  (async () => {
221
+ console.log(`Fetching manifest...`);
217
222
  const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
218
223
 
219
224
  console.log(`Fetching depot key...`);
220
225
  let depotKey = await fetchDepotKey(manifest.depot_id);
221
226
  depotKey = Buffer.from(depotKey, "hex");
222
227
 
228
+ let remaining_chunks;
223
229
  await downloadAndInstall(
224
230
  manifest,
225
231
  depotKey,
226
- undefined /*(num_chunks) => {}*/,
232
+ (num_chunks) => {
233
+ remaining_chunks = num_chunks;
234
+ if (remaining_chunks == 0) {
235
+ console.log("Done with downloading. Installing may still take a bit.");
236
+ }
237
+ },
227
238
  (path, host) => { console.log(`${path}: Downloading from ${host}`); },
228
- (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
239
+ (path, status, host) => {
240
+ console.log(`${path}: Got ${status/*} from ${host*/}`);
241
+ if (status == 200 && --remaining_chunks == 0) {
242
+ console.log("Done with downloading. Installing may still take a bit.");
243
+ }
244
+ },
229
245
  (path, err) => { console.log(`${path}: `, err); },
230
246
  hosts,
231
247
  installDir
@@ -244,10 +260,11 @@ switch (tool) {
244
260
  }
245
261
 
246
262
  const fs = require("fs");
247
- const ContentManifest = require("steam-user/components/content_manifest");
263
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
248
264
  const { fetchManifest, fetchDepotKey } = require("./lib.js");
249
265
 
250
266
  (async () => {
267
+ console.log(`Fetching manifest...`);
251
268
  const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
252
269
  if (manifest.filenames_encrypted && !args["--no-decrypt"]) {
253
270
  console.log("Fetching depot key to decrypt filenames...");
@@ -263,8 +280,7 @@ switch (tool) {
263
280
  } break;
264
281
 
265
282
  case "to-hashdeep-auditfile": {
266
- const depotId = process.argv[3];
267
- const manifestId = process.argv[4];
283
+ const [depotId, manifestId] = parseArguments(3);
268
284
  if (!depotId || !manifestId) {
269
285
  console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id>");
270
286
  process.exit(1);
@@ -272,10 +288,11 @@ switch (tool) {
272
288
 
273
289
  const fs = require("fs");
274
290
  const path = require("path");
275
- const ContentManifest = require("steam-user/components/content_manifest");
291
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
276
292
  const { fetchManifest, fetchDepotKey } = require("./lib.js");
277
293
 
278
294
  (async () => {
295
+ console.log(`Fetching manifest...`);
279
296
  const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
280
297
  if (manifest.filenames_encrypted) {
281
298
  console.log("Manifest has encrypted filenames. Fetching depot key...");
@@ -301,15 +318,13 @@ switch (tool) {
301
318
  } break;
302
319
 
303
320
  case "to-torrent": {
304
- const depotId = process.argv[3];
305
- const manifestId = process.argv[4];
306
- const fileHash = process.argv[5];
321
+ const [depotId, manifestId, fileHash] = parseArguments(3);
307
322
  if (!depotId || !manifestId || !fileHash) {
308
323
  console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
309
324
  process.exit(1);
310
325
  }
311
326
 
312
- const ContentManifest = require("steam-user/components/content_manifest");
327
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
313
328
 
314
329
  function toPieces(hexArray) {
315
330
  const bufs = hexArray.map(h => {
@@ -323,6 +338,7 @@ switch (tool) {
323
338
 
324
339
  (async () => {
325
340
  const { fetchManifest } = require("./lib.js");
341
+ console.log(`Fetching manifest...`);
326
342
  const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
327
343
  const file = manifest.files.find(x => x.sha_content == fileHash);
328
344
  if (!file) {
@@ -357,7 +373,7 @@ switch (tool) {
357
373
  } break;
358
374
 
359
375
  case "verify-chunks": {
360
- const depotId = process.argv[3];
376
+ const [depotId] = parseArguments(3);
361
377
  if (!depotId) {
362
378
  console.log("Syntax: mango verify-chunks <depot id>");
363
379
  process.exit(1);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "steam-manifest-tools",
3
- "version": "0.1.7",
3
+ "version": "0.1.9",
4
4
  "main": "lib.js",
5
5
  "repository": {
6
6
  "type": "git",
@@ -12,7 +12,7 @@
12
12
  "dependencies": {
13
13
  "@doctormckay/steam-crypto": "^1.2.0",
14
14
  "adm-zip": "^0.5.16",
15
- "steam-user": "^5.2.3"
15
+ "lean-and-mean-steam-user": "^0.1.0"
16
16
  },
17
17
  "devDependencies": {
18
18
  "express": "^5.1.0",