steam-manifest-tools 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/lib.js +70 -55
  2. package/mango.js +33 -25
  3. package/package.json +2 -2
package/lib.js CHANGED
@@ -1,11 +1,11 @@
1
1
  const crypto = require("node:crypto");
2
2
  const fs = require("node:fs");
3
3
  const fsPromises = require("node:fs/promises");
4
+ const os = require("node:os");
4
5
  const path = require("node:path");
5
6
  const worker_threads = require("node:worker_threads");
6
7
  const AdmZip = require("adm-zip");
7
- const CdnCompression = require("steam-user/components/cdn_compression");
8
- const ContentManifest = require("steam-user/components/content_manifest");
8
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
9
9
  const SteamCrypto = require("@doctormckay/steam-crypto");
10
10
 
11
11
  const DEFAULT_HOSTS = [
@@ -31,6 +31,8 @@ const DEFAULT_HOSTS = [
31
31
  "https://google2.cdn.steampipe.steamcontent.com",
32
32
  ];
33
33
 
34
+ const MAX_WORKERS = Math.ceil(os.cpus().length / 2);
35
+
34
36
  const workers = [];
35
37
 
36
38
  const getAvailableWorker = () => {
@@ -40,7 +42,7 @@ const getAvailableWorker = () => {
40
42
  w.busy = true;
41
43
  return Promise.resolve(w);
42
44
  }
43
- if (workers.length < 8) {
45
+ if (workers.length < MAX_WORKERS) {
44
46
  //console.log(`Creating a new worker`);
45
47
  const w = {
46
48
  inst: new worker_threads.Worker(__filename),
@@ -71,7 +73,7 @@ const getAvailableWorker = () => {
71
73
  clearInterval(i);
72
74
  resolve(w);
73
75
  }
74
- }, 4);
76
+ }, 15);
75
77
  });
76
78
  };
77
79
 
@@ -86,16 +88,18 @@ const doJobInWorker = (job) => {
86
88
  };
87
89
 
88
90
  if (!worker_threads.isMainThread) {
91
+ const CdnCompression = require("lean-and-mean-steam-user/components/cdn_compression");
92
+
89
93
  worker_threads.parentPort.on("message", (job) => {
90
94
  if ("getChunk" in job) {
91
95
  (async () => {
92
96
  const [ depotId, depotKey, hash ] = job.getChunk;
93
97
  try {
94
- let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
98
+ let data = fs.readFileSync(`depot/${depotId}/chunk/${hash}`);
95
99
  data = SteamCrypto.symmetricDecrypt(data, depotKey);
96
100
  data = await CdnCompression.unzip(data);
97
101
  if (sha1(data) != hash) {
98
- throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
102
+ throw new Error(`Hash mismatch`);
99
103
  }
100
104
  worker_threads.parentPort.postMessage(data);
101
105
  }
@@ -103,7 +107,7 @@ if (!worker_threads.isMainThread) {
103
107
  if (e.code == "ENOENT") {
104
108
  throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
105
109
  }
106
- throw e;
110
+ throw new Error(`depot/${depotId}/chunk/${hash} seems to have a problem: ${e.message}`);
107
111
  }
108
112
  })();
109
113
  }
@@ -129,7 +133,7 @@ const compress = (buf) => {
129
133
  return zip.toBuffer();
130
134
  };
131
135
 
132
- // https://stackoverflow.com/a/45130990
136
+ // deprecated
133
137
  const getFiles = async (dir) => {
134
138
  const dirents = await fsPromises.readdir(dir, { withFileTypes: true });
135
139
  const files = await Promise.all(dirents.map((dirent) => {
@@ -139,6 +143,7 @@ const getFiles = async (dir) => {
139
143
  return Array.prototype.concat(...files);
140
144
  };
141
145
 
146
+ // deprecated
142
147
  const getFileContents = async (file) => {
143
148
  while (true) {
144
149
  try {
@@ -227,6 +232,43 @@ const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishD
227
232
  });
228
233
  };
229
234
 
235
+ const install = async (manifest, depotKey, installDir, onFileWritten) => {
236
+ ContentManifest.decryptFilenames(manifest, depotKey);
237
+ installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
238
+ await fsPromises.mkdir(installDir, { recursive: true });
239
+ for (const file of manifest.files) {
240
+ const filename = file.filename.replace(/\\/g, "/");
241
+ if (file.flags & 64) {
242
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
243
+ continue;
244
+ }
245
+ const exists = fs.existsSync(path.join(installDir, filename));
246
+ if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
247
+ for (const chunk of file.chunks) {
248
+ chunk.offset = parseInt(chunk.offset);
249
+ }
250
+ file.chunks.sort((a, b) => a.offset - b.offset);
251
+
252
+ await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
253
+ const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
254
+ for (let i = 0; i != file.chunks.length; ) {
255
+ const j = Math.min(i + 100, file.chunks.length);
256
+ const promises = [];
257
+ for (; i != j; ++i) {
258
+ promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
259
+ }
260
+ for (const p of promises) {
261
+ await writeStream.write(await p);
262
+ }
263
+ }
264
+ await writeStream.close();
265
+ if (onFileWritten) {
266
+ onFileWritten(filename, exists);
267
+ }
268
+ }
269
+ }
270
+ };
271
+
230
272
  module.exports = {
231
273
  DEFAULT_HOSTS,
232
274
  sha1,
@@ -316,42 +358,7 @@ module.exports = {
316
358
  }
317
359
  },
318
360
  downloadChunks,
319
- install: async (manifest, depotKey, installDir, onFileWritten) => {
320
- ContentManifest.decryptFilenames(manifest, depotKey);
321
- installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
322
- await fsPromises.mkdir(installDir, { recursive: true });
323
- for (const file of manifest.files) {
324
- const filename = file.filename.replace(/\\/g, "/");
325
- if (file.flags & 64) {
326
- await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
327
- continue;
328
- }
329
- const exists = fs.existsSync(path.join(installDir, filename));
330
- if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
331
- for (const chunk of file.chunks) {
332
- chunk.offset = parseInt(chunk.offset);
333
- }
334
- file.chunks.sort((a, b) => a.offset - b.offset);
335
-
336
- await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
337
- const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
338
- for (let i = 0; i != file.chunks.length; ) {
339
- const j = Math.min(i + 100, file.chunks.length);
340
- const promises = [];
341
- for (; i != j; ++i) {
342
- promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
343
- }
344
- for (const p of promises) {
345
- await writeStream.write(await p);
346
- }
347
- }
348
- await writeStream.close();
349
- if (onFileWritten) {
350
- onFileWritten(filename, exists);
351
- }
352
- }
353
- }
354
- },
361
+ install,
355
362
  downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
356
363
  const depotId = manifest.depot_id;
357
364
 
@@ -399,11 +406,17 @@ module.exports = {
399
406
  }
400
407
  };
401
408
 
409
+ let just_do_an_install = false;
402
410
  await downloadChunks(
403
411
  manifest,
404
412
  (num_chunks, alreadyDownloaded) => {
405
- for (const path of alreadyDownloaded) {
406
- ioAddPath(path);
413
+ if (num_chunks == 0) {
414
+ just_do_an_install = true;
415
+ }
416
+ else {
417
+ for (const path of alreadyDownloaded) {
418
+ ioAddPath(path);
419
+ }
407
420
  }
408
421
  if (onStartDownloading) {
409
422
  onStartDownloading(num_chunks);
@@ -429,31 +442,33 @@ module.exports = {
429
442
  for (const writeStream of Object.values(writeStreams)) {
430
443
  await writeStream.close();
431
444
  }
445
+
446
+ if (just_do_an_install) {
447
+ await install(manifest, depotKey, installDir);
448
+ }
432
449
  },
433
450
  verifyChunks: async (depotId, depotKey, onDeletedFile) => {
434
- const files = await getFiles(`depot/${depotId}/chunk`);
451
+ const files = await fsPromises.readdir(`depot/${depotId}/chunk`);
435
452
  let promises = [];
436
453
  for (const file of files) {
437
454
  const hash = file.substr(file.length - 40);
438
- if (promises.length > 40) {
455
+ if (promises.length > 100) {
439
456
  await Promise.all(promises);
440
457
  promises = [];
441
458
  }
442
- promises.push(getFileContents(file).then(async data => {
459
+ promises.push((async () => {
443
460
  try {
444
- data = SteamCrypto.symmetricDecrypt(data, depotKey);
445
- data = await CdnCompression.unzip(data);
446
- if (sha1(data) == hash) {
447
- return;
448
- }
461
+ await getChunk(depotId, depotKey, hash);
462
+ return;
449
463
  }
450
464
  catch (e) {
465
+ //console.log(e);
451
466
  }
452
467
  await fsPromises.unlink(file);
453
468
  if (onDeletedFile) {
454
469
  onDeletedFile(file, hash);
455
470
  }
456
- }));
471
+ })());
457
472
  }
458
473
  await Promise.all(promises);
459
474
  },
package/mango.js CHANGED
@@ -2,8 +2,13 @@ const parseArguments = (base, bools = []) => {
2
2
  const res = [];
3
3
  let j = 0;
4
4
  for (let i = base; i != process.argv.length; ++i) {
5
- if (bools.indexOf(process.argv[i]) != -1) {
6
- res[process.argv[i]] = true;
5
+ if (process.argv[i].startsWith("--")) {
6
+ if (bools.indexOf(process.argv[i]) != -1) {
7
+ res[process.argv[i]] = true;
8
+ }
9
+ else {
10
+ console.log(`Ignoring unknown argument: ${process.argv[i]}`);
11
+ }
7
12
  }
8
13
  else {
9
14
  res[j++] = process.argv[i];
@@ -20,8 +25,7 @@ switch (tool) {
20
25
  } break;
21
26
 
22
27
  case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
23
- const depotId = process.argv[3];
24
- const manifestId = process.argv[4];
28
+ const [depotId, manifestId] = parseArguments(3);
25
29
  if (!depotId || !manifestId) {
26
30
  console.log("Syntax: mango download-manifest <depot id> <manifest id>");
27
31
  process.exit(1);
@@ -43,7 +47,7 @@ switch (tool) {
43
47
  process.exit(1);
44
48
  }
45
49
 
46
- const ContentManifest = require("steam-user/components/content_manifest");
50
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
47
51
  const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
48
52
 
49
53
  const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
@@ -63,15 +67,13 @@ switch (tool) {
63
67
  } break;
64
68
 
65
69
  case "populate-chunks": {
66
- const depotId = process.argv[3];
67
- const manifestId = process.argv[4];
68
- const installDir = process.argv[5];
70
+ const [depotId, manifestId, installDir] = parseArguments(3);
69
71
  if (!depotId || !manifestId || !installDir) {
70
72
  console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir>");
71
73
  process.exit(1);
72
74
  }
73
75
 
74
- const ContentManifest = require("steam-user/components/content_manifest");
76
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
75
77
  const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
76
78
 
77
79
  (async () => {
@@ -177,15 +179,13 @@ switch (tool) {
177
179
  } break;
178
180
 
179
181
  case "install": {
180
- const depotId = process.argv[3];
181
- const manifestId = process.argv[4];
182
- const installDir = process.argv[5];
182
+ const [depotId, manifestId, installDir] = parseArguments(3);
183
183
  if (!depotId || !manifestId) {
184
184
  console.log("Syntax: mango install <depot id> <manifest id> [install dir]");
185
185
  process.exit(1);
186
186
  }
187
187
 
188
- const ContentManifest = require("steam-user/components/content_manifest");
188
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
189
189
  const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
190
190
 
191
191
  (async () => {
@@ -208,7 +208,7 @@ switch (tool) {
208
208
  process.exit(1);
209
209
  }
210
210
 
211
- const ContentManifest = require("steam-user/components/content_manifest");
211
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
212
212
  const { DEFAULT_HOSTS, fetchManifest, fetchDepotKey, downloadAndInstall } = require("./lib.js");
213
213
 
214
214
  const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
@@ -220,12 +220,23 @@ switch (tool) {
220
220
  let depotKey = await fetchDepotKey(manifest.depot_id);
221
221
  depotKey = Buffer.from(depotKey, "hex");
222
222
 
223
+ let remaining_chunks;
223
224
  await downloadAndInstall(
224
225
  manifest,
225
226
  depotKey,
226
- undefined /*(num_chunks) => {}*/,
227
+ (num_chunks) => {
228
+ remaining_chunks = num_chunks;
229
+ if (remaining_chunks == 0) {
230
+ console.log("Done with downloading. Installing may still take a bit.");
231
+ }
232
+ },
227
233
  (path, host) => { console.log(`${path}: Downloading from ${host}`); },
228
- (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
234
+ (path, status, host) => {
235
+ console.log(`${path}: Got ${status/*} from ${host*/}`);
236
+ if (status == 200 && --remaining_chunks == 0) {
237
+ console.log("Done with downloading. Installing may still take a bit.");
238
+ }
239
+ },
229
240
  (path, err) => { console.log(`${path}: `, err); },
230
241
  hosts,
231
242
  installDir
@@ -244,7 +255,7 @@ switch (tool) {
244
255
  }
245
256
 
246
257
  const fs = require("fs");
247
- const ContentManifest = require("steam-user/components/content_manifest");
258
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
248
259
  const { fetchManifest, fetchDepotKey } = require("./lib.js");
249
260
 
250
261
  (async () => {
@@ -263,8 +274,7 @@ switch (tool) {
263
274
  } break;
264
275
 
265
276
  case "to-hashdeep-auditfile": {
266
- const depotId = process.argv[3];
267
- const manifestId = process.argv[4];
277
+ const [depotId, manifestId] = parseArguments(3);
268
278
  if (!depotId || !manifestId) {
269
279
  console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id>");
270
280
  process.exit(1);
@@ -272,7 +282,7 @@ switch (tool) {
272
282
 
273
283
  const fs = require("fs");
274
284
  const path = require("path");
275
- const ContentManifest = require("steam-user/components/content_manifest");
285
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
276
286
  const { fetchManifest, fetchDepotKey } = require("./lib.js");
277
287
 
278
288
  (async () => {
@@ -301,15 +311,13 @@ switch (tool) {
301
311
  } break;
302
312
 
303
313
  case "to-torrent": {
304
- const depotId = process.argv[3];
305
- const manifestId = process.argv[4];
306
- const fileHash = process.argv[5];
314
+ const [depotId, manifestId, fileHash] = parseArguments(3);
307
315
  if (!depotId || !manifestId || !fileHash) {
308
316
  console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
309
317
  process.exit(1);
310
318
  }
311
319
 
312
- const ContentManifest = require("steam-user/components/content_manifest");
320
+ const ContentManifest = require("lean-and-mean-steam-user/components/content_manifest");
313
321
 
314
322
  function toPieces(hexArray) {
315
323
  const bufs = hexArray.map(h => {
@@ -357,7 +365,7 @@ switch (tool) {
357
365
  } break;
358
366
 
359
367
  case "verify-chunks": {
360
- const depotId = process.argv[3];
368
+ const [depotId] = parseArguments(3);
361
369
  if (!depotId) {
362
370
  console.log("Syntax: mango verify-chunks <depot id>");
363
371
  process.exit(1);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "steam-manifest-tools",
3
- "version": "0.1.7",
3
+ "version": "0.1.8",
4
4
  "main": "lib.js",
5
5
  "repository": {
6
6
  "type": "git",
@@ -12,7 +12,7 @@
12
12
  "dependencies": {
13
13
  "@doctormckay/steam-crypto": "^1.2.0",
14
14
  "adm-zip": "^0.5.16",
15
- "steam-user": "^5.2.3"
15
+ "lean-and-mean-steam-user": "^0.1.0"
16
16
  },
17
17
  "devDependencies": {
18
18
  "express": "^5.1.0",