steam-manifest-tools 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/lib.js +119 -44
  2. package/package.json +1 -1
package/lib.js CHANGED
@@ -1,7 +1,8 @@
1
- const crypto = require("crypto");
2
- const fs = require("fs");
3
- const fsPromises = require("fs/promises");
4
- const path = require("path");
1
+ const crypto = require("node:crypto");
2
+ const fs = require("node:fs");
3
+ const fsPromises = require("node:fs/promises");
4
+ const path = require("node:path");
5
+ const worker_threads = require("node:worker_threads");
5
6
  const AdmZip = require("adm-zip");
6
7
  const CdnCompression = require("steam-user/components/cdn_compression");
7
8
  const ContentManifest = require("steam-user/components/content_manifest");
@@ -30,6 +31,85 @@ const DEFAULT_HOSTS = [
30
31
  "https://google2.cdn.steampipe.steamcontent.com",
31
32
  ];
32
33
 
34
+ const workers = [];
35
+
36
+ const getAvailableWorker = () => {
37
+ const w = workers.find(x => !x.busy);
38
+ if (w) {
39
+ //console.log(`Found an idle worker`);
40
+ w.busy = true;
41
+ return Promise.resolve(w);
42
+ }
43
+ if (workers.length < 8) {
44
+ //console.log(`Creating a new worker`);
45
+ const w = {
46
+ inst: new worker_threads.Worker(__filename),
47
+ busy: true,
48
+ }
49
+ w.inst.on("message", (x) => {
50
+ w.busy = false;
51
+ w.resolve(x);
52
+ });
53
+ w.inst.on("error", (x) => {
54
+ w.busy = false;
55
+ w.reject(x);
56
+ });
57
+ w.inst.on("exit", (code) => {
58
+ console.log(`Worker exited with code ${code}`);
59
+ workers.splice(workers.indexOf(w), 1);
60
+ });
61
+ workers.push(w);
62
+ return Promise.resolve(w);
63
+ }
64
+ //console.log("All workers busy, waiting");
65
+ return new Promise(resolve => {
66
+ let i;
67
+ i = setInterval(() => {
68
+ const w = workers.find(x => !x.busy);
69
+ if (w) {
70
+ w.busy = true;
71
+ clearInterval(i);
72
+ resolve(w);
73
+ }
74
+ }, 4);
75
+ });
76
+ };
77
+
78
+ const doJobInWorker = (job) => {
79
+ return new Promise((resolve, reject) => {
80
+ getAvailableWorker().then((w) => {
81
+ w.resolve = resolve;
82
+ w.reject = reject;
83
+ w.inst.postMessage(job);
84
+ });
85
+ });
86
+ };
87
+
88
+ if (!worker_threads.isMainThread) {
89
+ worker_threads.parentPort.on("message", (job) => {
90
+ if ("getChunk" in job) {
91
+ (async () => {
92
+ const [ depotId, depotKey, hash ] = job.getChunk;
93
+ try {
94
+ let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
95
+ data = SteamCrypto.symmetricDecrypt(data, depotKey);
96
+ data = await CdnCompression.unzip(data);
97
+ if (sha1(data) != hash) {
98
+ throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
99
+ }
100
+ worker_threads.parentPort.postMessage(data);
101
+ }
102
+ catch (e) {
103
+ if (e.code == "ENOENT") {
104
+ throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
105
+ }
106
+ throw e;
107
+ }
108
+ })();
109
+ }
110
+ });
111
+ }
112
+
33
113
  const sha1 = (data) => crypto.createHash("sha1").update(data).digest("hex");
34
114
 
35
115
  const sha1file = async (file) => {
@@ -74,22 +154,10 @@ const getFileContents = async (file) => {
74
154
  }
75
155
  };
76
156
 
77
- const getChunk = async (depotId, depotKey, hash) => {
78
- try {
79
- let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
80
- data = SteamCrypto.symmetricDecrypt(data, depotKey);
81
- data = await CdnCompression.unzip(data);
82
- if (sha1(data) != hash) {
83
- throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
84
- }
85
- return data;
86
- }
87
- catch (e) {
88
- if (e.code == "ENOENT") {
89
- throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
90
- }
91
- throw e;
92
- }
157
+ const getChunk = (depotId, depotKey, hash) => {
158
+ return doJobInWorker({
159
+ getChunk: [ depotId, depotKey, hash ]
160
+ });
93
161
  };
94
162
 
95
163
  const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
@@ -252,34 +320,37 @@ module.exports = {
252
320
  ContentManifest.decryptFilenames(manifest, depotKey);
253
321
  installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
254
322
  await fsPromises.mkdir(installDir, { recursive: true });
255
- const promises = [];
256
323
  for (const file of manifest.files) {
257
- promises.push((async () => {
258
- const filename = file.filename.replace(/\\/g, "/");
259
- if (file.flags & 64) {
260
- await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
261
- return;
324
+ const filename = file.filename.replace(/\\/g, "/");
325
+ if (file.flags & 64) {
326
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
327
+ continue;
328
+ }
329
+ const exists = fs.existsSync(path.join(installDir, filename));
330
+ if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
331
+ for (const chunk of file.chunks) {
332
+ chunk.offset = parseInt(chunk.offset);
262
333
  }
263
- const exists = fs.existsSync(path.join(installDir, filename));
264
- if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
265
- for (const chunk of file.chunks) {
266
- chunk.offset = parseInt(chunk.offset);
267
- }
268
- file.chunks.sort((a, b) => a.offset - b.offset);
334
+ file.chunks.sort((a, b) => a.offset - b.offset);
269
335
 
270
- await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
271
- const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
272
- for (const chunk of file.chunks) {
273
- await writeStream.write(await getChunk(manifest.depot_id, depotKey, chunk.sha));
336
+ await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
337
+ const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
338
+ for (let i = 0; i != file.chunks.length; ) {
339
+ const j = Math.min(i + 100, file.chunks.length);
340
+ const promises = [];
341
+ for (; i != j; ++i) {
342
+ promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
274
343
  }
275
- await writeStream.close();
276
- if (onFileWritten) {
277
- onFileWritten(filename, exists);
344
+ for (const p of promises) {
345
+ await writeStream.write(await p);
278
346
  }
279
347
  }
280
- })());
348
+ await writeStream.close();
349
+ if (onFileWritten) {
350
+ onFileWritten(filename, exists);
351
+ }
352
+ }
281
353
  }
282
- await Promise.all(promises);
283
354
  },
284
355
  downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
285
356
  const depotId = manifest.depot_id;
@@ -305,11 +376,15 @@ module.exports = {
305
376
  const ioLoop = async () => {
306
377
  ioLoopRunning = true;
307
378
  while (ioQueue.length > 0) {
308
- const chunkSha = ioQueue.shift();
309
- const data = await getChunk(depotId, depotKey, chunkSha);
379
+ const completedChunks = {};
380
+ for (let i = 0; ioQueue.length > 0 && i != 100; ++i) {
381
+ const chunkSha = ioQueue.shift();
382
+ completedChunks[chunkSha] = getChunk(depotId, depotKey, chunkSha);
383
+ }
310
384
  for (const file of manifest.files) {
311
385
  for (const chunk of file.chunks) {
312
- if (chunk.sha == chunkSha) {
386
+ if (chunk.sha in completedChunks) {
387
+ const data = await completedChunks[chunk.sha];
313
388
  await writeStreams[file.filename].write(data, 0, data.byteLength, parseInt(chunk.offset));
314
389
  }
315
390
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "steam-manifest-tools",
3
- "version": "0.1.6",
3
+ "version": "0.1.7",
4
4
  "main": "lib.js",
5
5
  "repository": {
6
6
  "type": "git",