steam-manifest-tools 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +6 -5
  2. package/lib.js +260 -96
  3. package/mango.js +375 -322
  4. package/package.json +1 -1
package/README.md CHANGED
@@ -1,14 +1,15 @@
1
1
  Tools to interact with Steam depots and manifests.
2
2
  - `mango download-chunks <depot id> <manifest id> [--lancache]` (basically the same as prefilling a LAN cache, optionally downloads from a LAN cache)
3
- - `mango populate-chunks <depot id> <manifest id> <install dir> [depot key]` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
3
+ - `mango populate-chunks <depot id> <manifest id> <install dir>` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
4
4
  - `mango lancache`
5
5
  - Must run as root to obtain port 80
6
6
  - `lancache.steamcontent.com` must resolve to your LAN cache server IP (either via custom DNS server or OS hosts file)
7
- - `mango install <depot id> <manifest id> [depot key]` (requires chunks to have been populated and/or downloaded beforehand)
8
- - `mango to-json <depot id> <manifest id> [depot key|--no-decrypt]`
9
- - `mango to-hashdeep-auditfile <depot id> <manifest id> [depot key]`
7
+ - `mango install <depot id> <manifest id> [install dir]` (requires chunks to have been populated and/or downloaded beforehand)
8
+ - `mango download-and-install <depot id> <manifest id> [install dir] [--lancache]`
9
+ - `mango to-json <depot id> <manifest id> [--no-decrypt]`
10
+ - `mango to-hashdeep-auditfile <depot id> <manifest id>`
10
11
  - Note that hashdeep requires exact path matches and this script makes a few assumptions
11
12
  - Example verification usage: `hashdeep -a -l -v -v -k 241561_8497448424664183398.auditfile -r 8497448424664183398`
12
13
  - `mango to-torrent <depot id> <manifest id> <file hash>`
13
- - `mango verify-chunks <depot id> [depot key]`
14
+ - `mango verify-chunks <depot id>`
14
15
  - `mango download-manifest <depot id> <manifest id>`
package/lib.js CHANGED
@@ -1,7 +1,8 @@
1
- const crypto = require("crypto");
2
- const fs = require("fs");
3
- const fsPromises = require("fs/promises");
4
- const path = require("path");
1
+ const crypto = require("node:crypto");
2
+ const fs = require("node:fs");
3
+ const fsPromises = require("node:fs/promises");
4
+ const path = require("node:path");
5
+ const worker_threads = require("node:worker_threads");
5
6
  const AdmZip = require("adm-zip");
6
7
  const CdnCompression = require("steam-user/components/cdn_compression");
7
8
  const ContentManifest = require("steam-user/components/content_manifest");
@@ -30,6 +31,85 @@ const DEFAULT_HOSTS = [
30
31
  "https://google2.cdn.steampipe.steamcontent.com",
31
32
  ];
32
33
 
34
+ const workers = [];
35
+
36
+ const getAvailableWorker = () => {
37
+ const w = workers.find(x => !x.busy);
38
+ if (w) {
39
+ //console.log(`Found an idle worker`);
40
+ w.busy = true;
41
+ return Promise.resolve(w);
42
+ }
43
+ if (workers.length < 8) {
44
+ //console.log(`Creating a new worker`);
45
+ const w = {
46
+ inst: new worker_threads.Worker(__filename),
47
+ busy: true,
48
+ }
49
+ w.inst.on("message", (x) => {
50
+ w.busy = false;
51
+ w.resolve(x);
52
+ });
53
+ w.inst.on("error", (x) => {
54
+ w.busy = false;
55
+ w.reject(x);
56
+ });
57
+ w.inst.on("exit", (code) => {
58
+ console.log(`Worker exited with code ${code}`);
59
+ workers.splice(workers.indexOf(w), 1);
60
+ });
61
+ workers.push(w);
62
+ return Promise.resolve(w);
63
+ }
64
+ //console.log("All workers busy, waiting");
65
+ return new Promise(resolve => {
66
+ let i;
67
+ i = setInterval(() => {
68
+ const w = workers.find(x => !x.busy);
69
+ if (w) {
70
+ w.busy = true;
71
+ clearInterval(i);
72
+ resolve(w);
73
+ }
74
+ }, 4);
75
+ });
76
+ };
77
+
78
+ const doJobInWorker = (job) => {
79
+ return new Promise((resolve, reject) => {
80
+ getAvailableWorker().then((w) => {
81
+ w.resolve = resolve;
82
+ w.reject = reject;
83
+ w.inst.postMessage(job);
84
+ });
85
+ });
86
+ };
87
+
88
+ if (!worker_threads.isMainThread) {
89
+ worker_threads.parentPort.on("message", (job) => {
90
+ if ("getChunk" in job) {
91
+ (async () => {
92
+ const [ depotId, depotKey, hash ] = job.getChunk;
93
+ try {
94
+ let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
95
+ data = SteamCrypto.symmetricDecrypt(data, depotKey);
96
+ data = await CdnCompression.unzip(data);
97
+ if (sha1(data) != hash) {
98
+ throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
99
+ }
100
+ worker_threads.parentPort.postMessage(data);
101
+ }
102
+ catch (e) {
103
+ if (e.code == "ENOENT") {
104
+ throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
105
+ }
106
+ throw e;
107
+ }
108
+ })();
109
+ }
110
+ });
111
+ }
112
+
33
113
  const sha1 = (data) => crypto.createHash("sha1").update(data).digest("hex");
34
114
 
35
115
  const sha1file = async (file) => {
@@ -74,22 +154,77 @@ const getFileContents = async (file) => {
74
154
  }
75
155
  };
76
156
 
77
- const getChunk = async (depotId, depotKey, hash) => {
78
- try {
79
- let data = await getFileContents(`depot/${depotId}/chunk/${hash}`);
80
- data = SteamCrypto.symmetricDecrypt(data, depotKey);
81
- data = await CdnCompression.unzip(data);
82
- if (sha1(data) != hash) {
83
- throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
157
+ const getChunk = (depotId, depotKey, hash) => {
158
+ return doJobInWorker({
159
+ getChunk: [ depotId, depotKey, hash ]
160
+ });
161
+ };
162
+
163
+ const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
164
+ hosts ??= DEFAULT_HOSTS;
165
+ return new Promise(resolve => {
166
+ fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
167
+ const toDownload = {};
168
+ const alreadyDownloaded = [];
169
+ for (const file of manifest.files) {
170
+ for (const chunk of file.chunks) {
171
+ if (fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
172
+ alreadyDownloaded.push(`depot/${manifest.depot_id}/chunk/${chunk.sha}`);
173
+ }
174
+ else {
175
+ toDownload[`depot/${manifest.depot_id}/chunk/${chunk.sha}`] = true;
176
+ }
177
+ }
84
178
  }
85
- return data;
86
- }
87
- catch (e) {
88
- if (e.code == "ENOENT") {
89
- throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
179
+ let remaining_chunks = Object.keys(toDownload).length;
180
+ if (onStartDownloading) {
181
+ onStartDownloading(remaining_chunks, alreadyDownloaded);
90
182
  }
91
- throw e;
92
- }
183
+ if (remaining_chunks == 0) {
184
+ resolve();
185
+ }
186
+ let /*host_i = 0,*/ running = 0;
187
+ const concurrency = hosts.length == 1 ? 4 : (hosts.length * 2);
188
+ const loop = () => {
189
+ while (running < concurrency) {
190
+ const path = Object.keys(toDownload)[0];
191
+ if (!path) {
192
+ break;
193
+ }
194
+ ++running;
195
+ delete toDownload[path];
196
+ const host = hosts[Math.floor(Math.random()*hosts.length)];
197
+ //const host = hosts[host_i]; host_i = (host_i + 1) % hosts.length;
198
+ if (onStartDownload) {
199
+ onStartDownload(path, host);
200
+ }
201
+ fetch(`${host}/${path}`).then(async res => {
202
+ if (res.status == 200) {
203
+ const ab = await res.arrayBuffer();
204
+ await fsPromises.writeFile(path, Buffer.from(ab));
205
+ if (--remaining_chunks == 0) {
206
+ resolve();
207
+ }
208
+ }
209
+ else {
210
+ toDownload[path] = true;
211
+ }
212
+ if (onFinishDownload) {
213
+ onFinishDownload(path, res.status, host);
214
+ }
215
+ }).catch(err => {
216
+ if (onErroredDownload) {
217
+ onErroredDownload(path, err);
218
+ }
219
+ toDownload[path] = true;
220
+ }).finally(() => {
221
+ --running;
222
+ loop();
223
+ });
224
+ }
225
+ };
226
+ loop();
227
+ });
93
228
  };
94
229
 
95
230
  module.exports = {
@@ -125,17 +260,25 @@ module.exports = {
125
260
  throw new Error(`Could not find the given manifest (${manifestId}). Double-check with https://steamdb.info/depot/${depotId}/manifests/ and report an issue in https://github.com/Sainan/k25FCdfEOoEJ42S6/issues if you're sure the manifest exists.`);
126
261
  },
127
262
  fetchDepotKey: async (depotId) => {
263
+ try {
264
+ return await fsPromises.readFile(`depot/${depotId}/key.txt`, "utf-8");
265
+ }
266
+ catch (e) {
267
+ // fallthrough
268
+ }
128
269
  const depotkeys = await fetch("https://raw.githubusercontent.com/SteamAutoCracks/ManifestHub/refs/heads/main/depotkeys.json").then(x => x.json());
129
270
  if (!depotkeys[depotId]) {
130
271
  throw new Error(`Failed to get the depot key.`);
131
272
  }
273
+ await fsPromises.mkdir(`depot/${depotId}`, { recursive: true });
274
+ fsPromises.writeFile(`depot/${depotId}/key.txt`, depotkeys[depotId], "utf-8");
132
275
  return depotkeys[depotId];
133
276
  },
134
277
  populateChunks: async (manifest, depotKey, installDir, onProgress) => {
135
278
  if (manifest.filenames_encrypted) {
136
279
  ContentManifest.decryptFilenames(manifest, depotKey);
137
280
  }
138
- fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
281
+ await fsPromises.mkdir(`depot/${manifest.depot_id}/chunk`, { recursive: true });
139
282
  let file_i = 0;
140
283
  for (const file of manifest.files) {
141
284
  if (file.flags & 64) {
@@ -172,99 +315,120 @@ module.exports = {
172
315
  ++file_i;
173
316
  }
174
317
  },
175
- downloadChunks: (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
176
- hosts ??= DEFAULT_HOSTS;
177
- return new Promise(resolve => {
178
- fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
179
- const toDownload = {};
180
- for (const file of manifest.files) {
318
+ downloadChunks,
319
+ install: async (manifest, depotKey, installDir, onFileWritten) => {
320
+ ContentManifest.decryptFilenames(manifest, depotKey);
321
+ installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
322
+ await fsPromises.mkdir(installDir, { recursive: true });
323
+ for (const file of manifest.files) {
324
+ const filename = file.filename.replace(/\\/g, "/");
325
+ if (file.flags & 64) {
326
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
327
+ continue;
328
+ }
329
+ const exists = fs.existsSync(path.join(installDir, filename));
330
+ if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
181
331
  for (const chunk of file.chunks) {
182
- if (!fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
183
- toDownload[`depot/${manifest.depot_id}/chunk/${chunk.sha}`] = true;
184
- }
332
+ chunk.offset = parseInt(chunk.offset);
185
333
  }
186
- }
187
- let remaining_chunks = Object.keys(toDownload).length;
188
- if (onStartDownloading) {
189
- onStartDownloading(remaining_chunks);
190
- }
191
- if (remaining_chunks == 0) {
192
- resolve();
193
- }
194
- let /*host_i = 0,*/ running = 0;
195
- const concurrency = hosts.length == 1 ? 4 : (hosts.length * 2);
196
- const loop = () => {
197
- while (running < concurrency) {
198
- const path = Object.keys(toDownload)[0];
199
- if (!path) {
200
- break;
334
+ file.chunks.sort((a, b) => a.offset - b.offset);
335
+
336
+ await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
337
+ const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
338
+ for (let i = 0; i != file.chunks.length; ) {
339
+ const j = Math.min(i + 100, file.chunks.length);
340
+ const promises = [];
341
+ for (; i != j; ++i) {
342
+ promises.push(getChunk(manifest.depot_id, depotKey, file.chunks[i].sha));
201
343
  }
202
- ++running;
203
- delete toDownload[path];
204
- const host = hosts[Math.floor(Math.random()*hosts.length)];
205
- //const host = hosts[host_i]; host_i = (host_i + 1) % hosts.length;
206
- if (onStartDownload) {
207
- onStartDownload(path, host);
344
+ for (const p of promises) {
345
+ await writeStream.write(await p);
208
346
  }
209
- fetch(`${host}/${path}`).then(async res => {
210
- if (onFinishDownload) {
211
- onFinishDownload(path, res.status, host);
212
- }
213
- if (res.status == 200) {
214
- const ab = await res.arrayBuffer();
215
- await fsPromises.writeFile(path, Buffer.from(ab));
216
- if (--remaining_chunks == 0) {
217
- resolve();
218
- }
219
- }
220
- else {
221
- toDownload[path] = true;
222
- }
223
- }).catch(err => {
224
- if (onErroredDownload) {
225
- onErroredDownload(path, err);
226
- }
227
- toDownload[path] = true;
228
- }).finally(() => {
229
- --running;
230
- loop();
231
- });
232
347
  }
233
- };
234
- loop();
235
- });
348
+ await writeStream.close();
349
+ if (onFileWritten) {
350
+ onFileWritten(filename, exists);
351
+ }
352
+ }
353
+ }
236
354
  },
237
- install: async (manifest, depotKey, installDir, onFileWritten) => {
355
+ downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
356
+ const depotId = manifest.depot_id;
357
+
238
358
  ContentManifest.decryptFilenames(manifest, depotKey);
239
359
  installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
240
- fs.mkdirSync(installDir, { recursive: true });
241
- const promises = [];
360
+
361
+ const writeStreams = {};
242
362
  for (const file of manifest.files) {
363
+ const filename = file.filename.replace(/\\/g, "/");
243
364
  if (file.flags & 64) {
244
- continue;
365
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
245
366
  }
246
- promises.push((async () => {
247
- const filename = file.filename.replace(/\\/g, "/");
248
- const exists = fs.existsSync(path.join(installDir, filename));
249
- if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
250
- for (const chunk of file.chunks) {
251
- chunk.offset = parseInt(chunk.offset);
252
- }
253
- file.chunks.sort((a, b) => a.offset - b.offset);
367
+ else {
368
+ await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
369
+ writeStreams[file.filename] = await fsPromises.open(path.join(installDir, filename), "w");
370
+ }
371
+ }
254
372
 
255
- await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
256
- const writeStream = await fsPromises.open(path.join(installDir, filename), "w");
373
+ const ioQueue = [];
374
+ let ioLoopRunning = false;
375
+ let ioLoopPromise;
376
+ const ioLoop = async () => {
377
+ ioLoopRunning = true;
378
+ while (ioQueue.length > 0) {
379
+ const completedChunks = {};
380
+ for (let i = 0; ioQueue.length > 0 && i != 100; ++i) {
381
+ const chunkSha = ioQueue.shift();
382
+ completedChunks[chunkSha] = getChunk(depotId, depotKey, chunkSha);
383
+ }
384
+ for (const file of manifest.files) {
257
385
  for (const chunk of file.chunks) {
258
- await writeStream.write(await getChunk(manifest.depot_id, depotKey, chunk.sha));
259
- }
260
- await writeStream.close();
261
- if (onFileWritten) {
262
- onFileWritten(filename, exists);
386
+ if (chunk.sha in completedChunks) {
387
+ const data = await completedChunks[chunk.sha];
388
+ await writeStreams[file.filename].write(data, 0, data.byteLength, parseInt(chunk.offset));
389
+ }
263
390
  }
264
391
  }
265
- })());
392
+ }
393
+ ioLoopRunning = false;
394
+ };
395
+ const ioAddPath = (path) => {
396
+ ioQueue.push(path.substr(path.length - 40));
397
+ if (!ioLoopRunning) {
398
+ ioLoopPromise = ioLoop();
399
+ }
400
+ };
401
+
402
+ await downloadChunks(
403
+ manifest,
404
+ (num_chunks, alreadyDownloaded) => {
405
+ for (const path of alreadyDownloaded) {
406
+ ioAddPath(path);
407
+ }
408
+ if (onStartDownloading) {
409
+ onStartDownloading(num_chunks);
410
+ }
411
+ },
412
+ onStartDownload,
413
+ (path, status, host) => {
414
+ if (status == 200) {
415
+ ioAddPath(path);
416
+ }
417
+ if (onFinishDownload) {
418
+ onFinishDownload(path, status, host);
419
+ }
420
+ },
421
+ onErroredDownload,
422
+ hosts
423
+ );
424
+
425
+ if (ioLoopRunning) {
426
+ await ioLoopPromise;
427
+ }
428
+
429
+ for (const writeStream of Object.values(writeStreams)) {
430
+ await writeStream.close();
266
431
  }
267
- await Promise.all(promises);
268
432
  },
269
433
  verifyChunks: async (depotId, depotKey, onDeletedFile) => {
270
434
  const files = await getFiles(`depot/${depotId}/chunk`);
package/mango.js CHANGED
@@ -1,325 +1,378 @@
1
+ const parseArguments = (base, bools = []) => {
2
+ const res = [];
3
+ let j = 0;
4
+ for (let i = base; i != process.argv.length; ++i) {
5
+ if (bools.indexOf(process.argv[i]) != -1) {
6
+ res[process.argv[i]] = true;
7
+ }
8
+ else {
9
+ res[j++] = process.argv[i];
10
+ }
11
+ }
12
+ return res;
13
+ };
14
+
1
15
  const tool = process.argv[2];
2
16
  switch (tool) {
3
- default: {
4
- console.error(tool ? `Unknown tool: ${tool}` : "Syntax: mango <tool>");
5
- process.exit(1);
6
- } break;
7
-
8
- case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
9
- const depotId = process.argv[3];
10
- const manifestId = process.argv[4];
11
- if (!depotId || !manifestId) {
12
- console.log("Syntax: mango download-manifest <depot id> <manifest id>");
13
- process.exit(1);
14
- }
15
-
16
- const { fetchManifest } = require("./lib.js");
17
-
18
- (async () => {
19
- await fetchManifest(depotId, manifestId);
20
- console.log(`Saved in manifests/${depotId}_${manifestId}.manifest`);
21
- })();
22
- } break;
23
-
24
- case "download-chunks": {
25
- let depotId = process.argv[3];
26
- let manifestId = process.argv[4];
27
- let lancache = (process.argv[5] == "--lancache");
28
- if (!depotId || !manifestId) {
29
- console.log("Syntax: mango download-chunks <depot id> <manifest id> [--lancache]");
30
- process.exit(1);
31
- }
32
-
33
- const ContentManifest = require("steam-user/components/content_manifest");
34
- const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
35
-
36
- const hosts = lancache ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
37
-
38
- (async () => {
39
- const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
40
- await downloadChunks(
41
- manifest,
42
- undefined /*(num_chunks) => {}*/,
43
- (path, host) => { console.log(`${path}: Downloading from ${host}`); },
44
- (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
45
- (path, err) => { console.log(`${path}: `, err); },
46
- hosts
47
- )
48
- process.exit(0);
49
- })();
50
- } break;
51
-
52
- case "populate-chunks": {
53
- const depotId = process.argv[3];
54
- const manifestId = process.argv[4];
55
- const installDir = process.argv[5];
56
- let depotKey = process.argv[6];
57
- if (!depotId || !manifestId || !installDir) {
58
- console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir> [depot key]");
59
- process.exit(1);
60
- }
61
-
62
- const ContentManifest = require("steam-user/components/content_manifest");
63
- const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
64
-
65
- (async () => {
66
- const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
67
- if (!depotKey) {
68
- console.log(`Depot key was not supplied, attempting to fetch it...`);
69
- depotKey = await fetchDepotKey(manifest.depot_id);
70
- }
71
- depotKey = Buffer.from(depotKey, "hex");
72
- await populateChunks(manifest, depotKey, installDir, (file, file_i, num_files, chunk_i, num_chunks) => {
73
- if (chunk_i == 0) {
74
- console.log(`Processing ${file}...`);
75
- }
76
- });
77
- process.exit(0);
78
- })();
79
- } break;
80
-
81
- case "lancache": {
82
- const fs = require("fs");
83
- const fsPromises = require("fs/promises");
84
- const path = require("path");
85
- const express = require("express");
86
- const net = require("net");
87
- const sni = require("sni");
88
-
89
- const app = express();
90
-
91
- app.use((req, res, next) => {
92
- res.on("finish", () => {
93
- console.log(`${req.socket.remoteAddress} - ${req.url} - ${res.statusCode}`);
94
- });
95
- next();
96
- });
97
-
98
- app.get("/depot/:depotId/chunk/:sha", async (req, res) => {
99
- if (fs.existsSync(__dirname + req.url)) {
100
- res.sendFile(__dirname + req.url);
101
- }
102
- else if (req.host.endsWith(".steamcontent.com")) {
103
- const fr = await fetch(`http://${req.host}${req.url}`);
104
- if (fr.status == 200) {
105
- const buf = Buffer.from(await fr.arrayBuffer());
106
- await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
107
- await fsPromises.writeFile(__dirname + req.url, buf);
108
- res.send(buf).end();
109
- }
110
- else {
111
- res.status(fr.status).send("Upstream error").end();
112
- }
113
- }
114
- else {
115
- res.status(400).send("Don't have this chunk").end();
116
- }
117
- });
118
-
119
- app.use((req, res) => {
120
- res.status(400).send("Don't know this endpoint").end();
121
- });
122
-
123
- app.listen(80, () => {
124
- console.log("Listening on port 80");
125
- });
126
-
127
- // Transparently forward traffic on port 443 to avoid breaking HTTPS connections
128
-
129
- net.createServer(socket => {
130
- socket.once("data", firstPacket => {
131
- const hostname = sni(firstPacket);
132
- if (!hostname) {
133
- return;
134
- }
135
- console.log(`${socket.remoteAddress} - Starting TLS proxy to ${hostname}`);
136
- const upstream = net.connect(443, hostname, () => {
137
- upstream.write(firstPacket);
138
- socket.pipe(upstream).pipe(socket);
139
- });
140
- upstream.on("error", () => socket.end());
141
- });
142
- socket.on("error", () => {});
143
- }).listen(443, () => {
144
- console.log("Listening on port 443");
145
- });
146
- } break;
147
-
148
- case "install": {
149
- const depotId = process.argv[3];
150
- const manifestId = process.argv[4];
151
- let depotKey = process.argv[5];
152
- if (!depotId || !manifestId) {
153
- console.log("Syntax: mango install <depot id> <manifest id> [depot key]");
154
- process.exit(1);
155
- }
156
-
157
- const ContentManifest = require("steam-user/components/content_manifest");
158
- const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
159
-
160
- (async () => {
161
- const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
162
- if (!depotKey) {
163
- console.log(`Depot key was not supplied, attempting to fetch it...`);
164
- depotKey = await fetchDepotKey(manifest.depot_id);
165
- }
166
- depotKey = Buffer.from(depotKey, "hex");
167
- await install(manifest, depotKey, undefined, (file, existed) => {
168
- console.log(`${existed ? "Repaired" : "Created"} ${file}`);
169
- });
170
- process.exit(0);
171
- })();
172
- } break;
173
-
174
- case "to-json": {
175
- const depotId = process.argv[3];
176
- const manifestId = process.argv[4];
177
- let depotKey = process.argv[5];
178
- if (!depotId || !manifestId) {
179
- console.log("Syntax: mango to-json <depot id> <manifest id> [depot key|--no-decrypt]");
180
- process.exit(1);
181
- }
182
-
183
- const fs = require("fs");
184
- const ContentManifest = require("steam-user/components/content_manifest");
185
- const { fetchManifest, fetchDepotKey } = require("./lib.js");
186
-
187
- (async () => {
188
- const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
189
- if (manifest.filenames_encrypted && depotKey != "--no-decrypt") {
190
- if (!depotKey) {
191
- console.log("Manifest has encrypted filenames and --no-decrypt was not specified. A depot key will be needed.");
192
- console.log(`Depot key was not supplied, attempting to fetch it...`);
193
- depotKey = await fetchDepotKey(manifest.depot_id);
194
- }
195
- ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex")); // Sets filenames_encrypted to false
196
- //manifest.filenames_decrypted = true; // Indicate that this transformation took place in the JSON export // Kinda pointless because some .manifest files are already decrypted by their sources
197
- }
198
- fs.writeFileSync(`${depotId}_${manifestId}.json`, JSON.stringify(manifest, null, 2));
199
- console.log(`Saved in ${depotId}_${manifestId}.json`);
200
-
201
- process.exit(0);
202
- })();
203
- } break;
204
-
205
- case "to-hashdeep-auditfile": {
206
- const depotId = process.argv[3];
207
- const manifestId = process.argv[4];
208
- let depotKey = process.argv[5];
209
- if (!depotId || !manifestId) {
210
- console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id> [depot key]");
211
- process.exit(1);
212
- }
213
-
214
- const fs = require("fs");
215
- const path = require("path");
216
- const ContentManifest = require("steam-user/components/content_manifest");
217
- const { fetchManifest, fetchDepotKey } = require("./lib.js");
218
-
219
- (async () => {
220
- const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
221
- if (manifest.filenames_encrypted) {
222
- if (!depotKey) {
223
- console.log("Manifest has encrypted filenames. A depot key will be needed.");
224
- console.log(`Depot key was not supplied, attempting to fetch it...`);
225
- depotKey = await fetchDepotKey(manifest.depot_id);
226
- }
227
- ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex"));
228
- }
229
-
230
- const fh = fs.createWriteStream(`${depotId}_${manifestId}.auditfile`);
231
- fh.write("%%%% HASHDEEP-1.0\n");
232
- fh.write("%%%% size,sha1,filename\n");
233
- for (const file of manifest.files) {
234
- if ((file.flags & 64) === 0) {
235
- fh.write(`${file.size},${file.sha_content},${manifest.gid_manifest}${path.sep}${path.sep != '\\' ? file.filename.split('\\').join(path.sep) : file.filename.split('/').join(path.sep)}\n`);
236
- }
237
- }
238
- fh.end();
239
- fh.on("finish", () => {
240
- console.log(`Saved in ${depotId}_${manifestId}.auditfile`);
241
-
242
- process.exit(0);
243
- });
244
- })();
245
- } break;
246
-
247
- case "to-torrent": {
248
- const depotId = process.argv[3];
249
- const manifestId = process.argv[4];
250
- const fileHash = process.argv[5];
251
- if (!depotId || !manifestId || !fileHash) {
252
- console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
253
- process.exit(1);
254
- }
255
-
256
- const ContentManifest = require("steam-user/components/content_manifest");
257
-
258
- function toPieces(hexArray) {
259
- const bufs = hexArray.map(h => {
260
- if (typeof h !== "string" || h.length !== 40) {
261
- throw new Error(`Invalid SHA1 hex string: ${h}`);
262
- }
263
- return Buffer.from(h, "hex");
264
- });
265
- return Buffer.concat(bufs);
266
- }
267
-
268
- (async () => {
269
- const { fetchManifest } = require("./lib.js");
270
- const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
271
- const file = manifest.files.find(x => x.sha_content == fileHash);
272
- if (!file) {
273
- console.log(`No file with content hash ${fileHash} found in ${depotId}_${manifestId}.manifest`);
274
- process.exit(1);
275
- }
276
- for (const chunk of file.chunks) {
277
- chunk.offset = parseInt(chunk.offset);
278
- }
279
- file.chunks.sort((a, b) => a.offset - b.offset);
280
- let pieceSize = file.chunks[0].cb_original;
281
- for (let i = 1; i < file.chunks.length - 1; ++i) {
282
- if (pieceSize != file.chunks[i].cb_original) {
283
- console.log(`This file has non-uniformly sized chunks and is therefore unrepresentable in BitTorrent.`);
284
- process.exit(2);
285
- }
286
- }
287
- const { toTorrentFile } = await import("parse-torrent");
288
- const buf = toTorrentFile({
289
- info: {
290
- name: file.sha_content,
291
- length: parseInt(file.size),
292
- 'piece length': pieceSize,
293
- pieces: toPieces(file.chunks.map(chunk => chunk.sha))
294
- }
295
- });
296
- const fs = require("node:fs");
297
- fs.writeFileSync(`${file.sha_content}.torrent`, buf);
298
- console.log(`Saved in ${file.sha_content}.torrent`);
299
- process.exit(0);
300
- })();
301
- } break;
302
-
303
- case "verify-chunks": {
304
- const depotId = process.argv[3];
305
- let depotKey = process.argv[4];
306
- if (!depotId) {
307
- console.log("Syntax: mango verify-chunks <depot id> [depot key]");
308
- process.exit(1);
309
- }
310
-
311
- const { fetchDepotKey, verifyChunks } = require("./lib.js");
312
- (async () => {
313
- if (!depotKey) {
314
- console.log(`Depot key was not supplied, attempting to fetch it...`);
315
- depotKey = await fetchDepotKey(depotId);
316
- }
317
- depotKey = Buffer.from(depotKey, "hex");
318
- console.log(`Verifying chunks...`);
319
- await verifyChunks(depotId, depotKey, (file, expectedHash) => {
320
- console.log(`Deleted ${file}`);
321
- });
322
- process.exit(0);
323
- })();
324
- } break;
17
+ default: {
18
+ console.error(tool ? `Unknown tool: ${tool}` : "Syntax: mango <tool>");
19
+ process.exit(1);
20
+ } break;
21
+
22
+ case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
23
+ const depotId = process.argv[3];
24
+ const manifestId = process.argv[4];
25
+ if (!depotId || !manifestId) {
26
+ console.log("Syntax: mango download-manifest <depot id> <manifest id>");
27
+ process.exit(1);
28
+ }
29
+
30
+ const { fetchManifest } = require("./lib.js");
31
+
32
+ (async () => {
33
+ await fetchManifest(depotId, manifestId);
34
+ console.log(`Saved in manifests/${depotId}_${manifestId}.manifest`);
35
+ })();
36
+ } break;
37
+
38
+ case "download-chunks": {
39
+ const args = parseArguments(3, ["--lancache"]);
40
+ const [depotId, manifestId] = args;
41
+ if (!depotId || !manifestId) {
42
+ console.log("Syntax: mango download-chunks <depot id> <manifest id> [--lancache]");
43
+ process.exit(1);
44
+ }
45
+
46
+ const ContentManifest = require("steam-user/components/content_manifest");
47
+ const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
48
+
49
+ const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
50
+
51
+ (async () => {
52
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
53
+ await downloadChunks(
54
+ manifest,
55
+ undefined /*(num_chunks) => {}*/,
56
+ (path, host) => { console.log(`${path}: Downloading from ${host}`); },
57
+ (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
58
+ (path, err) => { console.log(`${path}: `, err); },
59
+ hosts
60
+ )
61
+ process.exit(0);
62
+ })();
63
+ } break;
64
+
65
+ case "populate-chunks": {
66
+ const depotId = process.argv[3];
67
+ const manifestId = process.argv[4];
68
+ const installDir = process.argv[5];
69
+ if (!depotId || !manifestId || !installDir) {
70
+ console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir>");
71
+ process.exit(1);
72
+ }
73
+
74
+ const ContentManifest = require("steam-user/components/content_manifest");
75
+ const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
76
+
77
+ (async () => {
78
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
79
+ console.log(`Fetching depot key...`);
80
+ let depotKey = await fetchDepotKey(manifest.depot_id);
81
+ depotKey = Buffer.from(depotKey, "hex");
82
+ await populateChunks(manifest, depotKey, installDir, (file, file_i, num_files, chunk_i, num_chunks) => {
83
+ if (chunk_i == 0) {
84
+ console.log(`Processing ${file}...`);
85
+ }
86
+ });
87
+ process.exit(0);
88
+ })();
89
+ } break;
90
+
91
+ case "lancache": {
92
+ const fs = require("fs");
93
+ const fsPromises = require("fs/promises");
94
+ const path = require("path");
95
+ const express = require("express");
96
+ const net = require("net");
97
+ const sni = require("sni");
98
+
99
+ const app = express();
100
+
101
+ app.use((req, res, next) => {
102
+ res.on("finish", () => {
103
+ console.log(`${req.socket.remoteAddress} - ${req.url} - ${res.statusCode}`);
104
+ });
105
+ next();
106
+ });
107
+
108
+ app.get("/depot/:depotId/chunk/:sha", async (req, res) => {
109
+ if (fs.existsSync(__dirname + req.url)) {
110
+ res.sendFile(__dirname + req.url);
111
+ }
112
+ else if (req.host.endsWith(".steamcontent.com")) {
113
+ const fr = await fetch(`http://${req.host}${req.url}`);
114
+ if (fr.status == 200) {
115
+ const buf = Buffer.from(await fr.arrayBuffer());
116
+ await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
117
+ await fsPromises.writeFile(__dirname + req.url, buf);
118
+ res.send(buf).end();
119
+ }
120
+ else {
121
+ res.status(fr.status).send("Upstream error").end();
122
+ }
123
+ }
124
+ else {
125
+ res.status(400).send("Don't have this chunk").end();
126
+ }
127
+ });
128
+
129
+ app.get("/depot/:depotId/patch/:oldManifestId/:newManifestId", async (req, res) => {
130
+ if (fs.existsSync(__dirname + req.url)) {
131
+ res.sendFile(__dirname + req.url);
132
+ }
133
+ else if (req.host.endsWith(".steamcontent.com")) {
134
+ const fr = await fetch(`http://${req.host}${req.url}`);
135
+ if (fr.status == 200) {
136
+ const buf = Buffer.from(await fr.arrayBuffer());
137
+ //await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
138
+ //await fsPromises.writeFile(__dirname + req.url, buf);
139
+ res.send(buf).end();
140
+ }
141
+ else {
142
+ res.status(fr.status).send("Upstream error").end();
143
+ }
144
+ }
145
+ else {
146
+ res.status(400).send("Don't have this patch").end();
147
+ }
148
+ });
149
+
150
+ app.use((req, res) => {
151
+ res.status(400).send("Don't know this endpoint").end();
152
+ });
153
+
154
+ app.listen(80, () => {
155
+ console.log("Listening on port 80");
156
+ });
157
+
158
+ // Transparently forward traffic on port 443 to avoid breaking HTTPS connections
159
+
160
+ net.createServer(socket => {
161
+ socket.once("data", firstPacket => {
162
+ const hostname = sni(firstPacket);
163
+ if (!hostname) {
164
+ return;
165
+ }
166
+ console.log(`${socket.remoteAddress} - Starting TLS proxy to ${hostname}`);
167
+ const upstream = net.connect(443, hostname, () => {
168
+ upstream.write(firstPacket);
169
+ socket.pipe(upstream).pipe(socket);
170
+ });
171
+ upstream.on("error", () => socket.end());
172
+ });
173
+ socket.on("error", () => {});
174
+ }).listen(443, () => {
175
+ console.log("Listening on port 443");
176
+ });
177
+ } break;
178
+
179
+ case "install": {
180
+ const depotId = process.argv[3];
181
+ const manifestId = process.argv[4];
182
+ const installDir = process.argv[5];
183
+ if (!depotId || !manifestId) {
184
+ console.log("Syntax: mango install <depot id> <manifest id> [install dir]");
185
+ process.exit(1);
186
+ }
187
+
188
+ const ContentManifest = require("steam-user/components/content_manifest");
189
+ const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
190
+
191
+ (async () => {
192
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
193
+ console.log(`Fetching depot key...`);
194
+ let depotKey = await fetchDepotKey(manifest.depot_id);
195
+ depotKey = Buffer.from(depotKey, "hex");
196
+ await install(manifest, depotKey, installDir, (file, existed) => {
197
+ console.log(`${existed ? "Repaired" : "Created"} ${file}`);
198
+ });
199
+ process.exit(0);
200
+ })();
201
+ } break;
202
+
203
+ case "download-and-install": {
204
+ const args = parseArguments(3, ["--lancache"]);
205
+ const [depotId, manifestId, installDir] = args;
206
+ if (!depotId || !manifestId) {
207
+ console.log("Syntax: mango download-and-install <depot id> <manifest id> [install dir] [--lancache]");
208
+ process.exit(1);
209
+ }
210
+
211
+ const ContentManifest = require("steam-user/components/content_manifest");
212
+ const { DEFAULT_HOSTS, fetchManifest, fetchDepotKey, downloadAndInstall } = require("./lib.js");
213
+
214
+ const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
215
+
216
+ (async () => {
217
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
218
+
219
+ console.log(`Fetching depot key...`);
220
+ let depotKey = await fetchDepotKey(manifest.depot_id);
221
+ depotKey = Buffer.from(depotKey, "hex");
222
+
223
+ await downloadAndInstall(
224
+ manifest,
225
+ depotKey,
226
+ undefined /*(num_chunks) => {}*/,
227
+ (path, host) => { console.log(`${path}: Downloading from ${host}`); },
228
+ (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
229
+ (path, err) => { console.log(`${path}: `, err); },
230
+ hosts,
231
+ installDir
232
+ );
233
+
234
+ process.exit(0);
235
+ })();
236
+ } break;
237
+
238
+ case "to-json": {
239
+ const args = parseArguments(3, ["--no-decrypt"]);
240
+ const [depotId, manifestId] = args;
241
+ if (!depotId || !manifestId) {
242
+ console.log("Syntax: mango to-json <depot id> <manifest id> [--no-decrypt]");
243
+ process.exit(1);
244
+ }
245
+
246
+ const fs = require("fs");
247
+ const ContentManifest = require("steam-user/components/content_manifest");
248
+ const { fetchManifest, fetchDepotKey } = require("./lib.js");
249
+
250
+ (async () => {
251
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
252
+ if (manifest.filenames_encrypted && !args["--no-decrypt"]) {
253
+ console.log("Fetching depot key to decrypt filenames...");
254
+ const depotKey = await fetchDepotKey(manifest.depot_id);
255
+ ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex")); // Sets filenames_encrypted to false
256
+ //manifest.filenames_decrypted = true; // Indicate that this transformation took place in the JSON export // Kinda pointless because some .manifest files are already decrypted by their sources
257
+ }
258
+ fs.writeFileSync(`${depotId}_${manifestId}.json`, JSON.stringify(manifest, null, 2));
259
+ console.log(`Saved in ${depotId}_${manifestId}.json`);
260
+
261
+ process.exit(0);
262
+ })();
263
+ } break;
264
+
265
+ case "to-hashdeep-auditfile": {
266
+ const depotId = process.argv[3];
267
+ const manifestId = process.argv[4];
268
+ if (!depotId || !manifestId) {
269
+ console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id>");
270
+ process.exit(1);
271
+ }
272
+
273
+ const fs = require("fs");
274
+ const path = require("path");
275
+ const ContentManifest = require("steam-user/components/content_manifest");
276
+ const { fetchManifest, fetchDepotKey } = require("./lib.js");
277
+
278
+ (async () => {
279
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
280
+ if (manifest.filenames_encrypted) {
281
+ console.log("Manifest has encrypted filenames. Fetching depot key...");
282
+ const depotKey = await fetchDepotKey(manifest.depot_id);
283
+ ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex"));
284
+ }
285
+
286
+ const fh = fs.createWriteStream(`${depotId}_${manifestId}.auditfile`);
287
+ fh.write("%%%% HASHDEEP-1.0\n");
288
+ fh.write("%%%% size,sha1,filename\n");
289
+ for (const file of manifest.files) {
290
+ if ((file.flags & 64) === 0) {
291
+ fh.write(`${file.size},${file.sha_content},${manifest.gid_manifest}${path.sep}${path.sep != '\\' ? file.filename.split('\\').join(path.sep) : file.filename.split('/').join(path.sep)}\n`);
292
+ }
293
+ }
294
+ fh.end();
295
+ fh.on("finish", () => {
296
+ console.log(`Saved in ${depotId}_${manifestId}.auditfile`);
297
+
298
+ process.exit(0);
299
+ });
300
+ })();
301
+ } break;
302
+
303
+ case "to-torrent": {
304
+ const depotId = process.argv[3];
305
+ const manifestId = process.argv[4];
306
+ const fileHash = process.argv[5];
307
+ if (!depotId || !manifestId || !fileHash) {
308
+ console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
309
+ process.exit(1);
310
+ }
311
+
312
+ const ContentManifest = require("steam-user/components/content_manifest");
313
+
314
+ function toPieces(hexArray) {
315
+ const bufs = hexArray.map(h => {
316
+ if (typeof h !== "string" || h.length !== 40) {
317
+ throw new Error(`Invalid SHA1 hex string: ${h}`);
318
+ }
319
+ return Buffer.from(h, "hex");
320
+ });
321
+ return Buffer.concat(bufs);
322
+ }
323
+
324
+ (async () => {
325
+ const { fetchManifest } = require("./lib.js");
326
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
327
+ const file = manifest.files.find(x => x.sha_content == fileHash);
328
+ if (!file) {
329
+ console.log(`No file with content hash ${fileHash} found in ${depotId}_${manifestId}.manifest`);
330
+ process.exit(1);
331
+ }
332
+ for (const chunk of file.chunks) {
333
+ chunk.offset = parseInt(chunk.offset);
334
+ }
335
+ file.chunks.sort((a, b) => a.offset - b.offset);
336
+ let pieceSize = file.chunks[0].cb_original;
337
+ for (let i = 1; i < file.chunks.length - 1; ++i) {
338
+ if (pieceSize != file.chunks[i].cb_original) {
339
+ console.log(`This file has non-uniformly sized chunks and is therefore unrepresentable in BitTorrent.`);
340
+ process.exit(2);
341
+ }
342
+ }
343
+ const { toTorrentFile } = await import("parse-torrent");
344
+ const buf = toTorrentFile({
345
+ info: {
346
+ name: file.sha_content,
347
+ length: parseInt(file.size),
348
+ 'piece length': pieceSize,
349
+ pieces: toPieces(file.chunks.map(chunk => chunk.sha))
350
+ }
351
+ });
352
+ const fs = require("node:fs");
353
+ fs.writeFileSync(`${file.sha_content}.torrent`, buf);
354
+ console.log(`Saved in ${file.sha_content}.torrent`);
355
+ process.exit(0);
356
+ })();
357
+ } break;
358
+
359
+ case "verify-chunks": {
360
+ const depotId = process.argv[3];
361
+ if (!depotId) {
362
+ console.log("Syntax: mango verify-chunks <depot id>");
363
+ process.exit(1);
364
+ }
365
+
366
+ const { fetchDepotKey, verifyChunks } = require("./lib.js");
367
+ (async () => {
368
+ console.log(`Fetching depot key...`);
369
+ let depotKey = await fetchDepotKey(depotId);
370
+ depotKey = Buffer.from(depotKey, "hex");
371
+ console.log(`Verifying chunks...`);
372
+ await verifyChunks(depotId, depotKey, (file, expectedHash) => {
373
+ console.log(`Deleted ${file}`);
374
+ });
375
+ process.exit(0);
376
+ })();
377
+ } break;
325
378
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "steam-manifest-tools",
3
- "version": "0.1.5",
3
+ "version": "0.1.7",
4
4
  "main": "lib.js",
5
5
  "repository": {
6
6
  "type": "git",