steam-manifest-tools 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +11 -10
  2. package/lib.js +169 -72
  3. package/mango.js +375 -309
  4. package/package.json +2 -2
package/README.md CHANGED
@@ -1,14 +1,15 @@
1
- Tools to interact with Steam manifest files.
2
- - `mango download-manifest <depot id> <manifest id>`
3
- - `mango download-chunks <manifest file> [--lancache]` (basically the same as prefilling a LAN cache, optionally downloads from a LAN cache)
4
- - `mango populate-chunks <manifest file> <install dir> [depot key]` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
1
+ Tools to interact with Steam depots and manifests.
2
+ - `mango download-chunks <depot id> <manifest id> [--lancache]` (basically the same as prefilling a LAN cache, optionally downloads from a LAN cache)
3
+ - `mango populate-chunks <depot id> <manifest id> <install dir>` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
5
4
  - `mango lancache`
6
5
  - Must run as root to obtain port 80
7
6
  - `lancache.steamcontent.com` must resolve to your LAN cache server IP (either via custom DNS server or OS hosts file)
8
- - `mango install <manifest file> [depot key]` (requires chunks to have been populated and/or downloaded beforehand)
9
- - `mango to-json <manifest file> [depot key]`
10
- - `mango to-hashdeep-auditfile <manifest file> [depot key]`
7
+ - `mango install <depot id> <manifest id> [install dir]` (requires chunks to have been populated and/or downloaded beforehand)
8
+ - `mango download-and-install <depot id> <manifest id> [install dir] [--lancache]`
9
+ - `mango to-json <depot id> <manifest id> [--no-decrypt]`
10
+ - `mango to-hashdeep-auditfile <depot id> <manifest id>`
11
11
  - Note that hashdeep requires exact path matches and this script makes a few assumptions
12
- - Example verification usage: `hashdeep -a -l -v -v -k 241561_8497448424664183398.manifest.auditfile -r 8497448424664183398`
13
- - `mango to-torrent <manifest file> <file hash>`
14
- - `mango verify-chunks <depot id> [depot key]`
12
+ - Example verification usage: `hashdeep -a -l -v -v -k 241561_8497448424664183398.auditfile -r 8497448424664183398`
13
+ - `mango to-torrent <depot id> <manifest id> <file hash>`
14
+ - `mango verify-chunks <depot id>`
15
+ - `mango download-manifest <depot id> <manifest id>`
package/lib.js CHANGED
@@ -92,6 +92,73 @@ const getChunk = async (depotId, depotKey, hash) => {
92
92
  }
93
93
  };
94
94
 
95
+ const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
96
+ hosts ??= DEFAULT_HOSTS;
97
+ return new Promise(resolve => {
98
+ fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
99
+ const toDownload = {};
100
+ const alreadyDownloaded = [];
101
+ for (const file of manifest.files) {
102
+ for (const chunk of file.chunks) {
103
+ if (fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
104
+ alreadyDownloaded.push(`depot/${manifest.depot_id}/chunk/${chunk.sha}`);
105
+ }
106
+ else {
107
+ toDownload[`depot/${manifest.depot_id}/chunk/${chunk.sha}`] = true;
108
+ }
109
+ }
110
+ }
111
+ let remaining_chunks = Object.keys(toDownload).length;
112
+ if (onStartDownloading) {
113
+ onStartDownloading(remaining_chunks, alreadyDownloaded);
114
+ }
115
+ if (remaining_chunks == 0) {
116
+ resolve();
117
+ }
118
+ let /*host_i = 0,*/ running = 0;
119
+ const concurrency = hosts.length == 1 ? 4 : (hosts.length * 2);
120
+ const loop = () => {
121
+ while (running < concurrency) {
122
+ const path = Object.keys(toDownload)[0];
123
+ if (!path) {
124
+ break;
125
+ }
126
+ ++running;
127
+ delete toDownload[path];
128
+ const host = hosts[Math.floor(Math.random()*hosts.length)];
129
+ //const host = hosts[host_i]; host_i = (host_i + 1) % hosts.length;
130
+ if (onStartDownload) {
131
+ onStartDownload(path, host);
132
+ }
133
+ fetch(`${host}/${path}`).then(async res => {
134
+ if (res.status == 200) {
135
+ const ab = await res.arrayBuffer();
136
+ await fsPromises.writeFile(path, Buffer.from(ab));
137
+ if (--remaining_chunks == 0) {
138
+ resolve();
139
+ }
140
+ }
141
+ else {
142
+ toDownload[path] = true;
143
+ }
144
+ if (onFinishDownload) {
145
+ onFinishDownload(path, res.status, host);
146
+ }
147
+ }).catch(err => {
148
+ if (onErroredDownload) {
149
+ onErroredDownload(path, err);
150
+ }
151
+ toDownload[path] = true;
152
+ }).finally(() => {
153
+ --running;
154
+ loop();
155
+ });
156
+ }
157
+ };
158
+ loop();
159
+ });
160
+ };
161
+
95
162
  module.exports = {
96
163
  DEFAULT_HOSTS,
97
164
  sha1,
@@ -101,33 +168,49 @@ module.exports = {
101
168
  getFileContents,
102
169
  getChunk,
103
170
  fetchManifest: async (depotId, manifestId) => {
171
+ try {
172
+ return await fsPromises.readFile(`manifests/${depotId}_${manifestId}.manifest`);
173
+ }
174
+ catch (e) {
175
+ // fallthrough
176
+ }
104
177
  const forkers = [
105
178
  "qwe213312",
106
179
  "mejikuhibiniu1",
107
180
  "Sainan",
108
- "FreakyObservatory",
181
+ //"FreakyObservatory", // Seems inactive now and seems to have no manifests missing in the above.
109
182
  ];
110
183
  for (const forker of forkers) {
111
184
  const res = await fetch(`https://raw.githubusercontent.com/${forker}/k25FCdfEOoEJ42S6/refs/heads/main/${depotId}_${manifestId}.manifest`);
112
185
  if (res.status == 200) {
113
- return await res.arrayBuffer();
186
+ const ab = await res.arrayBuffer();
187
+ await fsPromises.mkdir(`manifests`, { recursive: true });
188
+ await fsPromises.writeFile(`manifests/${depotId}_${manifestId}.manifest`, Buffer.from(ab));
189
+ return ab;
114
190
  }
115
191
  }
192
+ throw new Error(`Could not find the given manifest (${manifestId}). Double-check with https://steamdb.info/depot/${depotId}/manifests/ and report an issue in https://github.com/Sainan/k25FCdfEOoEJ42S6/issues if you're sure the manifest exists.`);
116
193
  },
117
194
  fetchDepotKey: async (depotId) => {
118
- console.log(`Depot key was not supplied, attempting to fetch it...`);
195
+ try {
196
+ return await fsPromises.readFile(`depot/${depotId}/key.txt`, "utf-8");
197
+ }
198
+ catch (e) {
199
+ // fallthrough
200
+ }
119
201
  const depotkeys = await fetch("https://raw.githubusercontent.com/SteamAutoCracks/ManifestHub/refs/heads/main/depotkeys.json").then(x => x.json());
120
202
  if (!depotkeys[depotId]) {
121
- console.log(`Failed to get the depot key. Please provide it manually.`);
122
- process.exit(1);
203
+ throw new Error(`Failed to get the depot key.`);
123
204
  }
205
+ await fsPromises.mkdir(`depot/${depotId}`, { recursive: true });
206
+ fsPromises.writeFile(`depot/${depotId}/key.txt`, depotkeys[depotId], "utf-8");
124
207
  return depotkeys[depotId];
125
208
  },
126
209
  populateChunks: async (manifest, depotKey, installDir, onProgress) => {
127
210
  if (manifest.filenames_encrypted) {
128
211
  ContentManifest.decryptFilenames(manifest, depotKey);
129
212
  }
130
- fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
213
+ await fsPromises.mkdir(`depot/${manifest.depot_id}/chunk`, { recursive: true });
131
214
  let file_i = 0;
132
215
  for (const file of manifest.files) {
133
216
  if (file.flags & 64) {
@@ -164,79 +247,19 @@ module.exports = {
164
247
  ++file_i;
165
248
  }
166
249
  },
167
- downloadChunks: (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
168
- hosts ??= DEFAULT_HOSTS;
169
- return new Promise(resolve => {
170
- fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
171
- const toDownload = {};
172
- for (const file of manifest.files) {
173
- for (const chunk of file.chunks) {
174
- if (!fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
175
- toDownload[`depot/${manifest.depot_id}/chunk/${chunk.sha}`] = true;
176
- }
177
- }
178
- }
179
- let remaining_chunks = Object.keys(toDownload).length;
180
- if (onStartDownloading) {
181
- onStartDownloading(remaining_chunks);
182
- }
183
- if (remaining_chunks == 0) {
184
- resolve();
185
- }
186
- let /*host_i = 0,*/ running = 0;
187
- const concurrency = hosts.length == 1 ? 4 : (hosts.length * 2);
188
- const loop = () => {
189
- while (running < concurrency) {
190
- const path = Object.keys(toDownload)[0];
191
- if (!path) {
192
- break;
193
- }
194
- ++running;
195
- delete toDownload[path];
196
- const host = hosts[Math.floor(Math.random()*hosts.length)];
197
- //const host = hosts[host_i]; host_i = (host_i + 1) % hosts.length;
198
- if (onStartDownload) {
199
- onStartDownload(path, host);
200
- }
201
- fetch(`${host}/${path}`).then(async res => {
202
- if (onFinishDownload) {
203
- onFinishDownload(path, res.status, host);
204
- }
205
- if (res.status == 200) {
206
- const ab = await res.arrayBuffer();
207
- await fsPromises.writeFile(path, Buffer.from(ab));
208
- if (--remaining_chunks == 0) {
209
- resolve();
210
- }
211
- }
212
- else {
213
- toDownload[path] = true;
214
- }
215
- }).catch(err => {
216
- if (onErroredDownload) {
217
- onErroredDownload(path, err);
218
- }
219
- toDownload[path] = true;
220
- }).finally(() => {
221
- --running;
222
- loop();
223
- });
224
- }
225
- };
226
- loop();
227
- });
228
- },
250
+ downloadChunks,
229
251
  install: async (manifest, depotKey, installDir, onFileWritten) => {
230
252
  ContentManifest.decryptFilenames(manifest, depotKey);
231
253
  installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
232
- fs.mkdirSync(installDir, { recursive: true });
254
+ await fsPromises.mkdir(installDir, { recursive: true });
233
255
  const promises = [];
234
256
  for (const file of manifest.files) {
235
- if (file.flags & 64) {
236
- continue;
237
- }
238
257
  promises.push((async () => {
239
258
  const filename = file.filename.replace(/\\/g, "/");
259
+ if (file.flags & 64) {
260
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
261
+ return;
262
+ }
240
263
  const exists = fs.existsSync(path.join(installDir, filename));
241
264
  if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
242
265
  for (const chunk of file.chunks) {
@@ -258,6 +281,80 @@ module.exports = {
258
281
  }
259
282
  await Promise.all(promises);
260
283
  },
284
+ downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
285
+ const depotId = manifest.depot_id;
286
+
287
+ ContentManifest.decryptFilenames(manifest, depotKey);
288
+ installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
289
+
290
+ const writeStreams = {};
291
+ for (const file of manifest.files) {
292
+ const filename = file.filename.replace(/\\/g, "/");
293
+ if (file.flags & 64) {
294
+ await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
295
+ }
296
+ else {
297
+ await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
298
+ writeStreams[file.filename] = await fsPromises.open(path.join(installDir, filename), "w");
299
+ }
300
+ }
301
+
302
+ const ioQueue = [];
303
+ let ioLoopRunning = false;
304
+ let ioLoopPromise;
305
+ const ioLoop = async () => {
306
+ ioLoopRunning = true;
307
+ while (ioQueue.length > 0) {
308
+ const chunkSha = ioQueue.shift();
309
+ const data = await getChunk(depotId, depotKey, chunkSha);
310
+ for (const file of manifest.files) {
311
+ for (const chunk of file.chunks) {
312
+ if (chunk.sha == chunkSha) {
313
+ await writeStreams[file.filename].write(data, 0, data.byteLength, parseInt(chunk.offset));
314
+ }
315
+ }
316
+ }
317
+ }
318
+ ioLoopRunning = false;
319
+ };
320
+ const ioAddPath = (path) => {
321
+ ioQueue.push(path.substr(path.length - 40));
322
+ if (!ioLoopRunning) {
323
+ ioLoopPromise = ioLoop();
324
+ }
325
+ };
326
+
327
+ await downloadChunks(
328
+ manifest,
329
+ (num_chunks, alreadyDownloaded) => {
330
+ for (const path of alreadyDownloaded) {
331
+ ioAddPath(path);
332
+ }
333
+ if (onStartDownloading) {
334
+ onStartDownloading(num_chunks);
335
+ }
336
+ },
337
+ onStartDownload,
338
+ (path, status, host) => {
339
+ if (status == 200) {
340
+ ioAddPath(path);
341
+ }
342
+ if (onFinishDownload) {
343
+ onFinishDownload(path, status, host);
344
+ }
345
+ },
346
+ onErroredDownload,
347
+ hosts
348
+ );
349
+
350
+ if (ioLoopRunning) {
351
+ await ioLoopPromise;
352
+ }
353
+
354
+ for (const writeStream of Object.values(writeStreams)) {
355
+ await writeStream.close();
356
+ }
357
+ },
261
358
  verifyChunks: async (depotId, depotKey, onDeletedFile) => {
262
359
  const files = await getFiles(`depot/${depotId}/chunk`);
263
360
  let promises = [];
package/mango.js CHANGED
@@ -1,312 +1,378 @@
1
+ const parseArguments = (base, bools = []) => {
2
+ const res = [];
3
+ let j = 0;
4
+ for (let i = base; i != process.argv.length; ++i) {
5
+ if (bools.indexOf(process.argv[i]) != -1) {
6
+ res[process.argv[i]] = true;
7
+ }
8
+ else {
9
+ res[j++] = process.argv[i];
10
+ }
11
+ }
12
+ return res;
13
+ };
14
+
1
15
  const tool = process.argv[2];
2
16
  switch (tool) {
3
- default: {
4
- console.error(tool ? `Unknown tool: ${tool}` : "Syntax: mango <tool>");
5
- process.exit(1);
6
- } break;
7
-
8
- case "download-manifest": {
9
- const depotId = process.argv[1 + 2];
10
- const manifestId = process.argv[1 + 3];
11
- if (!depotId || !manifestId) {
12
- console.log("Syntax: mango download-manifest <depot id> <manifest id>");
13
- process.exit(1);
14
- }
15
-
16
- const fsPromises = require("fs/promises");
17
- const { fetchManifest } = require(".");
18
-
19
- (async () => {
20
- const ab = await fetchManifest(depotId, manifestId);
21
- if (ab) {
22
- await fsPromises.writeFile(`${depotId}_${manifestId}.manifest`, Buffer.from(ab));
23
- console.log(`Saved in ${depotId}_${manifestId}.manifest`);
24
- process.exit(0);
25
- }
26
- else {
27
- console.log(`Could not find the given manifest. Double-check with https://steamdb.info/depot/${depotId}/manifests/ and report an issue in https://github.com/Sainan/k25FCdfEOoEJ42S6/issues if you're sure the manifest exists.`);
28
- process.exit(1);
29
- }
30
- })();
31
- } break;
32
-
33
- case "download-chunks": {
34
- let manifestFile = process.argv[1 + 2];
35
- let lancache = (process.argv[1 + 3] == "--lancache");
36
- if (manifestFile == "--lancache") {
37
- lancache = true;
38
- manifestFile = process.argv[1 + 3];
39
- }
40
- if (!manifestFile) {
41
- console.log("Syntax: mango download-chunks <manifest file> [--lancache]");
42
- process.exit(1);
43
- }
44
-
45
- const fs = require("fs");
46
- const ContentManifest = require("steam-user/components/content_manifest");
47
- const { DEFAULT_HOSTS, downloadChunks } = require(".");
48
-
49
- const hosts = lancache ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
50
-
51
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
52
- downloadChunks(
53
- manifest,
54
- undefined /*(num_chunks) => {}*/,
55
- (path, host) => { console.log(`${path}: Downloading from ${host}`); },
56
- (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
57
- (path, err) => { console.log(`${path}: `, err); },
58
- hosts
59
- ).then(() => process.exit(0));
60
- } break;
61
-
62
- case "populate-chunks": {
63
- const manifestFile = process.argv[1 + 2];
64
- const installDir = process.argv[1 + 3];
65
- let depotKey = process.argv[1 + 4];
66
- if (!manifestFile || !installDir) {
67
- console.log("Syntax: mango populate-chunks <manifest file> <install dir> [depot key]");
68
- process.exit(1);
69
- }
70
-
71
- const fs = require("fs");
72
- const ContentManifest = require("steam-user/components/content_manifest");
73
- const { fetchDepotKey, populateChunks } = require(".");
74
-
75
- (async () => {
76
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
77
- if (!depotKey) {
78
- depotKey = await fetchDepotKey(manifest.depot_id);
79
- }
80
- depotKey = Buffer.from(depotKey, "hex");
81
- await populateChunks(manifest, depotKey, installDir, (file, file_i, num_files, chunk_i, num_chunks) => {
82
- if (chunk_i == 0) {
83
- console.log(`Processing ${file}...`);
84
- }
85
- });
86
- process.exit(0);
87
- })();
88
- } break;
89
-
90
- case "lancache": {
91
- const fs = require("fs");
92
- const fsPromises = require("fs/promises");
93
- const path = require("path");
94
- const express = require("express");
95
- const net = require("net");
96
- const sni = require("sni");
97
-
98
- const app = express();
99
-
100
- app.use((req, res, next) => {
101
- res.on("finish", () => {
102
- console.log(`${req.socket.remoteAddress} - ${req.url} - ${res.statusCode}`);
103
- });
104
- next();
105
- });
106
-
107
- app.get("/depot/:depotId/chunk/:sha", async (req, res) => {
108
- if (fs.existsSync(__dirname + req.url)) {
109
- res.sendFile(__dirname + req.url);
110
- }
111
- else if (req.host.endsWith(".steamcontent.com")) {
112
- const fr = await fetch(`http://${req.host}${req.url}`);
113
- if (fr.status == 200) {
114
- const buf = Buffer.from(await fr.arrayBuffer());
115
- await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
116
- await fsPromises.writeFile(__dirname + req.url, buf);
117
- res.send(buf).end();
118
- }
119
- else {
120
- res.status(fr.status).send("Upstream error").end();
121
- }
122
- }
123
- else {
124
- res.status(400).send("Don't have this chunk").end();
125
- }
126
- });
127
-
128
- app.use((req, res) => {
129
- res.status(400).send("Don't know this endpoint").end();
130
- });
131
-
132
- app.listen(80, () => {
133
- console.log("Listening on port 80");
134
- });
135
-
136
- // Transparently forward traffic on port 443 to avoid breaking HTTPS connections
137
-
138
- net.createServer(socket => {
139
- socket.once("data", firstPacket => {
140
- const hostname = sni(firstPacket);
141
- if (!hostname) {
142
- return;
143
- }
144
- console.log(`${socket.remoteAddress} - Starting TLS proxy to ${hostname}`);
145
- const upstream = net.connect(443, hostname, () => {
146
- upstream.write(firstPacket);
147
- socket.pipe(upstream).pipe(socket);
148
- });
149
- upstream.on("error", () => socket.end());
150
- });
151
- socket.on("error", () => {});
152
- }).listen(443, () => {
153
- console.log("Listening on port 443");
154
- });
155
- } break;
156
-
157
- case "install": {
158
- const manifestFile = process.argv[1 + 2];
159
- let depotKey = process.argv[1 + 3];
160
- if (!manifestFile) {
161
- console.log("Syntax: mango install <manifest file> [depot key]");
162
- process.exit(1);
163
- }
164
-
165
- const fs = require("fs");
166
- const ContentManifest = require("steam-user/components/content_manifest");
167
- const { fetchDepotKey, install } = require(".");
168
-
169
- (async () => {
170
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
171
- if (!depotKey) {
172
- depotKey = await fetchDepotKey(manifest.depot_id);
173
- }
174
- depotKey = Buffer.from(depotKey, "hex");
175
- await install(manifest, depotKey, undefined, (file, existed) => {
176
- console.log(`${existed ? "Repaired" : "Created"} ${file}`);
177
- });
178
- process.exit(0);
179
- })();
180
- } break;
181
-
182
- case "to-json": {
183
- const fs = require("fs");
184
- const ContentManifest = require("steam-user/components/content_manifest");
185
-
186
- const file = process.argv[1 + 2];
187
- const depotKey = process.argv[1 + 3];
188
- if (!file) {
189
- console.log("Syntax: mango to-json <manifest file> [depot key]");
190
- process.exit(1);
191
- }
192
-
193
- const buf = fs.readFileSync(file);
194
- const manifest = ContentManifest.parse(buf);
195
- if (manifest.filenames_encrypted) {
196
- if (depotKey) {
197
- ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex")); // Sets filenames_encrypted to false
198
- //manifest.filenames_decrypted = true; // Indicate that this transformation took place in the JSON export // Kinda pointless because some .manifest files are already decrypted by their sources
199
- } else {
200
- console.log("Manifest has encrypted filenames, suggest supplying depot key");
201
- }
202
-
203
- }
204
- fs.writeFileSync(file + ".json", JSON.stringify(manifest, null, 2));
205
-
206
- process.exit(0);
207
- } break;
208
-
209
- case "to-hashdeep-auditfile": {
210
- const fs = require("fs");
211
- const path = require("path");
212
- const ContentManifest = require("steam-user/components/content_manifest");
213
-
214
- const file = process.argv[1 + 2];
215
- let depotKey = process.argv[1 + 3];
216
- if (!file) {
217
- console.log("Syntax: mango to-hashdeep-auditfile <manifest file> [depot key]");
218
- process.exit(1);
219
- }
220
-
221
- (async () => {
222
- const buf = fs.readFileSync(file);
223
- const manifest = ContentManifest.parse(buf);
224
- if (manifest.filenames_encrypted) {
225
- if (!depotKey) {
226
- console.log("Manifest has encrypted filenames. A depot key will be needed.");
227
- const { fetchDepotKey } = require(".");
228
- depotKey = await fetchDepotKey(manifest.depot_id);
229
- }
230
- ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex"));
231
- }
232
-
233
- const fh = fs.createWriteStream(`${file}.auditfile`);
234
- fh.write("%%%% HASHDEEP-1.0\n");
235
- fh.write("%%%% size,sha1,filename\n");
236
- for (const file of manifest.files) {
237
- if ((file.flags & 64) === 0) {
238
- fh.write(`${file.size},${file.sha_content},${manifest.gid_manifest}${path.sep}${path.sep != '\\' ? file.filename.split('\\').join(path.sep) : file.filename.split('/').join(path.sep)}\n`);
239
- }
240
- }
241
- fh.end();
242
-
243
- process.exit(0);
244
- })();
245
- } break;
246
-
247
- case "to-torrent": {
248
- const fs = require("fs");
249
- const ContentManifest = require("steam-user/components/content_manifest");
250
-
251
- function toPieces(hexArray) {
252
- const bufs = hexArray.map(h => {
253
- if (typeof h !== "string" || h.length !== 40) {
254
- throw new Error(`Invalid SHA1 hex string: ${h}`);
255
- }
256
- return Buffer.from(h, "hex");
257
- });
258
- return Buffer.concat(bufs);
259
- }
260
-
261
- const manifestFile = process.argv[1 + 2];
262
- const fileHash = process.argv[1 + 3];
263
- if (!manifestFile || !fileHash) {
264
- console.log("Syntax: mango to-torrent <manifest file> <file hash>");
265
- process.exit(1);
266
- }
267
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
268
- const file = manifest.files.find(x => x.sha_content == fileHash);
269
- if (!file) {
270
- console.log(`No file with content hash ${fileHash} found in ${manifestFile}`);
271
- process.exit(1);
272
- }
273
- import("parse-torrent").then(({ toTorrentFile }) => {
274
- for (const chunk of file.chunks) {
275
- chunk.offset = parseInt(chunk.offset);
276
- }
277
- file.chunks.sort((a, b) => a.offset - b.offset);
278
- const buf = toTorrentFile({
279
- info: {
280
- name: file.sha_content,
281
- length: parseInt(file.size),
282
- 'piece length': 1048576,
283
- pieces: toPieces(file.chunks.map(chunk => chunk.sha))
284
- }
285
- })
286
- fs.writeFileSync(`${file.sha_content}.torrent`, buf);
287
- process.exit(0);
288
- });
289
- } break;
290
-
291
- case "verify-chunks": {
292
- const depotId = process.argv[1 + 2];
293
- let depotKey = process.argv[1 + 3];
294
- if (!depotId) {
295
- console.log("Syntax: mango verify-chunks <depot id> [depot key]");
296
- process.exit(1);
297
- }
298
-
299
- const { fetchDepotKey, verifyChunks } = require(".");
300
- (async () => {
301
- if (!depotKey) {
302
- depotKey = await fetchDepotKey(depotId);
303
- }
304
- depotKey = Buffer.from(depotKey, "hex");
305
- console.log(`Verifying chunks...`);
306
- await verifyChunks(depotId, depotKey, (file, expectedHash) => {
307
- console.log(`Deleted ${file}`);
308
- });
309
- process.exit(0);
310
- })();
311
- } break;
17
+ default: {
18
+ console.error(tool ? `Unknown tool: ${tool}` : "Syntax: mango <tool>");
19
+ process.exit(1);
20
+ } break;
21
+
22
+ case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
23
+ const depotId = process.argv[3];
24
+ const manifestId = process.argv[4];
25
+ if (!depotId || !manifestId) {
26
+ console.log("Syntax: mango download-manifest <depot id> <manifest id>");
27
+ process.exit(1);
28
+ }
29
+
30
+ const { fetchManifest } = require("./lib.js");
31
+
32
+ (async () => {
33
+ await fetchManifest(depotId, manifestId);
34
+ console.log(`Saved in manifests/${depotId}_${manifestId}.manifest`);
35
+ })();
36
+ } break;
37
+
38
+ case "download-chunks": {
39
+ const args = parseArguments(3, ["--lancache"]);
40
+ const [depotId, manifestId] = args;
41
+ if (!depotId || !manifestId) {
42
+ console.log("Syntax: mango download-chunks <depot id> <manifest id> [--lancache]");
43
+ process.exit(1);
44
+ }
45
+
46
+ const ContentManifest = require("steam-user/components/content_manifest");
47
+ const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
48
+
49
+ const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
50
+
51
+ (async () => {
52
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
53
+ await downloadChunks(
54
+ manifest,
55
+ undefined /*(num_chunks) => {}*/,
56
+ (path, host) => { console.log(`${path}: Downloading from ${host}`); },
57
+ (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
58
+ (path, err) => { console.log(`${path}: `, err); },
59
+ hosts
60
+ )
61
+ process.exit(0);
62
+ })();
63
+ } break;
64
+
65
+ case "populate-chunks": {
66
+ const depotId = process.argv[3];
67
+ const manifestId = process.argv[4];
68
+ const installDir = process.argv[5];
69
+ if (!depotId || !manifestId || !installDir) {
70
+ console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir>");
71
+ process.exit(1);
72
+ }
73
+
74
+ const ContentManifest = require("steam-user/components/content_manifest");
75
+ const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
76
+
77
+ (async () => {
78
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
79
+ console.log(`Fetching depot key...`);
80
+ let depotKey = await fetchDepotKey(manifest.depot_id);
81
+ depotKey = Buffer.from(depotKey, "hex");
82
+ await populateChunks(manifest, depotKey, installDir, (file, file_i, num_files, chunk_i, num_chunks) => {
83
+ if (chunk_i == 0) {
84
+ console.log(`Processing ${file}...`);
85
+ }
86
+ });
87
+ process.exit(0);
88
+ })();
89
+ } break;
90
+
91
+ case "lancache": {
92
+ const fs = require("fs");
93
+ const fsPromises = require("fs/promises");
94
+ const path = require("path");
95
+ const express = require("express");
96
+ const net = require("net");
97
+ const sni = require("sni");
98
+
99
+ const app = express();
100
+
101
+ app.use((req, res, next) => {
102
+ res.on("finish", () => {
103
+ console.log(`${req.socket.remoteAddress} - ${req.url} - ${res.statusCode}`);
104
+ });
105
+ next();
106
+ });
107
+
108
+ app.get("/depot/:depotId/chunk/:sha", async (req, res) => {
109
+ if (fs.existsSync(__dirname + req.url)) {
110
+ res.sendFile(__dirname + req.url);
111
+ }
112
+ else if (req.host.endsWith(".steamcontent.com")) {
113
+ const fr = await fetch(`http://${req.host}${req.url}`);
114
+ if (fr.status == 200) {
115
+ const buf = Buffer.from(await fr.arrayBuffer());
116
+ await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
117
+ await fsPromises.writeFile(__dirname + req.url, buf);
118
+ res.send(buf).end();
119
+ }
120
+ else {
121
+ res.status(fr.status).send("Upstream error").end();
122
+ }
123
+ }
124
+ else {
125
+ res.status(400).send("Don't have this chunk").end();
126
+ }
127
+ });
128
+
129
+ app.get("/depot/:depotId/patch/:oldManifestId/:newManifestId", async (req, res) => {
130
+ if (fs.existsSync(__dirname + req.url)) {
131
+ res.sendFile(__dirname + req.url);
132
+ }
133
+ else if (req.host.endsWith(".steamcontent.com")) {
134
+ const fr = await fetch(`http://${req.host}${req.url}`);
135
+ if (fr.status == 200) {
136
+ const buf = Buffer.from(await fr.arrayBuffer());
137
+ //await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
138
+ //await fsPromises.writeFile(__dirname + req.url, buf);
139
+ res.send(buf).end();
140
+ }
141
+ else {
142
+ res.status(fr.status).send("Upstream error").end();
143
+ }
144
+ }
145
+ else {
146
+ res.status(400).send("Don't have this patch").end();
147
+ }
148
+ });
149
+
150
+ app.use((req, res) => {
151
+ res.status(400).send("Don't know this endpoint").end();
152
+ });
153
+
154
+ app.listen(80, () => {
155
+ console.log("Listening on port 80");
156
+ });
157
+
158
+ // Transparently forward traffic on port 443 to avoid breaking HTTPS connections
159
+
160
+ net.createServer(socket => {
161
+ socket.once("data", firstPacket => {
162
+ const hostname = sni(firstPacket);
163
+ if (!hostname) {
164
+ return;
165
+ }
166
+ console.log(`${socket.remoteAddress} - Starting TLS proxy to ${hostname}`);
167
+ const upstream = net.connect(443, hostname, () => {
168
+ upstream.write(firstPacket);
169
+ socket.pipe(upstream).pipe(socket);
170
+ });
171
+ upstream.on("error", () => socket.end());
172
+ });
173
+ socket.on("error", () => {});
174
+ }).listen(443, () => {
175
+ console.log("Listening on port 443");
176
+ });
177
+ } break;
178
+
179
+ case "install": {
180
+ const depotId = process.argv[3];
181
+ const manifestId = process.argv[4];
182
+ const installDir = process.argv[5];
183
+ if (!depotId || !manifestId) {
184
+ console.log("Syntax: mango install <depot id> <manifest id> [install dir]");
185
+ process.exit(1);
186
+ }
187
+
188
+ const ContentManifest = require("steam-user/components/content_manifest");
189
+ const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
190
+
191
+ (async () => {
192
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
193
+ console.log(`Fetching depot key...`);
194
+ let depotKey = await fetchDepotKey(manifest.depot_id);
195
+ depotKey = Buffer.from(depotKey, "hex");
196
+ await install(manifest, depotKey, installDir, (file, existed) => {
197
+ console.log(`${existed ? "Repaired" : "Created"} ${file}`);
198
+ });
199
+ process.exit(0);
200
+ })();
201
+ } break;
202
+
203
+ case "download-and-install": {
204
+ const args = parseArguments(3, ["--lancache"]);
205
+ const [depotId, manifestId, installDir] = args;
206
+ if (!depotId || !manifestId) {
207
+ console.log("Syntax: mango download-and-install <depot id> <manifest id> [install dir] [--lancache]");
208
+ process.exit(1);
209
+ }
210
+
211
+ const ContentManifest = require("steam-user/components/content_manifest");
212
+ const { DEFAULT_HOSTS, fetchManifest, fetchDepotKey, downloadAndInstall } = require("./lib.js");
213
+
214
+ const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
215
+
216
+ (async () => {
217
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
218
+
219
+ console.log(`Fetching depot key...`);
220
+ let depotKey = await fetchDepotKey(manifest.depot_id);
221
+ depotKey = Buffer.from(depotKey, "hex");
222
+
223
+ await downloadAndInstall(
224
+ manifest,
225
+ depotKey,
226
+ undefined /*(num_chunks) => {}*/,
227
+ (path, host) => { console.log(`${path}: Downloading from ${host}`); },
228
+ (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
229
+ (path, err) => { console.log(`${path}: `, err); },
230
+ hosts,
231
+ installDir
232
+ );
233
+
234
+ process.exit(0);
235
+ })();
236
+ } break;
237
+
238
+ case "to-json": {
239
+ const args = parseArguments(3, ["--no-decrypt"]);
240
+ const [depotId, manifestId] = args;
241
+ if (!depotId || !manifestId) {
242
+ console.log("Syntax: mango to-json <depot id> <manifest id> [--no-decrypt]");
243
+ process.exit(1);
244
+ }
245
+
246
+ const fs = require("fs");
247
+ const ContentManifest = require("steam-user/components/content_manifest");
248
+ const { fetchManifest, fetchDepotKey } = require("./lib.js");
249
+
250
+ (async () => {
251
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
252
+ if (manifest.filenames_encrypted && !args["--no-decrypt"]) {
253
+ console.log("Fetching depot key to decrypt filenames...");
254
+ const depotKey = await fetchDepotKey(manifest.depot_id);
255
+ ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex")); // Sets filenames_encrypted to false
256
+ //manifest.filenames_decrypted = true; // Indicate that this transformation took place in the JSON export // Kinda pointless because some .manifest files are already decrypted by their sources
257
+ }
258
+ fs.writeFileSync(`${depotId}_${manifestId}.json`, JSON.stringify(manifest, null, 2));
259
+ console.log(`Saved in ${depotId}_${manifestId}.json`);
260
+
261
+ process.exit(0);
262
+ })();
263
+ } break;
264
+
265
+ case "to-hashdeep-auditfile": {
266
+ const depotId = process.argv[3];
267
+ const manifestId = process.argv[4];
268
+ if (!depotId || !manifestId) {
269
+ console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id>");
270
+ process.exit(1);
271
+ }
272
+
273
+ const fs = require("fs");
274
+ const path = require("path");
275
+ const ContentManifest = require("steam-user/components/content_manifest");
276
+ const { fetchManifest, fetchDepotKey } = require("./lib.js");
277
+
278
+ (async () => {
279
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
280
+ if (manifest.filenames_encrypted) {
281
+ console.log("Manifest has encrypted filenames. Fetching depot key...");
282
+ const depotKey = await fetchDepotKey(manifest.depot_id);
283
+ ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex"));
284
+ }
285
+
286
+ const fh = fs.createWriteStream(`${depotId}_${manifestId}.auditfile`);
287
+ fh.write("%%%% HASHDEEP-1.0\n");
288
+ fh.write("%%%% size,sha1,filename\n");
289
+ for (const file of manifest.files) {
290
+ if ((file.flags & 64) === 0) {
291
+ fh.write(`${file.size},${file.sha_content},${manifest.gid_manifest}${path.sep}${path.sep != '\\' ? file.filename.split('\\').join(path.sep) : file.filename.split('/').join(path.sep)}\n`);
292
+ }
293
+ }
294
+ fh.end();
295
+ fh.on("finish", () => {
296
+ console.log(`Saved in ${depotId}_${manifestId}.auditfile`);
297
+
298
+ process.exit(0);
299
+ });
300
+ })();
301
+ } break;
302
+
303
+ case "to-torrent": {
304
+ const depotId = process.argv[3];
305
+ const manifestId = process.argv[4];
306
+ const fileHash = process.argv[5];
307
+ if (!depotId || !manifestId || !fileHash) {
308
+ console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
309
+ process.exit(1);
310
+ }
311
+
312
+ const ContentManifest = require("steam-user/components/content_manifest");
313
+
314
+ function toPieces(hexArray) {
315
+ const bufs = hexArray.map(h => {
316
+ if (typeof h !== "string" || h.length !== 40) {
317
+ throw new Error(`Invalid SHA1 hex string: ${h}`);
318
+ }
319
+ return Buffer.from(h, "hex");
320
+ });
321
+ return Buffer.concat(bufs);
322
+ }
323
+
324
+ (async () => {
325
+ const { fetchManifest } = require("./lib.js");
326
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
327
+ const file = manifest.files.find(x => x.sha_content == fileHash);
328
+ if (!file) {
329
+ console.log(`No file with content hash ${fileHash} found in ${depotId}_${manifestId}.manifest`);
330
+ process.exit(1);
331
+ }
332
+ for (const chunk of file.chunks) {
333
+ chunk.offset = parseInt(chunk.offset);
334
+ }
335
+ file.chunks.sort((a, b) => a.offset - b.offset);
336
+ let pieceSize = file.chunks[0].cb_original;
337
+ for (let i = 1; i < file.chunks.length - 1; ++i) {
338
+ if (pieceSize != file.chunks[i].cb_original) {
339
+ console.log(`This file has non-uniformly sized chunks and is therefore unrepresentable in BitTorrent.`);
340
+ process.exit(2);
341
+ }
342
+ }
343
+ const { toTorrentFile } = await import("parse-torrent");
344
+ const buf = toTorrentFile({
345
+ info: {
346
+ name: file.sha_content,
347
+ length: parseInt(file.size),
348
+ 'piece length': pieceSize,
349
+ pieces: toPieces(file.chunks.map(chunk => chunk.sha))
350
+ }
351
+ });
352
+ const fs = require("node:fs");
353
+ fs.writeFileSync(`${file.sha_content}.torrent`, buf);
354
+ console.log(`Saved in ${file.sha_content}.torrent`);
355
+ process.exit(0);
356
+ })();
357
+ } break;
358
+
359
+ case "verify-chunks": {
360
+ const depotId = process.argv[3];
361
+ if (!depotId) {
362
+ console.log("Syntax: mango verify-chunks <depot id>");
363
+ process.exit(1);
364
+ }
365
+
366
+ const { fetchDepotKey, verifyChunks } = require("./lib.js");
367
+ (async () => {
368
+ console.log(`Fetching depot key...`);
369
+ let depotKey = await fetchDepotKey(depotId);
370
+ depotKey = Buffer.from(depotKey, "hex");
371
+ console.log(`Verifying chunks...`);
372
+ await verifyChunks(depotId, depotKey, (file, expectedHash) => {
373
+ console.log(`Deleted ${file}`);
374
+ });
375
+ process.exit(0);
376
+ })();
377
+ } break;
312
378
  }
package/package.json CHANGED
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "steam-manifest-tools",
3
- "version": "0.1.4",
3
+ "version": "0.1.6",
4
4
  "main": "lib.js",
5
5
  "repository": {
6
6
  "type": "git",
7
7
  "url": "git+https://github.com/Sainan/steam-manifest-tools.git"
8
8
  },
9
9
  "scripts": {
10
- "build": "bun build mango.js --compile"
10
+ "build": "mv node_modules nm && nexe mango.js -t x64-22.21.1 --build && mv nm node_modules"
11
11
  },
12
12
  "dependencies": {
13
13
  "@doctormckay/steam-crypto": "^1.2.0",