steam-manifest-tools 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/lib.js +156 -67
- package/mango.js +375 -322
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
Tools to interact with Steam depots and manifests.
|
|
2
2
|
- `mango download-chunks <depot id> <manifest id> [--lancache]` (basically the same as prefilling a LAN cache, optionally downloads from a LAN cache)
|
|
3
|
-
- `mango populate-chunks <depot id> <manifest id> <install dir
|
|
3
|
+
- `mango populate-chunks <depot id> <manifest id> <install dir>` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
|
|
4
4
|
- `mango lancache`
|
|
5
5
|
- Must run as root to obtain port 80
|
|
6
6
|
- `lancache.steamcontent.com` must resolve to your LAN cache server IP (either via custom DNS server or OS hosts file)
|
|
7
|
-
- `mango install <depot id> <manifest id> [
|
|
8
|
-
- `mango
|
|
9
|
-
- `mango to-
|
|
7
|
+
- `mango install <depot id> <manifest id> [install dir]` (requires chunks to have been populated and/or downloaded beforehand)
|
|
8
|
+
- `mango download-and-install <depot id> <manifest id> [install dir] [--lancache]`
|
|
9
|
+
- `mango to-json <depot id> <manifest id> [--no-decrypt]`
|
|
10
|
+
- `mango to-hashdeep-auditfile <depot id> <manifest id>`
|
|
10
11
|
- Note that hashdeep requires exact path matches and this script makes a few assumptions
|
|
11
12
|
- Example verification usage: `hashdeep -a -l -v -v -k 241561_8497448424664183398.auditfile -r 8497448424664183398`
|
|
12
13
|
- `mango to-torrent <depot id> <manifest id> <file hash>`
|
|
13
|
-
- `mango verify-chunks <depot id
|
|
14
|
+
- `mango verify-chunks <depot id>`
|
|
14
15
|
- `mango download-manifest <depot id> <manifest id>`
|
package/lib.js
CHANGED
|
@@ -92,6 +92,73 @@ const getChunk = async (depotId, depotKey, hash) => {
|
|
|
92
92
|
}
|
|
93
93
|
};
|
|
94
94
|
|
|
95
|
+
const downloadChunks = (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
|
|
96
|
+
hosts ??= DEFAULT_HOSTS;
|
|
97
|
+
return new Promise(resolve => {
|
|
98
|
+
fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
|
|
99
|
+
const toDownload = {};
|
|
100
|
+
const alreadyDownloaded = [];
|
|
101
|
+
for (const file of manifest.files) {
|
|
102
|
+
for (const chunk of file.chunks) {
|
|
103
|
+
if (fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
|
|
104
|
+
alreadyDownloaded.push(`depot/${manifest.depot_id}/chunk/${chunk.sha}`);
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
toDownload[`depot/${manifest.depot_id}/chunk/${chunk.sha}`] = true;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
let remaining_chunks = Object.keys(toDownload).length;
|
|
112
|
+
if (onStartDownloading) {
|
|
113
|
+
onStartDownloading(remaining_chunks, alreadyDownloaded);
|
|
114
|
+
}
|
|
115
|
+
if (remaining_chunks == 0) {
|
|
116
|
+
resolve();
|
|
117
|
+
}
|
|
118
|
+
let /*host_i = 0,*/ running = 0;
|
|
119
|
+
const concurrency = hosts.length == 1 ? 4 : (hosts.length * 2);
|
|
120
|
+
const loop = () => {
|
|
121
|
+
while (running < concurrency) {
|
|
122
|
+
const path = Object.keys(toDownload)[0];
|
|
123
|
+
if (!path) {
|
|
124
|
+
break;
|
|
125
|
+
}
|
|
126
|
+
++running;
|
|
127
|
+
delete toDownload[path];
|
|
128
|
+
const host = hosts[Math.floor(Math.random()*hosts.length)];
|
|
129
|
+
//const host = hosts[host_i]; host_i = (host_i + 1) % hosts.length;
|
|
130
|
+
if (onStartDownload) {
|
|
131
|
+
onStartDownload(path, host);
|
|
132
|
+
}
|
|
133
|
+
fetch(`${host}/${path}`).then(async res => {
|
|
134
|
+
if (res.status == 200) {
|
|
135
|
+
const ab = await res.arrayBuffer();
|
|
136
|
+
await fsPromises.writeFile(path, Buffer.from(ab));
|
|
137
|
+
if (--remaining_chunks == 0) {
|
|
138
|
+
resolve();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
toDownload[path] = true;
|
|
143
|
+
}
|
|
144
|
+
if (onFinishDownload) {
|
|
145
|
+
onFinishDownload(path, res.status, host);
|
|
146
|
+
}
|
|
147
|
+
}).catch(err => {
|
|
148
|
+
if (onErroredDownload) {
|
|
149
|
+
onErroredDownload(path, err);
|
|
150
|
+
}
|
|
151
|
+
toDownload[path] = true;
|
|
152
|
+
}).finally(() => {
|
|
153
|
+
--running;
|
|
154
|
+
loop();
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
loop();
|
|
159
|
+
});
|
|
160
|
+
};
|
|
161
|
+
|
|
95
162
|
module.exports = {
|
|
96
163
|
DEFAULT_HOSTS,
|
|
97
164
|
sha1,
|
|
@@ -125,17 +192,25 @@ module.exports = {
|
|
|
125
192
|
throw new Error(`Could not find the given manifest (${manifestId}). Double-check with https://steamdb.info/depot/${depotId}/manifests/ and report an issue in https://github.com/Sainan/k25FCdfEOoEJ42S6/issues if you're sure the manifest exists.`);
|
|
126
193
|
},
|
|
127
194
|
fetchDepotKey: async (depotId) => {
|
|
195
|
+
try {
|
|
196
|
+
return await fsPromises.readFile(`depot/${depotId}/key.txt`, "utf-8");
|
|
197
|
+
}
|
|
198
|
+
catch (e) {
|
|
199
|
+
// fallthrough
|
|
200
|
+
}
|
|
128
201
|
const depotkeys = await fetch("https://raw.githubusercontent.com/SteamAutoCracks/ManifestHub/refs/heads/main/depotkeys.json").then(x => x.json());
|
|
129
202
|
if (!depotkeys[depotId]) {
|
|
130
203
|
throw new Error(`Failed to get the depot key.`);
|
|
131
204
|
}
|
|
205
|
+
await fsPromises.mkdir(`depot/${depotId}`, { recursive: true });
|
|
206
|
+
fsPromises.writeFile(`depot/${depotId}/key.txt`, depotkeys[depotId], "utf-8");
|
|
132
207
|
return depotkeys[depotId];
|
|
133
208
|
},
|
|
134
209
|
populateChunks: async (manifest, depotKey, installDir, onProgress) => {
|
|
135
210
|
if (manifest.filenames_encrypted) {
|
|
136
211
|
ContentManifest.decryptFilenames(manifest, depotKey);
|
|
137
212
|
}
|
|
138
|
-
|
|
213
|
+
await fsPromises.mkdir(`depot/${manifest.depot_id}/chunk`, { recursive: true });
|
|
139
214
|
let file_i = 0;
|
|
140
215
|
for (const file of manifest.files) {
|
|
141
216
|
if (file.flags & 64) {
|
|
@@ -172,79 +247,19 @@ module.exports = {
|
|
|
172
247
|
++file_i;
|
|
173
248
|
}
|
|
174
249
|
},
|
|
175
|
-
downloadChunks
|
|
176
|
-
hosts ??= DEFAULT_HOSTS;
|
|
177
|
-
return new Promise(resolve => {
|
|
178
|
-
fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
|
|
179
|
-
const toDownload = {};
|
|
180
|
-
for (const file of manifest.files) {
|
|
181
|
-
for (const chunk of file.chunks) {
|
|
182
|
-
if (!fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
|
|
183
|
-
toDownload[`depot/${manifest.depot_id}/chunk/${chunk.sha}`] = true;
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
let remaining_chunks = Object.keys(toDownload).length;
|
|
188
|
-
if (onStartDownloading) {
|
|
189
|
-
onStartDownloading(remaining_chunks);
|
|
190
|
-
}
|
|
191
|
-
if (remaining_chunks == 0) {
|
|
192
|
-
resolve();
|
|
193
|
-
}
|
|
194
|
-
let /*host_i = 0,*/ running = 0;
|
|
195
|
-
const concurrency = hosts.length == 1 ? 4 : (hosts.length * 2);
|
|
196
|
-
const loop = () => {
|
|
197
|
-
while (running < concurrency) {
|
|
198
|
-
const path = Object.keys(toDownload)[0];
|
|
199
|
-
if (!path) {
|
|
200
|
-
break;
|
|
201
|
-
}
|
|
202
|
-
++running;
|
|
203
|
-
delete toDownload[path];
|
|
204
|
-
const host = hosts[Math.floor(Math.random()*hosts.length)];
|
|
205
|
-
//const host = hosts[host_i]; host_i = (host_i + 1) % hosts.length;
|
|
206
|
-
if (onStartDownload) {
|
|
207
|
-
onStartDownload(path, host);
|
|
208
|
-
}
|
|
209
|
-
fetch(`${host}/${path}`).then(async res => {
|
|
210
|
-
if (onFinishDownload) {
|
|
211
|
-
onFinishDownload(path, res.status, host);
|
|
212
|
-
}
|
|
213
|
-
if (res.status == 200) {
|
|
214
|
-
const ab = await res.arrayBuffer();
|
|
215
|
-
await fsPromises.writeFile(path, Buffer.from(ab));
|
|
216
|
-
if (--remaining_chunks == 0) {
|
|
217
|
-
resolve();
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
else {
|
|
221
|
-
toDownload[path] = true;
|
|
222
|
-
}
|
|
223
|
-
}).catch(err => {
|
|
224
|
-
if (onErroredDownload) {
|
|
225
|
-
onErroredDownload(path, err);
|
|
226
|
-
}
|
|
227
|
-
toDownload[path] = true;
|
|
228
|
-
}).finally(() => {
|
|
229
|
-
--running;
|
|
230
|
-
loop();
|
|
231
|
-
});
|
|
232
|
-
}
|
|
233
|
-
};
|
|
234
|
-
loop();
|
|
235
|
-
});
|
|
236
|
-
},
|
|
250
|
+
downloadChunks,
|
|
237
251
|
install: async (manifest, depotKey, installDir, onFileWritten) => {
|
|
238
252
|
ContentManifest.decryptFilenames(manifest, depotKey);
|
|
239
253
|
installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
|
|
240
|
-
|
|
254
|
+
await fsPromises.mkdir(installDir, { recursive: true });
|
|
241
255
|
const promises = [];
|
|
242
256
|
for (const file of manifest.files) {
|
|
243
|
-
if (file.flags & 64) {
|
|
244
|
-
continue;
|
|
245
|
-
}
|
|
246
257
|
promises.push((async () => {
|
|
247
258
|
const filename = file.filename.replace(/\\/g, "/");
|
|
259
|
+
if (file.flags & 64) {
|
|
260
|
+
await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
248
263
|
const exists = fs.existsSync(path.join(installDir, filename));
|
|
249
264
|
if (!exists || await sha1file(path.join(installDir, filename)) != file.sha_content) {
|
|
250
265
|
for (const chunk of file.chunks) {
|
|
@@ -266,6 +281,80 @@ module.exports = {
|
|
|
266
281
|
}
|
|
267
282
|
await Promise.all(promises);
|
|
268
283
|
},
|
|
284
|
+
downloadAndInstall: async (manifest, depotKey, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts, installDir) => {
|
|
285
|
+
const depotId = manifest.depot_id;
|
|
286
|
+
|
|
287
|
+
ContentManifest.decryptFilenames(manifest, depotKey);
|
|
288
|
+
installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
|
|
289
|
+
|
|
290
|
+
const writeStreams = {};
|
|
291
|
+
for (const file of manifest.files) {
|
|
292
|
+
const filename = file.filename.replace(/\\/g, "/");
|
|
293
|
+
if (file.flags & 64) {
|
|
294
|
+
await fsPromises.mkdir(path.join(installDir, filename), { recursive: true });
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
await fsPromises.mkdir(path.dirname(path.join(installDir, filename)), { recursive: true });
|
|
298
|
+
writeStreams[file.filename] = await fsPromises.open(path.join(installDir, filename), "w");
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const ioQueue = [];
|
|
303
|
+
let ioLoopRunning = false;
|
|
304
|
+
let ioLoopPromise;
|
|
305
|
+
const ioLoop = async () => {
|
|
306
|
+
ioLoopRunning = true;
|
|
307
|
+
while (ioQueue.length > 0) {
|
|
308
|
+
const chunkSha = ioQueue.shift();
|
|
309
|
+
const data = await getChunk(depotId, depotKey, chunkSha);
|
|
310
|
+
for (const file of manifest.files) {
|
|
311
|
+
for (const chunk of file.chunks) {
|
|
312
|
+
if (chunk.sha == chunkSha) {
|
|
313
|
+
await writeStreams[file.filename].write(data, 0, data.byteLength, parseInt(chunk.offset));
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
ioLoopRunning = false;
|
|
319
|
+
};
|
|
320
|
+
const ioAddPath = (path) => {
|
|
321
|
+
ioQueue.push(path.substr(path.length - 40));
|
|
322
|
+
if (!ioLoopRunning) {
|
|
323
|
+
ioLoopPromise = ioLoop();
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
await downloadChunks(
|
|
328
|
+
manifest,
|
|
329
|
+
(num_chunks, alreadyDownloaded) => {
|
|
330
|
+
for (const path of alreadyDownloaded) {
|
|
331
|
+
ioAddPath(path);
|
|
332
|
+
}
|
|
333
|
+
if (onStartDownloading) {
|
|
334
|
+
onStartDownloading(num_chunks);
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
onStartDownload,
|
|
338
|
+
(path, status, host) => {
|
|
339
|
+
if (status == 200) {
|
|
340
|
+
ioAddPath(path);
|
|
341
|
+
}
|
|
342
|
+
if (onFinishDownload) {
|
|
343
|
+
onFinishDownload(path, status, host);
|
|
344
|
+
}
|
|
345
|
+
},
|
|
346
|
+
onErroredDownload,
|
|
347
|
+
hosts
|
|
348
|
+
);
|
|
349
|
+
|
|
350
|
+
if (ioLoopRunning) {
|
|
351
|
+
await ioLoopPromise;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
for (const writeStream of Object.values(writeStreams)) {
|
|
355
|
+
await writeStream.close();
|
|
356
|
+
}
|
|
357
|
+
},
|
|
269
358
|
verifyChunks: async (depotId, depotKey, onDeletedFile) => {
|
|
270
359
|
const files = await getFiles(`depot/${depotId}/chunk`);
|
|
271
360
|
let promises = [];
|
package/mango.js
CHANGED
|
@@ -1,325 +1,378 @@
|
|
|
1
|
+
const parseArguments = (base, bools = []) => {
|
|
2
|
+
const res = [];
|
|
3
|
+
let j = 0;
|
|
4
|
+
for (let i = base; i != process.argv.length; ++i) {
|
|
5
|
+
if (bools.indexOf(process.argv[i]) != -1) {
|
|
6
|
+
res[process.argv[i]] = true;
|
|
7
|
+
}
|
|
8
|
+
else {
|
|
9
|
+
res[j++] = process.argv[i];
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return res;
|
|
13
|
+
};
|
|
14
|
+
|
|
1
15
|
const tool = process.argv[2];
|
|
2
16
|
switch (tool) {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
17
|
+
default: {
|
|
18
|
+
console.error(tool ? `Unknown tool: ${tool}` : "Syntax: mango <tool>");
|
|
19
|
+
process.exit(1);
|
|
20
|
+
} break;
|
|
21
|
+
|
|
22
|
+
case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
|
|
23
|
+
const depotId = process.argv[3];
|
|
24
|
+
const manifestId = process.argv[4];
|
|
25
|
+
if (!depotId || !manifestId) {
|
|
26
|
+
console.log("Syntax: mango download-manifest <depot id> <manifest id>");
|
|
27
|
+
process.exit(1);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const { fetchManifest } = require("./lib.js");
|
|
31
|
+
|
|
32
|
+
(async () => {
|
|
33
|
+
await fetchManifest(depotId, manifestId);
|
|
34
|
+
console.log(`Saved in manifests/${depotId}_${manifestId}.manifest`);
|
|
35
|
+
})();
|
|
36
|
+
} break;
|
|
37
|
+
|
|
38
|
+
case "download-chunks": {
|
|
39
|
+
const args = parseArguments(3, ["--lancache"]);
|
|
40
|
+
const [depotId, manifestId] = args;
|
|
41
|
+
if (!depotId || !manifestId) {
|
|
42
|
+
console.log("Syntax: mango download-chunks <depot id> <manifest id> [--lancache]");
|
|
43
|
+
process.exit(1);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const ContentManifest = require("steam-user/components/content_manifest");
|
|
47
|
+
const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
|
|
48
|
+
|
|
49
|
+
const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
|
|
50
|
+
|
|
51
|
+
(async () => {
|
|
52
|
+
const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
|
|
53
|
+
await downloadChunks(
|
|
54
|
+
manifest,
|
|
55
|
+
undefined /*(num_chunks) => {}*/,
|
|
56
|
+
(path, host) => { console.log(`${path}: Downloading from ${host}`); },
|
|
57
|
+
(path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
|
|
58
|
+
(path, err) => { console.log(`${path}: `, err); },
|
|
59
|
+
hosts
|
|
60
|
+
)
|
|
61
|
+
process.exit(0);
|
|
62
|
+
})();
|
|
63
|
+
} break;
|
|
64
|
+
|
|
65
|
+
case "populate-chunks": {
|
|
66
|
+
const depotId = process.argv[3];
|
|
67
|
+
const manifestId = process.argv[4];
|
|
68
|
+
const installDir = process.argv[5];
|
|
69
|
+
if (!depotId || !manifestId || !installDir) {
|
|
70
|
+
console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir>");
|
|
71
|
+
process.exit(1);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const ContentManifest = require("steam-user/components/content_manifest");
|
|
75
|
+
const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
|
|
76
|
+
|
|
77
|
+
(async () => {
|
|
78
|
+
const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
|
|
79
|
+
console.log(`Fetching depot key...`);
|
|
80
|
+
let depotKey = await fetchDepotKey(manifest.depot_id);
|
|
81
|
+
depotKey = Buffer.from(depotKey, "hex");
|
|
82
|
+
await populateChunks(manifest, depotKey, installDir, (file, file_i, num_files, chunk_i, num_chunks) => {
|
|
83
|
+
if (chunk_i == 0) {
|
|
84
|
+
console.log(`Processing ${file}...`);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
process.exit(0);
|
|
88
|
+
})();
|
|
89
|
+
} break;
|
|
90
|
+
|
|
91
|
+
case "lancache": {
|
|
92
|
+
const fs = require("fs");
|
|
93
|
+
const fsPromises = require("fs/promises");
|
|
94
|
+
const path = require("path");
|
|
95
|
+
const express = require("express");
|
|
96
|
+
const net = require("net");
|
|
97
|
+
const sni = require("sni");
|
|
98
|
+
|
|
99
|
+
const app = express();
|
|
100
|
+
|
|
101
|
+
app.use((req, res, next) => {
|
|
102
|
+
res.on("finish", () => {
|
|
103
|
+
console.log(`${req.socket.remoteAddress} - ${req.url} - ${res.statusCode}`);
|
|
104
|
+
});
|
|
105
|
+
next();
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
app.get("/depot/:depotId/chunk/:sha", async (req, res) => {
|
|
109
|
+
if (fs.existsSync(__dirname + req.url)) {
|
|
110
|
+
res.sendFile(__dirname + req.url);
|
|
111
|
+
}
|
|
112
|
+
else if (req.host.endsWith(".steamcontent.com")) {
|
|
113
|
+
const fr = await fetch(`http://${req.host}${req.url}`);
|
|
114
|
+
if (fr.status == 200) {
|
|
115
|
+
const buf = Buffer.from(await fr.arrayBuffer());
|
|
116
|
+
await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
|
|
117
|
+
await fsPromises.writeFile(__dirname + req.url, buf);
|
|
118
|
+
res.send(buf).end();
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
res.status(fr.status).send("Upstream error").end();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
res.status(400).send("Don't have this chunk").end();
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
app.get("/depot/:depotId/patch/:oldManifestId/:newManifestId", async (req, res) => {
|
|
130
|
+
if (fs.existsSync(__dirname + req.url)) {
|
|
131
|
+
res.sendFile(__dirname + req.url);
|
|
132
|
+
}
|
|
133
|
+
else if (req.host.endsWith(".steamcontent.com")) {
|
|
134
|
+
const fr = await fetch(`http://${req.host}${req.url}`);
|
|
135
|
+
if (fr.status == 200) {
|
|
136
|
+
const buf = Buffer.from(await fr.arrayBuffer());
|
|
137
|
+
//await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
|
|
138
|
+
//await fsPromises.writeFile(__dirname + req.url, buf);
|
|
139
|
+
res.send(buf).end();
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
res.status(fr.status).send("Upstream error").end();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
res.status(400).send("Don't have this patch").end();
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
app.use((req, res) => {
|
|
151
|
+
res.status(400).send("Don't know this endpoint").end();
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
app.listen(80, () => {
|
|
155
|
+
console.log("Listening on port 80");
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
// Transparently forward traffic on port 443 to avoid breaking HTTPS connections
|
|
159
|
+
|
|
160
|
+
net.createServer(socket => {
|
|
161
|
+
socket.once("data", firstPacket => {
|
|
162
|
+
const hostname = sni(firstPacket);
|
|
163
|
+
if (!hostname) {
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
console.log(`${socket.remoteAddress} - Starting TLS proxy to ${hostname}`);
|
|
167
|
+
const upstream = net.connect(443, hostname, () => {
|
|
168
|
+
upstream.write(firstPacket);
|
|
169
|
+
socket.pipe(upstream).pipe(socket);
|
|
170
|
+
});
|
|
171
|
+
upstream.on("error", () => socket.end());
|
|
172
|
+
});
|
|
173
|
+
socket.on("error", () => {});
|
|
174
|
+
}).listen(443, () => {
|
|
175
|
+
console.log("Listening on port 443");
|
|
176
|
+
});
|
|
177
|
+
} break;
|
|
178
|
+
|
|
179
|
+
case "install": {
|
|
180
|
+
const depotId = process.argv[3];
|
|
181
|
+
const manifestId = process.argv[4];
|
|
182
|
+
const installDir = process.argv[5];
|
|
183
|
+
if (!depotId || !manifestId) {
|
|
184
|
+
console.log("Syntax: mango install <depot id> <manifest id> [install dir]");
|
|
185
|
+
process.exit(1);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const ContentManifest = require("steam-user/components/content_manifest");
|
|
189
|
+
const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
|
|
190
|
+
|
|
191
|
+
(async () => {
|
|
192
|
+
const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
|
|
193
|
+
console.log(`Fetching depot key...`);
|
|
194
|
+
let depotKey = await fetchDepotKey(manifest.depot_id);
|
|
195
|
+
depotKey = Buffer.from(depotKey, "hex");
|
|
196
|
+
await install(manifest, depotKey, installDir, (file, existed) => {
|
|
197
|
+
console.log(`${existed ? "Repaired" : "Created"} ${file}`);
|
|
198
|
+
});
|
|
199
|
+
process.exit(0);
|
|
200
|
+
})();
|
|
201
|
+
} break;
|
|
202
|
+
|
|
203
|
+
case "download-and-install": {
|
|
204
|
+
const args = parseArguments(3, ["--lancache"]);
|
|
205
|
+
const [depotId, manifestId, installDir] = args;
|
|
206
|
+
if (!depotId || !manifestId) {
|
|
207
|
+
console.log("Syntax: mango download-and-install <depot id> <manifest id> [install dir] [--lancache]");
|
|
208
|
+
process.exit(1);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const ContentManifest = require("steam-user/components/content_manifest");
|
|
212
|
+
const { DEFAULT_HOSTS, fetchManifest, fetchDepotKey, downloadAndInstall } = require("./lib.js");
|
|
213
|
+
|
|
214
|
+
const hosts = args["--lancache"] ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
|
|
215
|
+
|
|
216
|
+
(async () => {
|
|
217
|
+
const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
|
|
218
|
+
|
|
219
|
+
console.log(`Fetching depot key...`);
|
|
220
|
+
let depotKey = await fetchDepotKey(manifest.depot_id);
|
|
221
|
+
depotKey = Buffer.from(depotKey, "hex");
|
|
222
|
+
|
|
223
|
+
await downloadAndInstall(
|
|
224
|
+
manifest,
|
|
225
|
+
depotKey,
|
|
226
|
+
undefined /*(num_chunks) => {}*/,
|
|
227
|
+
(path, host) => { console.log(`${path}: Downloading from ${host}`); },
|
|
228
|
+
(path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
|
|
229
|
+
(path, err) => { console.log(`${path}: `, err); },
|
|
230
|
+
hosts,
|
|
231
|
+
installDir
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
process.exit(0);
|
|
235
|
+
})();
|
|
236
|
+
} break;
|
|
237
|
+
|
|
238
|
+
case "to-json": {
|
|
239
|
+
const args = parseArguments(3, ["--no-decrypt"]);
|
|
240
|
+
const [depotId, manifestId] = args;
|
|
241
|
+
if (!depotId || !manifestId) {
|
|
242
|
+
console.log("Syntax: mango to-json <depot id> <manifest id> [--no-decrypt]");
|
|
243
|
+
process.exit(1);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const fs = require("fs");
|
|
247
|
+
const ContentManifest = require("steam-user/components/content_manifest");
|
|
248
|
+
const { fetchManifest, fetchDepotKey } = require("./lib.js");
|
|
249
|
+
|
|
250
|
+
(async () => {
|
|
251
|
+
const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
|
|
252
|
+
if (manifest.filenames_encrypted && !args["--no-decrypt"]) {
|
|
253
|
+
console.log("Fetching depot key to decrypt filenames...");
|
|
254
|
+
const depotKey = await fetchDepotKey(manifest.depot_id);
|
|
255
|
+
ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex")); // Sets filenames_encrypted to false
|
|
256
|
+
//manifest.filenames_decrypted = true; // Indicate that this transformation took place in the JSON export // Kinda pointless because some .manifest files are already decrypted by their sources
|
|
257
|
+
}
|
|
258
|
+
fs.writeFileSync(`${depotId}_${manifestId}.json`, JSON.stringify(manifest, null, 2));
|
|
259
|
+
console.log(`Saved in ${depotId}_${manifestId}.json`);
|
|
260
|
+
|
|
261
|
+
process.exit(0);
|
|
262
|
+
})();
|
|
263
|
+
} break;
|
|
264
|
+
|
|
265
|
+
case "to-hashdeep-auditfile": {
|
|
266
|
+
const depotId = process.argv[3];
|
|
267
|
+
const manifestId = process.argv[4];
|
|
268
|
+
if (!depotId || !manifestId) {
|
|
269
|
+
console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id>");
|
|
270
|
+
process.exit(1);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const fs = require("fs");
|
|
274
|
+
const path = require("path");
|
|
275
|
+
const ContentManifest = require("steam-user/components/content_manifest");
|
|
276
|
+
const { fetchManifest, fetchDepotKey } = require("./lib.js");
|
|
277
|
+
|
|
278
|
+
(async () => {
|
|
279
|
+
const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
|
|
280
|
+
if (manifest.filenames_encrypted) {
|
|
281
|
+
console.log("Manifest has encrypted filenames. Fetching depot key...");
|
|
282
|
+
const depotKey = await fetchDepotKey(manifest.depot_id);
|
|
283
|
+
ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex"));
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
const fh = fs.createWriteStream(`${depotId}_${manifestId}.auditfile`);
|
|
287
|
+
fh.write("%%%% HASHDEEP-1.0\n");
|
|
288
|
+
fh.write("%%%% size,sha1,filename\n");
|
|
289
|
+
for (const file of manifest.files) {
|
|
290
|
+
if ((file.flags & 64) === 0) {
|
|
291
|
+
fh.write(`${file.size},${file.sha_content},${manifest.gid_manifest}${path.sep}${path.sep != '\\' ? file.filename.split('\\').join(path.sep) : file.filename.split('/').join(path.sep)}\n`);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
fh.end();
|
|
295
|
+
fh.on("finish", () => {
|
|
296
|
+
console.log(`Saved in ${depotId}_${manifestId}.auditfile`);
|
|
297
|
+
|
|
298
|
+
process.exit(0);
|
|
299
|
+
});
|
|
300
|
+
})();
|
|
301
|
+
} break;
|
|
302
|
+
|
|
303
|
+
case "to-torrent": {
|
|
304
|
+
const depotId = process.argv[3];
|
|
305
|
+
const manifestId = process.argv[4];
|
|
306
|
+
const fileHash = process.argv[5];
|
|
307
|
+
if (!depotId || !manifestId || !fileHash) {
|
|
308
|
+
console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
|
|
309
|
+
process.exit(1);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const ContentManifest = require("steam-user/components/content_manifest");
|
|
313
|
+
|
|
314
|
+
function toPieces(hexArray) {
|
|
315
|
+
const bufs = hexArray.map(h => {
|
|
316
|
+
if (typeof h !== "string" || h.length !== 40) {
|
|
317
|
+
throw new Error(`Invalid SHA1 hex string: ${h}`);
|
|
318
|
+
}
|
|
319
|
+
return Buffer.from(h, "hex");
|
|
320
|
+
});
|
|
321
|
+
return Buffer.concat(bufs);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
(async () => {
|
|
325
|
+
const { fetchManifest } = require("./lib.js");
|
|
326
|
+
const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
|
|
327
|
+
const file = manifest.files.find(x => x.sha_content == fileHash);
|
|
328
|
+
if (!file) {
|
|
329
|
+
console.log(`No file with content hash ${fileHash} found in ${depotId}_${manifestId}.manifest`);
|
|
330
|
+
process.exit(1);
|
|
331
|
+
}
|
|
332
|
+
for (const chunk of file.chunks) {
|
|
333
|
+
chunk.offset = parseInt(chunk.offset);
|
|
334
|
+
}
|
|
335
|
+
file.chunks.sort((a, b) => a.offset - b.offset);
|
|
336
|
+
let pieceSize = file.chunks[0].cb_original;
|
|
337
|
+
for (let i = 1; i < file.chunks.length - 1; ++i) {
|
|
338
|
+
if (pieceSize != file.chunks[i].cb_original) {
|
|
339
|
+
console.log(`This file has non-uniformly sized chunks and is therefore unrepresentable in BitTorrent.`);
|
|
340
|
+
process.exit(2);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
const { toTorrentFile } = await import("parse-torrent");
|
|
344
|
+
const buf = toTorrentFile({
|
|
345
|
+
info: {
|
|
346
|
+
name: file.sha_content,
|
|
347
|
+
length: parseInt(file.size),
|
|
348
|
+
'piece length': pieceSize,
|
|
349
|
+
pieces: toPieces(file.chunks.map(chunk => chunk.sha))
|
|
350
|
+
}
|
|
351
|
+
});
|
|
352
|
+
const fs = require("node:fs");
|
|
353
|
+
fs.writeFileSync(`${file.sha_content}.torrent`, buf);
|
|
354
|
+
console.log(`Saved in ${file.sha_content}.torrent`);
|
|
355
|
+
process.exit(0);
|
|
356
|
+
})();
|
|
357
|
+
} break;
|
|
358
|
+
|
|
359
|
+
case "verify-chunks": {
|
|
360
|
+
const depotId = process.argv[3];
|
|
361
|
+
if (!depotId) {
|
|
362
|
+
console.log("Syntax: mango verify-chunks <depot id>");
|
|
363
|
+
process.exit(1);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
const { fetchDepotKey, verifyChunks } = require("./lib.js");
|
|
367
|
+
(async () => {
|
|
368
|
+
console.log(`Fetching depot key...`);
|
|
369
|
+
let depotKey = await fetchDepotKey(depotId);
|
|
370
|
+
depotKey = Buffer.from(depotKey, "hex");
|
|
371
|
+
console.log(`Verifying chunks...`);
|
|
372
|
+
await verifyChunks(depotId, depotKey, (file, expectedHash) => {
|
|
373
|
+
console.log(`Deleted ${file}`);
|
|
374
|
+
});
|
|
375
|
+
process.exit(0);
|
|
376
|
+
})();
|
|
377
|
+
} break;
|
|
325
378
|
}
|