steam-manifest-tools 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +9 -9
  2. package/lib.js +13 -5
  3. package/mango.js +132 -119
  4. package/package.json +2 -2
package/README.md CHANGED
@@ -1,14 +1,14 @@
1
- Tools to interact with Steam manifest files.
2
- - `mango download-manifest <depot id> <manifest id>`
3
- - `mango download-chunks <manifest file> [--lancache]` (basically the same as prefilling a LAN cache, optionally downloads from a LAN cache)
4
- - `mango populate-chunks <manifest file> <install dir> [depot key]` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
1
+ Tools to interact with Steam depots and manifests.
2
+ - `mango download-chunks <depot id> <manifest id> [--lancache]` (basically the same as prefilling a LAN cache, optionally downloads from a LAN cache)
3
+ - `mango populate-chunks <depot id> <manifest id> <install dir> [depot key]` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
5
4
  - `mango lancache`
6
5
  - Must run as root to obtain port 80
7
6
  - `lancache.steamcontent.com` must resolve to your LAN cache server IP (either via custom DNS server or OS hosts file)
8
- - `mango install <manifest file> [depot key]` (requires chunks to have been populated and/or downloaded beforehand)
9
- - `mango to-json <manifest file> [depot key]`
10
- - `mango to-hashdeep-auditfile <manifest file> [depot key]`
7
+ - `mango install <depot id> <manifest id> [depot key]` (requires chunks to have been populated and/or downloaded beforehand)
8
+ - `mango to-json <depot id> <manifest id> [depot key|--no-decrypt]`
9
+ - `mango to-hashdeep-auditfile <depot id> <manifest id> [depot key]`
11
10
  - Note that hashdeep requires exact path matches and this script makes a few assumptions
12
- - Example verification usage: `hashdeep -a -l -v -v -k 241561_8497448424664183398.manifest.auditfile -r 8497448424664183398`
13
- - `mango to-torrent <manifest file> <file hash>`
11
+ - Example verification usage: `hashdeep -a -l -v -v -k 241561_8497448424664183398.auditfile -r 8497448424664183398`
12
+ - `mango to-torrent <depot id> <manifest id> <file hash>`
14
13
  - `mango verify-chunks <depot id> [depot key]`
14
+ - `mango download-manifest <depot id> <manifest id>`
package/lib.js CHANGED
@@ -101,25 +101,33 @@ module.exports = {
101
101
  getFileContents,
102
102
  getChunk,
103
103
  fetchManifest: async (depotId, manifestId) => {
104
+ try {
105
+ return await fsPromises.readFile(`manifests/${depotId}_${manifestId}.manifest`);
106
+ }
107
+ catch (e) {
108
+ // fallthrough
109
+ }
104
110
  const forkers = [
105
111
  "qwe213312",
106
112
  "mejikuhibiniu1",
107
113
  "Sainan",
108
- "FreakyObservatory",
114
+ //"FreakyObservatory", // Seems inactive now and seems to have no manifests missing in the above.
109
115
  ];
110
116
  for (const forker of forkers) {
111
117
  const res = await fetch(`https://raw.githubusercontent.com/${forker}/k25FCdfEOoEJ42S6/refs/heads/main/${depotId}_${manifestId}.manifest`);
112
118
  if (res.status == 200) {
113
- return await res.arrayBuffer();
119
+ const ab = await res.arrayBuffer();
120
+ await fsPromises.mkdir(`manifests`, { recursive: true });
121
+ await fsPromises.writeFile(`manifests/${depotId}_${manifestId}.manifest`, Buffer.from(ab));
122
+ return ab;
114
123
  }
115
124
  }
125
+ throw new Error(`Could not find the given manifest (${manifestId}). Double-check with https://steamdb.info/depot/${depotId}/manifests/ and report an issue in https://github.com/Sainan/k25FCdfEOoEJ42S6/issues if you're sure the manifest exists.`);
116
126
  },
117
127
  fetchDepotKey: async (depotId) => {
118
- console.log(`Depot key was not supplied, attempting to fetch it...`);
119
128
  const depotkeys = await fetch("https://raw.githubusercontent.com/SteamAutoCracks/ManifestHub/refs/heads/main/depotkeys.json").then(x => x.json());
120
129
  if (!depotkeys[depotId]) {
121
- console.log(`Failed to get the depot key. Please provide it manually.`);
122
- process.exit(1);
130
+ throw new Error(`Failed to get the depot key.`);
123
131
  }
124
132
  return depotkeys[depotId];
125
133
  },
package/mango.js CHANGED
@@ -5,76 +5,67 @@ switch (tool) {
5
5
  process.exit(1);
6
6
  } break;
7
7
 
8
- case "download-manifest": {
9
- const depotId = process.argv[1 + 2];
10
- const manifestId = process.argv[1 + 3];
8
+ case "download-manifest": { // No need to use this manually as all commands that need a manifest will fetch it on demand.
9
+ const depotId = process.argv[3];
10
+ const manifestId = process.argv[4];
11
11
  if (!depotId || !manifestId) {
12
12
  console.log("Syntax: mango download-manifest <depot id> <manifest id>");
13
13
  process.exit(1);
14
14
  }
15
15
 
16
- const fsPromises = require("fs/promises");
17
- const { fetchManifest } = require(".");
16
+ const { fetchManifest } = require("./lib.js");
18
17
 
19
18
  (async () => {
20
- const ab = await fetchManifest(depotId, manifestId);
21
- if (ab) {
22
- await fsPromises.writeFile(`${depotId}_${manifestId}.manifest`, Buffer.from(ab));
23
- console.log(`Saved in ${depotId}_${manifestId}.manifest`);
24
- process.exit(0);
25
- }
26
- else {
27
- console.log(`Could not find the given manifest. Double-check with https://steamdb.info/depot/${depotId}/manifests/ and report an issue in https://github.com/Sainan/k25FCdfEOoEJ42S6/issues if you're sure the manifest exists.`);
28
- process.exit(1);
29
- }
19
+ await fetchManifest(depotId, manifestId);
20
+ console.log(`Saved in manifests/${depotId}_${manifestId}.manifest`);
30
21
  })();
31
22
  } break;
32
23
 
33
24
  case "download-chunks": {
34
- let manifestFile = process.argv[1 + 2];
35
- let lancache = (process.argv[1 + 3] == "--lancache");
36
- if (manifestFile == "--lancache") {
37
- lancache = true;
38
- manifestFile = process.argv[1 + 3];
39
- }
40
- if (!manifestFile) {
41
- console.log("Syntax: mango download-chunks <manifest file> [--lancache]");
25
+ let depotId = process.argv[3];
26
+ let manifestId = process.argv[4];
27
+ let lancache = (process.argv[5] == "--lancache");
28
+ if (!depotId || !manifestId) {
29
+ console.log("Syntax: mango download-chunks <depot id> <manifest id> [--lancache]");
42
30
  process.exit(1);
43
31
  }
44
-
45
- const fs = require("fs");
32
+
46
33
  const ContentManifest = require("steam-user/components/content_manifest");
47
- const { DEFAULT_HOSTS, downloadChunks } = require(".");
48
-
34
+ const { DEFAULT_HOSTS, fetchManifest, downloadChunks } = require("./lib.js");
35
+
49
36
  const hosts = lancache ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
50
-
51
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
52
- downloadChunks(
53
- manifest,
54
- undefined /*(num_chunks) => {}*/,
55
- (path, host) => { console.log(`${path}: Downloading from ${host}`); },
56
- (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
57
- (path, err) => { console.log(`${path}: `, err); },
58
- hosts
59
- ).then(() => process.exit(0));
37
+
38
+ (async () => {
39
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
40
+ await downloadChunks(
41
+ manifest,
42
+ undefined /*(num_chunks) => {}*/,
43
+ (path, host) => { console.log(`${path}: Downloading from ${host}`); },
44
+ (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
45
+ (path, err) => { console.log(`${path}: `, err); },
46
+ hosts
47
+ )
48
+ process.exit(0);
49
+ })();
60
50
  } break;
61
51
 
62
52
  case "populate-chunks": {
63
- const manifestFile = process.argv[1 + 2];
64
- const installDir = process.argv[1 + 3];
65
- let depotKey = process.argv[1 + 4];
66
- if (!manifestFile || !installDir) {
67
- console.log("Syntax: mango populate-chunks <manifest file> <install dir> [depot key]");
53
+ const depotId = process.argv[3];
54
+ const manifestId = process.argv[4];
55
+ const installDir = process.argv[5];
56
+ let depotKey = process.argv[6];
57
+ if (!depotId || !manifestId || !installDir) {
58
+ console.log("Syntax: mango populate-chunks <depot id> <manifest id> <install dir> [depot key]");
68
59
  process.exit(1);
69
60
  }
70
-
71
- const fs = require("fs");
61
+
72
62
  const ContentManifest = require("steam-user/components/content_manifest");
73
- const { fetchDepotKey, populateChunks } = require(".");
74
-
63
+ const { fetchManifest, fetchDepotKey, populateChunks } = require("./lib.js");
64
+
75
65
  (async () => {
76
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
66
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
77
67
  if (!depotKey) {
68
+ console.log(`Depot key was not supplied, attempting to fetch it...`);
78
69
  depotKey = await fetchDepotKey(manifest.depot_id);
79
70
  }
80
71
  depotKey = Buffer.from(depotKey, "hex");
@@ -94,16 +85,16 @@ switch (tool) {
94
85
  const express = require("express");
95
86
  const net = require("net");
96
87
  const sni = require("sni");
97
-
88
+
98
89
  const app = express();
99
-
90
+
100
91
  app.use((req, res, next) => {
101
92
  res.on("finish", () => {
102
93
  console.log(`${req.socket.remoteAddress} - ${req.url} - ${res.statusCode}`);
103
94
  });
104
95
  next();
105
96
  });
106
-
97
+
107
98
  app.get("/depot/:depotId/chunk/:sha", async (req, res) => {
108
99
  if (fs.existsSync(__dirname + req.url)) {
109
100
  res.sendFile(__dirname + req.url);
@@ -124,17 +115,17 @@ switch (tool) {
124
115
  res.status(400).send("Don't have this chunk").end();
125
116
  }
126
117
  });
127
-
118
+
128
119
  app.use((req, res) => {
129
120
  res.status(400).send("Don't know this endpoint").end();
130
121
  });
131
-
122
+
132
123
  app.listen(80, () => {
133
124
  console.log("Listening on port 80");
134
125
  });
135
-
126
+
136
127
  // Transparently forward traffic on port 443 to avoid breaking HTTPS connections
137
-
128
+
138
129
  net.createServer(socket => {
139
130
  socket.once("data", firstPacket => {
140
131
  const hostname = sni(firstPacket);
@@ -155,20 +146,21 @@ switch (tool) {
155
146
  } break;
156
147
 
157
148
  case "install": {
158
- const manifestFile = process.argv[1 + 2];
159
- let depotKey = process.argv[1 + 3];
160
- if (!manifestFile) {
161
- console.log("Syntax: mango install <manifest file> [depot key]");
149
+ const depotId = process.argv[3];
150
+ const manifestId = process.argv[4];
151
+ let depotKey = process.argv[5];
152
+ if (!depotId || !manifestId) {
153
+ console.log("Syntax: mango install <depot id> <manifest id> [depot key]");
162
154
  process.exit(1);
163
155
  }
164
-
165
- const fs = require("fs");
156
+
166
157
  const ContentManifest = require("steam-user/components/content_manifest");
167
- const { fetchDepotKey, install } = require(".");
168
-
158
+ const { fetchManifest, fetchDepotKey, install } = require("./lib.js");
159
+
169
160
  (async () => {
170
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
161
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
171
162
  if (!depotKey) {
163
+ console.log(`Depot key was not supplied, attempting to fetch it...`);
172
164
  depotKey = await fetchDepotKey(manifest.depot_id);
173
165
  }
174
166
  depotKey = Buffer.from(depotKey, "hex");
@@ -180,57 +172,62 @@ switch (tool) {
180
172
  } break;
181
173
 
182
174
  case "to-json": {
183
- const fs = require("fs");
184
- const ContentManifest = require("steam-user/components/content_manifest");
185
-
186
- const file = process.argv[1 + 2];
187
- const depotKey = process.argv[1 + 3];
188
- if (!file) {
189
- console.log("Syntax: mango to-json <manifest file> [depot key]");
175
+ const depotId = process.argv[3];
176
+ const manifestId = process.argv[4];
177
+ let depotKey = process.argv[5];
178
+ if (!depotId || !manifestId) {
179
+ console.log("Syntax: mango to-json <depot id> <manifest id> [depot key|--no-decrypt]");
190
180
  process.exit(1);
191
181
  }
192
-
193
- const buf = fs.readFileSync(file);
194
- const manifest = ContentManifest.parse(buf);
195
- if (manifest.filenames_encrypted) {
196
- if (depotKey) {
182
+
183
+ const fs = require("fs");
184
+ const ContentManifest = require("steam-user/components/content_manifest");
185
+ const { fetchManifest, fetchDepotKey } = require("./lib.js");
186
+
187
+ (async () => {
188
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
189
+ if (manifest.filenames_encrypted && depotKey != "--no-decrypt") {
190
+ if (!depotKey) {
191
+ console.log("Manifest has encrypted filenames and --no-decrypt was not specified. A depot key will be needed.");
192
+ console.log(`Depot key was not supplied, attempting to fetch it...`);
193
+ depotKey = await fetchDepotKey(manifest.depot_id);
194
+ }
197
195
  ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex")); // Sets filenames_encrypted to false
198
196
  //manifest.filenames_decrypted = true; // Indicate that this transformation took place in the JSON export // Kinda pointless because some .manifest files are already decrypted by their sources
199
- } else {
200
- console.log("Manifest has encrypted filenames, suggest supplying depot key");
201
197
  }
202
-
203
- }
204
- fs.writeFileSync(file + ".json", JSON.stringify(manifest, null, 2));
205
-
206
- process.exit(0);
198
+ fs.writeFileSync(`${depotId}_${manifestId}.json`, JSON.stringify(manifest, null, 2));
199
+ console.log(`Saved in ${depotId}_${manifestId}.json`);
200
+
201
+ process.exit(0);
202
+ })();
207
203
  } break;
208
204
 
209
205
  case "to-hashdeep-auditfile": {
206
+ const depotId = process.argv[3];
207
+ const manifestId = process.argv[4];
208
+ let depotKey = process.argv[5];
209
+ if (!depotId || !manifestId) {
210
+ console.log("Syntax: mango to-hashdeep-auditfile <depot id> <manifest id> [depot key]");
211
+ process.exit(1);
212
+ }
213
+
210
214
  const fs = require("fs");
211
215
  const path = require("path");
212
216
  const ContentManifest = require("steam-user/components/content_manifest");
213
-
214
- const file = process.argv[1 + 2];
215
- let depotKey = process.argv[1 + 3];
216
- if (!file) {
217
- console.log("Syntax: mango to-hashdeep-auditfile <manifest file> [depot key]");
218
- process.exit(1);
219
- }
220
-
217
+ const { fetchManifest, fetchDepotKey } = require("./lib.js");
218
+
221
219
  (async () => {
222
- const buf = fs.readFileSync(file);
223
- const manifest = ContentManifest.parse(buf);
220
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
224
221
  if (manifest.filenames_encrypted) {
225
222
  if (!depotKey) {
226
223
  console.log("Manifest has encrypted filenames. A depot key will be needed.");
227
- const { fetchDepotKey } = require(".");
224
+ console.log(`Depot key was not supplied, attempting to fetch it...`);
228
225
  depotKey = await fetchDepotKey(manifest.depot_id);
229
226
  }
230
227
  ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex"));
231
228
  }
232
-
233
- const fh = fs.createWriteStream(`${file}.auditfile`);
229
+
230
+ const fh = fs.createWriteStream(`${depotId}_${manifestId}.auditfile`);
234
231
  fh.write("%%%% HASHDEEP-1.0\n");
235
232
  fh.write("%%%% size,sha1,filename\n");
236
233
  for (const file of manifest.files) {
@@ -239,15 +236,25 @@ switch (tool) {
239
236
  }
240
237
  }
241
238
  fh.end();
242
-
243
- process.exit(0);
239
+ fh.on("finish", () => {
240
+ console.log(`Saved in ${depotId}_${manifestId}.auditfile`);
241
+
242
+ process.exit(0);
243
+ });
244
244
  })();
245
245
  } break;
246
246
 
247
247
  case "to-torrent": {
248
- const fs = require("fs");
248
+ const depotId = process.argv[3];
249
+ const manifestId = process.argv[4];
250
+ const fileHash = process.argv[5];
251
+ if (!depotId || !manifestId || !fileHash) {
252
+ console.log("Syntax: mango to-torrent <depot id> <manifest id> <file hash>");
253
+ process.exit(1);
254
+ }
255
+
249
256
  const ContentManifest = require("steam-user/components/content_manifest");
250
-
257
+
251
258
  function toPieces(hexArray) {
252
259
  const bufs = hexArray.map(h => {
253
260
  if (typeof h !== "string" || h.length !== 40) {
@@ -257,48 +264,54 @@ switch (tool) {
257
264
  });
258
265
  return Buffer.concat(bufs);
259
266
  }
260
-
261
- const manifestFile = process.argv[1 + 2];
262
- const fileHash = process.argv[1 + 3];
263
- if (!manifestFile || !fileHash) {
264
- console.log("Syntax: mango to-torrent <manifest file> <file hash>");
265
- process.exit(1);
266
- }
267
- const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
268
- const file = manifest.files.find(x => x.sha_content == fileHash);
269
- if (!file) {
270
- console.log(`No file with content hash ${fileHash} found in ${manifestFile}`);
271
- process.exit(1);
272
- }
273
- import("parse-torrent").then(({ toTorrentFile }) => {
267
+
268
+ (async () => {
269
+ const { fetchManifest } = require("./lib.js");
270
+ const manifest = ContentManifest.parse(await fetchManifest(depotId, manifestId));
271
+ const file = manifest.files.find(x => x.sha_content == fileHash);
272
+ if (!file) {
273
+ console.log(`No file with content hash ${fileHash} found in ${depotId}_${manifestId}.manifest`);
274
+ process.exit(1);
275
+ }
274
276
  for (const chunk of file.chunks) {
275
277
  chunk.offset = parseInt(chunk.offset);
276
278
  }
277
279
  file.chunks.sort((a, b) => a.offset - b.offset);
280
+ let pieceSize = file.chunks[0].cb_original;
281
+ for (let i = 1; i < file.chunks.length - 1; ++i) {
282
+ if (pieceSize != file.chunks[i].cb_original) {
283
+ console.log(`This file has non-uniformly sized chunks and is therefore unrepresentable in BitTorrent.`);
284
+ process.exit(2);
285
+ }
286
+ }
287
+ const { toTorrentFile } = await import("parse-torrent");
278
288
  const buf = toTorrentFile({
279
289
  info: {
280
290
  name: file.sha_content,
281
291
  length: parseInt(file.size),
282
- 'piece length': 1048576,
292
+ 'piece length': pieceSize,
283
293
  pieces: toPieces(file.chunks.map(chunk => chunk.sha))
284
294
  }
285
- })
295
+ });
296
+ const fs = require("node:fs");
286
297
  fs.writeFileSync(`${file.sha_content}.torrent`, buf);
298
+ console.log(`Saved in ${file.sha_content}.torrent`);
287
299
  process.exit(0);
288
- });
300
+ })();
289
301
  } break;
290
302
 
291
303
  case "verify-chunks": {
292
- const depotId = process.argv[1 + 2];
293
- let depotKey = process.argv[1 + 3];
304
+ const depotId = process.argv[3];
305
+ let depotKey = process.argv[4];
294
306
  if (!depotId) {
295
307
  console.log("Syntax: mango verify-chunks <depot id> [depot key]");
296
308
  process.exit(1);
297
309
  }
298
310
 
299
- const { fetchDepotKey, verifyChunks } = require(".");
311
+ const { fetchDepotKey, verifyChunks } = require("./lib.js");
300
312
  (async () => {
301
313
  if (!depotKey) {
314
+ console.log(`Depot key was not supplied, attempting to fetch it...`);
302
315
  depotKey = await fetchDepotKey(depotId);
303
316
  }
304
317
  depotKey = Buffer.from(depotKey, "hex");
package/package.json CHANGED
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "steam-manifest-tools",
3
- "version": "0.1.4",
3
+ "version": "0.1.5",
4
4
  "main": "lib.js",
5
5
  "repository": {
6
6
  "type": "git",
7
7
  "url": "git+https://github.com/Sainan/steam-manifest-tools.git"
8
8
  },
9
9
  "scripts": {
10
- "build": "bun build mango.js --compile"
10
+ "build": "mv node_modules nm && nexe mango.js -t x64-22.21.1 --build && mv nm node_modules"
11
11
  },
12
12
  "dependencies": {
13
13
  "@doctormckay/steam-crypto": "^1.2.0",