steam-manifest-tools 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md ADDED
@@ -0,0 +1,13 @@
1
+ Scripts to interact with Steam manifest files.
2
+ - `node download-manifest.js <depot id> <manifest id>`
3
+ - `node download-chunks.js <manifest file> [--lancache]` (basically the same as prefilling a LAN cache, optionally downloads from a LAN cache)
4
+ - `node populate-chunks.js <manifest file> <install dir> [depot key]` (can also be used to prefill a LAN cache, tho compression is not as good as Steam's)
5
+ - `node lancache.js`
6
+ - Must run as root to obtain port 80
7
+ - `lancache.steamcontent.com` must resolve to your LAN cache server IP (either via custom DNS server or OS hosts file)
8
+ - `node install.js <manifest file> [depot key]` (requires chunks to have been populated and/or downloaded beforehand)
9
+ - `node to-json.js <manifest file> [depot key]`
10
+ - `node to-hashdeep-auditfile.js <manifest file> [depot key]`
11
+ - Note that hashdeep requires exact path matches and this script makes a few assumptions
12
+ - Example verification usage: `hashdeep -a -l -v -v -k 241561_8497448424664183398.manifest.auditfile -r 8497448424664183398`
13
+ - `node to-torrent.js <manifest file> <file hash>`
@@ -0,0 +1,27 @@
1
+ let manifestFile = process.argv[2];
2
+ let lancache = (process.argv[3] == "--lancache");
3
+ if (manifestFile == "--lancache") {
4
+ lancache = true;
5
+ manifestFile = process.argv[3];
6
+ }
7
+ if (!manifestFile) {
8
+ console.log("Syntax: node download-chunks.js <manifest file> [--lancache]");
9
+ process.exit(1);
10
+ }
11
+
12
+ const fs = require("fs");
13
+ const fsPromises = require("fs/promises");
14
+ const ContentManifest = require("steam-user/components/content_manifest");
15
+ const { DEFAULT_HOSTS, downloadChunks } = require(".");
16
+
17
+ const hosts = lancache ? ["http://lancache.steamcontent.com"] : DEFAULT_HOSTS;
18
+
19
+ const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
20
+ downloadChunks(
21
+ manifest,
22
+ undefined /*(num_chunks) => {}*/,
23
+ (path, host) => { console.log(`${path}: Downloading from ${host}`); },
24
+ (path, status, host) => { console.log(`${path}: Got ${status/*} from ${host*/}`); },
25
+ (path, err) => { console.log(`${path}: `, err); },
26
+ hosts
27
+ );
@@ -0,0 +1,28 @@
1
+ const depotId = process.argv[2];
2
+ const manifestId = process.argv[3];
3
+ if (!depotId || !manifestId) {
4
+ console.log("Syntax: node download-manifest.js <depot id> <manifest id>");
5
+ process.exit(1);
6
+ }
7
+
8
+ const fsPromises = require("fs/promises");
9
+
10
+ const forkers = [
11
+ "qwe213312",
12
+ "mejikuhibiniu1",
13
+ "Sainan",
14
+ "FreakyObservatory",
15
+ ];
16
+ (async () => {
17
+ let i = 0;
18
+ for (const forker of forkers) {
19
+ const res = await fetch(`https://raw.githubusercontent.com/${forker}/k25FCdfEOoEJ42S6/refs/heads/main/${depotId}_${manifestId}.manifest`);
20
+ if (res.status == 200) {
21
+ const ab = await res.arrayBuffer();
22
+ await fsPromises.writeFile(`${depotId}_${manifestId}.manifest`, Buffer.from(ab));
23
+ console.log(`Saved in ${depotId}_${manifestId}.manifest`);
24
+ return;
25
+ }
26
+ }
27
+ console.log(`Could not find the given manifest. Double-check with https://steamdb.info/depot/${depotId}/manifests/ and report an issue in https://github.com/Sainan/k25FCdfEOoEJ42S6/issues if you're sure the manifest exists.`);
28
+ })();
@@ -0,0 +1,9 @@
1
+ module.exports = async (depotId) => {
2
+ console.log(`Depot key was not supplied, attempting to fetch it...`);
3
+ const depotkeys = await fetch("https://raw.githubusercontent.com/SteamAutoCracks/ManifestHub/refs/heads/main/depotkeys.json").then(x => x.json());
4
+ if (!depotkeys[depotId]) {
5
+ console.log(`Failed to get the depot key. Please provide it manually.`);
6
+ process.exit(1);
7
+ }
8
+ return depotkeys[depotId];
9
+ };
package/include/lib.js ADDED
@@ -0,0 +1,217 @@
1
+ const crypto = require("crypto");
2
+ const fs = require("fs");
3
+ const fsPromises = require("fs/promises");
4
+ const path = require("path");
5
+ const AdmZip = require("adm-zip");
6
+ const CdnCompression = require("steam-user/components/cdn_compression");
7
+ const ContentManifest = require("steam-user/components/content_manifest");
8
+ const SteamCrypto = require("@doctormckay/steam-crypto");
9
+
10
+ const DEFAULT_HOSTS = [
11
+ "https://cache8-sto1.steamcontent.com",
12
+ "https://cache6-sto1.steamcontent.com",
13
+ "https://cache1-sto1.steamcontent.com",
14
+ "https://cache2-sto1.steamcontent.com",
15
+ "https://cache3-sto1.steamcontent.com",
16
+ "https://cache7-sto1.steamcontent.com",
17
+ "https://cache2-sto2.steamcontent.com",
18
+ "https://cache6-sto2.steamcontent.com",
19
+ "https://cache1-sto2.steamcontent.com",
20
+ "https://cache9-sto1.steamcontent.com",
21
+ "https://cache4-sto2.steamcontent.com",
22
+ "https://cache4-sto1.steamcontent.com",
23
+ "https://cache5-sto1.steamcontent.com",
24
+ "https://cache3-sto2.steamcontent.com",
25
+ "https://cache5-sto2.steamcontent.com",
26
+ "http://alibaba.cdn.steampipe.steamcontent.com",
27
+ "http://edgenext.cdn.steampipe.steamcontent.com",
28
+ "https://steampipe.akamaized.net",
29
+ "https://fastly.cdn.steampipe.steamcontent.com",
30
+ "https://google2.cdn.steampipe.steamcontent.com",
31
+ ];
32
+
33
+ const sha1 = (data) => crypto.createHash("sha1").update(data).digest("hex");
34
+
35
+ const sha1file = async (file) => {
36
+ const st = crypto.createHash("sha1");
37
+ const readStream = fs.createReadStream(file, { highWaterMark: 1048576 });
38
+ for await (const chunk of readStream) {
39
+ st.update(chunk);
40
+ }
41
+ return st.digest("hex");
42
+ };
43
+
44
+ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
45
+
46
+ const compress = (buf) => {
47
+ const zip = new AdmZip();
48
+ zip.addFile("z", buf);
49
+ return zip.toBuffer();
50
+ };
51
+
52
+ // https://stackoverflow.com/a/45130990
53
+ const getFiles = async (dir) => {
54
+ const dirents = await fsPromises.readdir(dir, { withFileTypes: true });
55
+ const files = await Promise.all(dirents.map((dirent) => {
56
+ const res = path.resolve(dir, dirent.name);
57
+ return dirent.isDirectory() ? getFiles(res) : res;
58
+ }));
59
+ return Array.prototype.concat(...files);
60
+ };
61
+
62
+ const getChunk = async (depotId, depotKey, hash) => {
63
+ while (true) {
64
+ try {
65
+ let data = await fsPromises.readFile(`depot/${depotId}/chunk/${hash}`);
66
+ data = SteamCrypto.symmetricDecrypt(data, depotKey);
67
+ data = await CdnCompression.unzip(data);
68
+ if (sha1(data) != hash) {
69
+ throw new Error(`depot/${depotId}/chunk/${hash} does not match the expected hash`);
70
+ }
71
+ return data;
72
+ } catch (e) {
73
+ if (e.code == "ENOENT") {
74
+ throw new Error(`depot/${depotId}/chunk/${hash} is missing`);
75
+ }
76
+ if (e.code != "EMFILE") {
77
+ //console.log(`Error reading depot/${depotId}/chunk/${hash}`);
78
+ throw e;
79
+ }
80
+ await sleep(500);
81
+ }
82
+ }
83
+ };
84
+
85
+ module.exports = {
86
+ DEFAULT_HOSTS,
87
+ sha1,
88
+ sha1file,
89
+ compress,
90
+ getFiles,
91
+ getChunk,
92
+ populateChunks: async (manifest, depotKey, installDir, onProgress) => {
93
+ if (manifest.filenames_encrypted) {
94
+ ContentManifest.decryptFilenames(manifest, depotKey);
95
+ }
96
+ fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
97
+ let file_i = 0;
98
+ for (const file of manifest.files) {
99
+ if (file.flags & 64) {
100
+ continue;
101
+ }
102
+ if (fs.existsSync(path.join(installDir, file.filename))) {
103
+ const readSteam = await fsPromises.open(path.join(installDir, file.filename), "r");
104
+ let chunk_i = 0;
105
+ for (const chunk of file.chunks) {
106
+ if (!fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
107
+ let chunkBuf = Buffer.alloc(chunk.cb_original);
108
+ const { bytesRead } = await readSteam.read(chunkBuf, 0, chunk.cb_original, parseInt(chunk.offset));
109
+ if (bytesRead != chunk.cb_original) {
110
+ break;
111
+ }
112
+ if (sha1(chunkBuf) == chunk.sha) {
113
+ chunkBuf = compress(chunkBuf);
114
+ chunkBuf = SteamCrypto.symmetricEncrypt(chunkBuf, depotKey);
115
+ await fsPromises.writeFile(`depot/${manifest.depot_id}/chunk/${chunk.sha}`, chunkBuf);
116
+ if (onProgress) {
117
+ onProgress(file.filename, file_i, manifest.files.length, chunk_i++, file.chunks.length);
118
+ }
119
+ }
120
+ }
121
+ }
122
+ await readSteam.close();
123
+ }
124
+ ++file_i;
125
+ }
126
+ },
127
+ downloadChunks: (manifest, onStartDownloading, onStartDownload, onFinishDownload, onErroredDownload, hosts) => {
128
+ hosts ??= DEFAULT_HOSTS;
129
+ return new Promise(resolve => {
130
+ fs.mkdirSync(`depot/${manifest.depot_id}/chunk`, { recursive: true });
131
+ const toDownload = {};
132
+ for (const file of manifest.files) {
133
+ for (const chunk of file.chunks) {
134
+ if (!fs.existsSync(`depot/${manifest.depot_id}/chunk/${chunk.sha}`)) {
135
+ toDownload[`depot/${manifest.depot_id}/chunk/${chunk.sha}`] = true;
136
+ }
137
+ }
138
+ }
139
+ let remaining_chunks = Object.keys(toDownload).length;
140
+ if (onStartDownloading) {
141
+ onStartDownloading(remaining_chunks);
142
+ }
143
+ if (remaining_chunks == 0) {
144
+ resolve();
145
+ }
146
+ let /*host_i = 0,*/ running = 0;
147
+ const loop = () => {
148
+ while (running < 4) {
149
+ const path = Object.keys(toDownload)[0];
150
+ if (!path) {
151
+ break;
152
+ }
153
+ ++running;
154
+ delete toDownload[path];
155
+ const host = hosts[Math.floor(Math.random()*hosts.length)];
156
+ //const host = hosts[host_i]; host_i = (host_i + 1) % hosts.length;
157
+ if (onStartDownload) {
158
+ onStartDownload(path, host);
159
+ }
160
+ fetch(`${host}/${path}`).then(async res => {
161
+ if (onFinishDownload) {
162
+ onFinishDownload(path, res.status, host);
163
+ }
164
+ if (res.status == 200) {
165
+ const ab = await res.arrayBuffer();
166
+ await fsPromises.writeFile(path, Buffer.from(ab));
167
+ if (--remaining_chunks == 0) {
168
+ resolve();
169
+ }
170
+ }
171
+ else {
172
+ toDownload[path] = true;
173
+ }
174
+ }).catch(err => {
175
+ if (onErroredDownload) {
176
+ onErroredDownload(path, err);
177
+ }
178
+ toDownload[path] = true;
179
+ }).finally(() => {
180
+ --running;
181
+ loop();
182
+ });
183
+ }
184
+ };
185
+ loop();
186
+ });
187
+ },
188
+ install: async (manifest, depotKey, installDir, onFileWritten) => {
189
+ ContentManifest.decryptFilenames(manifest, depotKey);
190
+ installDir ??= `install/${manifest.depot_id}/${manifest.gid_manifest}`;
191
+ fs.mkdirSync(installDir, { recursive: true });
192
+ for (const file of manifest.files) {
193
+ if (file.flags & 64) {
194
+ continue;
195
+ }
196
+ const exists = fs.existsSync(path.join(installDir, file.filename));
197
+ if (!exists || await sha1file(path.join(installDir, file.filename)) != file.sha_content) {
198
+ for (const chunk of file.chunks) {
199
+ chunk.offset = parseInt(chunk.offset);
200
+ }
201
+ file.chunks.sort((a, b) => a.offset - b.offset);
202
+
203
+ const chunkPromises = [];
204
+ for (const chunk of file.chunks) {
205
+ chunkPromises.push(getChunk(manifest.depot_id, depotKey, chunk.sha));
206
+ }
207
+ const chunks = await Promise.all(chunkPromises);
208
+ const data = Buffer.concat(chunks);
209
+ await fsPromises.mkdir(path.dirname(path.join(installDir, file.filename)), { recursive: true });
210
+ await fsPromises.writeFile(path.join(installDir, file.filename), data);
211
+ if (onFileWritten) {
212
+ onFileWritten(file.filename, exists);
213
+ }
214
+ }
215
+ }
216
+ },
217
+ };
package/install.js ADDED
@@ -0,0 +1,21 @@
1
+ const manifestFile = process.argv[2];
2
+ let depotKey = process.argv[3];
3
+ if (!manifestFile) {
4
+ console.log("Syntax: node install.js <manifest file> [depot key]");
5
+ process.exit(1);
6
+ }
7
+
8
+ const fs = require("fs");
9
+ const ContentManifest = require("steam-user/components/content_manifest");
10
+ const { install } = require(".");
11
+
12
+ (async () => {
13
+ const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
14
+ if (!depotKey) {
15
+ depotKey = await require("./include/fetch-depot-key.js")(manifest.depot_id);
16
+ }
17
+ depotKey = Buffer.from(depotKey, "hex");
18
+ await install(manifest, depotKey, undefined, (file, existed) => {
19
+ console.log(`${existed ? "Repaired" : "Created"} ${file}`);
20
+ });
21
+ })();
package/lancache.js ADDED
@@ -0,0 +1,60 @@
1
+ const fs = require("fs");
2
+ const fsPromises = require("fs/promises");
3
+ const path = require("path");
4
+ const express = require("express");
5
+ const net = require("net");
6
+ const sni = require("sni");
7
+
8
+ const app = express();
9
+
10
+ app.use((req, res, next) => {
11
+ res.on("finish", () => {
12
+ console.log(`${req.socket.remoteAddress} - ${req.url} - ${res.statusCode}`);
13
+ });
14
+ next();
15
+ });
16
+
17
+ app.get("/depot/:depotId/chunk/:sha", async (req, res) => {
18
+ if (fs.existsSync(__dirname + req.url)) {
19
+ res.sendFile(__dirname + req.url);
20
+ }
21
+ else if (req.host.endsWith(".steamcontent.com")) {
22
+ const fr = await fetch(`http://${req.host}${req.url}`);
23
+ if (fr.status == 200) {
24
+ const buf = Buffer.from(await fr.arrayBuffer());
25
+ await fsPromises.mkdir(path.dirname(__dirname + req.url), { recursive: true });
26
+ await fsPromises.writeFile(__dirname + req.url, buf);
27
+ res.send(buf).end();
28
+ }
29
+ else {
30
+ res.status(fr.status).send("Upstream error").end();
31
+ }
32
+ }
33
+ else {
34
+ res.status(400).send("Don't have this chunk").end();
35
+ }
36
+ });
37
+
38
+ app.use((req, res) => {
39
+ res.status(400).send("Don't know this endpoint").end();
40
+ });
41
+
42
+ app.listen(80, () => {
43
+ console.log("Listening on port 80");
44
+ });
45
+
46
+ // Transparently forward traffic on port 443 to avoid breaking HTTPS connections
47
+
48
+ net.createServer(socket => {
49
+ socket.once("data", firstPacket => {
50
+ const hostname = sni(firstPacket);
51
+ console.log(`${socket.remoteAddress} - Starting TLS proxy to ${hostname}`);
52
+ const upstream = net.connect(443, hostname, () => {
53
+ upstream.write(firstPacket);
54
+ socket.pipe(upstream).pipe(socket);
55
+ });
56
+ upstream.on("error", () => socket.end());
57
+ });
58
+ }).listen(443, () => {
59
+ console.log("Listening on port 443");
60
+ });
package/package.json ADDED
@@ -0,0 +1,15 @@
1
+ {
2
+ "name": "steam-manifest-tools",
3
+ "version": "0.1.0",
4
+ "main": "include/lib.js",
5
+ "dependencies": {
6
+ "@doctormckay/steam-crypto": "^1.2.0",
7
+ "adm-zip": "^0.5.16",
8
+ "steam-user": "^5.2.3"
9
+ },
10
+ "devDependencies": {
11
+ "express": "^5.1.0",
12
+ "parse-torrent": "^11.0.18",
13
+ "sni": "^1.0.0"
14
+ }
15
+ }
@@ -0,0 +1,22 @@
1
+ const manifestFile = process.argv[2];
2
+ const installDir = process.argv[3];
3
+ let depotKey = process.argv[4];
4
+ if (!manifestFile || !installDir) {
5
+ console.log("Syntax: node populate-chunks.js <manifest file> <install dir> [depot key]");
6
+ process.exit(1);
7
+ }
8
+ (async () => {
9
+ const fs = require("fs");
10
+ const ContentManifest = require("steam-user/components/content_manifest");
11
+ const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
12
+ if (!depotKey) {
13
+ depotKey = await require("./include/fetch-depot-key.js")(manifest.depot_id);
14
+ }
15
+ depotKey = Buffer.from(depotKey, "hex");
16
+ const { populateChunks } = require(".");
17
+ await populateChunks(manifest, depotKey, installDir, (file, file_i, num_files, chunk_i, num_chunks) => {
18
+ if (chunk_i == 0) {
19
+ console.log(`Processing ${file}...`);
20
+ }
21
+ });
22
+ })();
@@ -0,0 +1,32 @@
1
+ const fs = require("fs");
2
+ const path = require("path");
3
+ const ContentManifest = require("steam-user/components/content_manifest");
4
+
5
+ const file = process.argv[2];
6
+ let depotKey = process.argv[3];
7
+ if (!file) {
8
+ console.log("Syntax: node to-hashdeep-auditfile.js <manifest file> [depot key]");
9
+ process.exit(1);
10
+ }
11
+
12
+ (async () => {
13
+ const buf = fs.readFileSync(file);
14
+ const manifest = ContentManifest.parse(buf);
15
+ if (manifest.filenames_encrypted) {
16
+ if (!depotKey) {
17
+ console.log("Manifest has encrypted filenames. A depot key will be needed.");
18
+ depotKey = await require("./include/fetch-depot-key.js")(manifest.depot_id);
19
+ }
20
+ ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex"));
21
+ }
22
+
23
+ const fh = fs.createWriteStream(`${file}.auditfile`);
24
+ fh.write("%%%% HASHDEEP-1.0\n");
25
+ fh.write("%%%% size,sha1,filename\n");
26
+ for (const file of manifest.files) {
27
+ if ((file.flags & 64) === 0) {
28
+ fh.write(`${file.size},${file.sha_content},${manifest.gid_manifest}${path.sep}${path.sep != '\\' ? file.filename.split('\\').join(path.sep) : file.filename.split('/').join(path.sep)}\n`);
29
+ }
30
+ }
31
+ fh.end();
32
+ })();
package/to-json.js ADDED
@@ -0,0 +1,22 @@
1
+ const fs = require("fs");
2
+ const ContentManifest = require("steam-user/components/content_manifest");
3
+
4
+ const file = process.argv[2];
5
+ const depotKey = process.argv[3];
6
+ if (!file) {
7
+ console.log("Syntax: node to-json.js <manifest file> [depot key]");
8
+ process.exit(1);
9
+ }
10
+
11
+ const buf = fs.readFileSync(file);
12
+ const manifest = ContentManifest.parse(buf);
13
+ if (manifest.filenames_encrypted) {
14
+ if (depotKey) {
15
+ ContentManifest.decryptFilenames(manifest, Buffer.from(depotKey, "hex")); // Sets filenames_encrypted to false
16
+ //manifest.filenames_decrypted = true; // Indicate that this transformation took place in the JSON export // Kinda pointless because some .manifest files are already decrypted by their sources
17
+ } else {
18
+ console.log("Manifest has encrypted filenames, suggest supplying depot key");
19
+ }
20
+
21
+ }
22
+ fs.writeFileSync(file + ".json", JSON.stringify(manifest, null, 2));
package/to-torrent.js ADDED
@@ -0,0 +1,40 @@
1
+ const fs = require("fs");
2
+ const ContentManifest = require("steam-user/components/content_manifest");
3
+
4
+ function toPieces(hexArray) {
5
+ const bufs = hexArray.map(h => {
6
+ if (typeof h !== "string" || h.length !== 40) {
7
+ throw new Error(`Invalid SHA1 hex string: ${h}`);
8
+ }
9
+ return Buffer.from(h, "hex");
10
+ });
11
+ return Buffer.concat(bufs);
12
+ }
13
+
14
+ const manifestFile = process.argv[2];
15
+ const fileHash = process.argv[3];
16
+ if (!manifestFile || !fileHash) {
17
+ console.log("Syntax: node to-torrent.js <manifest file> <file hash>");
18
+ process.exit(1);
19
+ }
20
+ const manifest = ContentManifest.parse(fs.readFileSync(manifestFile));
21
+ const file = manifest.files.find(x => x.sha_content == fileHash);
22
+ if (!file) {
23
+ console.log(`No file with content hash ${fileHash} found in ${manifestFile}`);
24
+ process.exit(1);
25
+ }
26
+ import("parse-torrent").then(({ toTorrentFile }) => {
27
+ for (const chunk of file.chunks) {
28
+ chunk.offset = parseInt(chunk.offset);
29
+ }
30
+ file.chunks.sort((a, b) => a.offset - b.offset);
31
+ const buf = toTorrentFile({
32
+ info: {
33
+ name: file.sha_content,
34
+ length: parseInt(file.size),
35
+ 'piece length': 1048576,
36
+ pieces: toPieces(file.chunks.map(chunk => chunk.sha))
37
+ }
38
+ })
39
+ fs.writeFileSync(`${file.sha_content}.torrent`, buf);
40
+ });