talon-agent 5.14.0 → 5.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -21
- package/LICENSE-MIT +21 -0
- package/NOTICE +16 -0
- package/README.md +14 -7
- package/package.json +6 -4
- package/prompts/system/heartbeat-agent.md +1 -1
- package/src/backend/agy/factory.ts +3 -0
- package/src/backend/agy/mcp/config.ts +14 -2
- package/src/backend/claude-sdk/factory.ts +3 -0
- package/src/backend/claude-sdk/options.ts +24 -3
- package/src/backend/codex/factory.ts +3 -0
- package/src/backend/codex/init.ts +4 -0
- package/src/backend/codex/mcp-config.ts +10 -0
- package/src/backend/codex/oauth-incompat.ts +1 -1
- package/src/backend/codex/token-usage.ts +2 -2
- package/src/backend/openai-agents/factory.ts +3 -0
- package/src/backend/openai-agents/mcp-pool.ts +4 -0
- package/src/backend/remote-server/factory.ts +3 -0
- package/src/backend/remote-server/mcp.ts +3 -0
- package/src/backend/runtime/prompt/prompt-format.ts +3 -3
- package/src/bootstrap.ts +8 -0
- package/src/cli/commands/backup.ts +61 -5
- package/src/cli/commands/mesh.ts +133 -0
- package/src/cli/config.ts +3 -1
- package/src/cli/daemon-api.ts +22 -0
- package/src/cli/index.ts +6 -0
- package/src/cli/install-sources.ts +40 -5
- package/src/cli/plugin.ts +10 -0
- package/src/cli/setup.ts +45 -4
- package/src/cli/skill.ts +3 -0
- package/src/core/agent-runtime/backend-registry.ts +16 -0
- package/src/core/backup/archive/crypt.ts +429 -0
- package/src/core/backup/archive/manifest-auth.ts +98 -0
- package/src/core/backup/passphrase.ts +130 -0
- package/src/core/backup/plan.ts +66 -13
- package/src/core/backup/restore-guard.ts +101 -0
- package/src/core/backup/restore.ts +249 -32
- package/src/core/backup/snapshot.ts +418 -60
- package/src/core/backup/sources/plugins.ts +223 -0
- package/src/core/backup/sources/relocate.ts +136 -0
- package/src/core/backup/sources/sessions.ts +198 -0
- package/src/core/backup/store.ts +3 -1
- package/src/core/backup/types.ts +66 -0
- package/src/core/backup/upload.ts +66 -6
- package/src/core/config/index.ts +140 -8
- package/src/core/daemon/control.ts +9 -0
- package/src/core/daemon/discovery.ts +7 -0
- package/src/core/engine/backend-router/headroom.ts +29 -5
- package/src/core/engine/backend-router/usage.ts +6 -2
- package/src/core/engine/gateway-actions/fetch-url/guard.ts +201 -0
- package/src/core/engine/gateway-actions/{fetch-url.ts → fetch-url/index.ts} +60 -32
- package/src/core/engine/gateway-actions/index.ts +4 -2
- package/src/core/engine/gateway-actions/native/index.ts +24 -0
- package/src/core/engine/gateway-actions/whatsapp-account.ts +1 -1
- package/src/core/engine/gateway-auth.ts +164 -0
- package/src/core/engine/gateway-routes.ts +100 -5
- package/src/core/engine/gateway.ts +12 -4
- package/src/core/mcp-hub/guest-scope.ts +170 -29
- package/src/core/mcp-hub/index.ts +33 -16
- package/src/core/mcp-hub/talon-server.ts +71 -14
- package/src/core/mesh/credentials/admin.ts +146 -0
- package/src/core/mesh/credentials/index.ts +19 -0
- package/src/core/mesh/credentials/store.ts +443 -0
- package/src/core/mesh/credentials/token.ts +45 -0
- package/src/core/mesh/credentials/types.ts +83 -0
- package/src/core/mesh/devices/service.ts +58 -6
- package/src/core/mesh/links/bridge-links.ts +46 -4
- package/src/core/mesh/links/node-binaries.ts +1 -1
- package/src/core/mesh/links/node-provision.ts +8 -1
- package/src/core/models/active-model.ts +1 -1
- package/src/core/plugin/loader.ts +4 -0
- package/src/core/plugin/mcp.ts +4 -0
- package/src/core/tools/bridge.ts +2 -1
- package/src/core/types.ts +13 -0
- package/src/core/weaver/weaver.ts +47 -15
- package/src/frontend/discord/callbacks/components/agent-buttons.ts +1 -0
- package/src/frontend/discord/handlers/delivery.ts +3 -0
- package/src/frontend/discord/handlers/queue.ts +1 -0
- package/src/frontend/native/bridge/auth-guard.ts +277 -0
- package/src/frontend/native/bridge/auth.ts +84 -1
- package/src/frontend/native/bridge/credentials/claims.ts +51 -0
- package/src/frontend/native/bridge/credentials/principal.ts +200 -0
- package/src/frontend/native/bridge/credentials/upgrade.ts +129 -0
- package/src/frontend/native/bridge/routes/auth.ts +37 -0
- package/src/frontend/native/bridge/routes/chats.ts +20 -2
- package/src/frontend/native/bridge/routes/host.ts +11 -1
- package/src/frontend/native/bridge/routes/index.ts +2 -0
- package/src/frontend/native/bridge/routes/mesh.ts +72 -15
- package/src/frontend/native/bridge/routes/table.ts +73 -51
- package/src/frontend/native/bridge/server.ts +266 -83
- package/src/frontend/native/index.ts +54 -3
- package/src/frontend/native/turn/turn.ts +2 -0
- package/src/frontend/teams/turn.ts +1 -0
- package/src/frontend/telegram/actions/outgoing-log.ts +70 -0
- package/src/frontend/telegram/actions/send.ts +4 -0
- package/src/frontend/telegram/admin.ts +20 -0
- package/src/frontend/telegram/commands/admin.ts +1 -1
- package/src/frontend/telegram/commands/state.ts +7 -9
- package/src/frontend/telegram/handlers/access.ts +31 -10
- package/src/frontend/telegram/handlers/delivery.ts +14 -1
- package/src/frontend/telegram/handlers/group-access.ts +50 -0
- package/src/frontend/telegram/handlers/messages.ts +1 -0
- package/src/frontend/telegram/handlers/queue.ts +14 -0
- package/src/frontend/telegram/handlers/state.ts +2 -2
- package/src/frontend/telegram/index.ts +32 -9
- package/src/frontend/telegram/middleware.ts +2 -2
- package/src/frontend/telegram/polling/poll-deadline.ts +52 -0
- package/src/frontend/telegram/{stale-command.ts → polling/stale-command.ts} +1 -1
- package/src/frontend/telegram/{update-offset.ts → polling/update-offset.ts} +1 -1
- package/src/frontend/telegram/userbot.ts +103 -10
- package/src/frontend/terminal/index.ts +8 -1
- package/src/frontend/whatsapp/commands.ts +3 -3
- package/src/frontend/whatsapp/messages/inbound.ts +1 -0
- package/src/plugins/playwright/index.ts +1 -1
- package/src/storage/backup/index.ts +1 -1
- package/src/storage/db.ts +23 -0
|
@@ -76,6 +76,14 @@ export interface BackendFactory {
|
|
|
76
76
|
id: string;
|
|
77
77
|
/** Display label used in `/status` and agent logs (e.g. "Kilo"). */
|
|
78
78
|
label: string;
|
|
79
|
+
/**
|
|
80
|
+
* Can this backend run a guest-scoped turn (a non-operator sender) with
|
|
81
|
+
* nothing but the hub's conversation-only tools? `"enforced"` means its
|
|
82
|
+
* own built-ins (shell, file tools) are dropped for guest turns;
|
|
83
|
+
* `"refused"` means they can't be, so the weaver refuses guest turns on
|
|
84
|
+
* it. Absent is treated as `"refused"` — fail closed.
|
|
85
|
+
*/
|
|
86
|
+
guestToolScope?: "enforced" | "refused";
|
|
79
87
|
/** Initialise the backend; called exactly once per Talon process. */
|
|
80
88
|
init(config: TalonConfig, ctx: BackendInitContext): Promise<BackendInstance>;
|
|
81
89
|
/**
|
|
@@ -136,6 +144,14 @@ export function clearBackends(): void {
|
|
|
136
144
|
backends.clear();
|
|
137
145
|
}
|
|
138
146
|
|
|
147
|
+
/**
|
|
148
|
+
* Whether guest-scoped turns may run on this backend. Unknown ids and
|
|
149
|
+
* factories that don't declare `guestToolScope` fail closed.
|
|
150
|
+
*/
|
|
151
|
+
export function backendEnforcesGuestScope(id: string): boolean {
|
|
152
|
+
return backends.get(id)?.guestToolScope === "enforced";
|
|
153
|
+
}
|
|
154
|
+
|
|
139
155
|
/** Whether a backend with this id is currently registered. */
|
|
140
156
|
export function hasBackend(id: string): boolean {
|
|
141
157
|
return backends.has(id);
|
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Encryption at rest — the form a part takes before it may leave the box.
|
|
3
|
+
*
|
|
4
|
+
* A part holds config.json (bot tokens, API keys), the secrets folder and
|
|
5
|
+
* private memory, so a remote copy must be unreadable without the
|
|
6
|
+
* operator's passphrase. Node's built-in crypto only:
|
|
7
|
+
*
|
|
8
|
+
* - Key: scrypt(passphrase, salt) → 32 bytes. N = 2^17, r = 8, p = 1
|
|
9
|
+
* (~128 MiB and a fraction of a second per part), a fresh random
|
|
10
|
+
* 16-byte salt per part. The parameters live in the header, so they
|
|
11
|
+
* can be raised later without breaking old parts.
|
|
12
|
+
* - Cipher: AES-256-GCM over fixed-size chunks (1 MiB by default).
|
|
13
|
+
* A single GCM stream would only authenticate at the very end, after
|
|
14
|
+
* every plaintext byte had already been handed to the tar extractor;
|
|
15
|
+
* chunking means nothing is released until its own tag has verified.
|
|
16
|
+
*
|
|
17
|
+
* Layout (all integers big-endian):
|
|
18
|
+
*
|
|
19
|
+
* header "TALONENC1" | kdf=1 (u8) | log2 N (u8) | r (u8) | p (u8)
|
|
20
|
+
* | chunk size (u32) | salt (16) | base IV (12) 45 bytes
|
|
21
|
+
* record flag (u8, 1 = final) | length (u32) | ciphertext | tag (16)
|
|
22
|
+
*
|
|
23
|
+
* Record i is sealed with nonce = base IV XOR i (in the low 64 bits) and
|
|
24
|
+
* with additional data = the whole header | i (u64) | flag. So the header
|
|
25
|
+
* is authenticated by every record, records cannot be reordered or
|
|
26
|
+
* replayed, and the final flag makes truncation at a record boundary
|
|
27
|
+
* detectable. The last record is always present (possibly empty) and
|
|
28
|
+
* nothing may follow it.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import {
|
|
32
|
+
createCipheriv,
|
|
33
|
+
createDecipheriv,
|
|
34
|
+
randomBytes,
|
|
35
|
+
scrypt,
|
|
36
|
+
} from "node:crypto";
|
|
37
|
+
import { createReadStream } from "node:fs";
|
|
38
|
+
import { open } from "node:fs/promises";
|
|
39
|
+
import { pipeline, Transform, type TransformCallback } from "node:stream";
|
|
40
|
+
import type { Readable } from "node:stream";
|
|
41
|
+
import { TalonError } from "../../errors.js";
|
|
42
|
+
|
|
43
|
+
/** File signature; the trailing 1 is the format version. */
|
|
44
|
+
export const ENCRYPTION_MAGIC = Buffer.from("TALONENC1", "ascii");
|
|
45
|
+
/** Suffix appended to the name of an encrypted part. */
|
|
46
|
+
export const ENCRYPTED_SUFFIX = ".enc";
|
|
47
|
+
|
|
48
|
+
const KDF_SCRYPT = 1;
|
|
49
|
+
const SALT_BYTES = 16;
|
|
50
|
+
const IV_BYTES = 12;
|
|
51
|
+
const TAG_BYTES = 16;
|
|
52
|
+
const KEY_BYTES = 32;
|
|
53
|
+
const RECORD_HEAD = 5;
|
|
54
|
+
const FLAG_FINAL = 1;
|
|
55
|
+
const DEFAULT_LOG2N = 17;
|
|
56
|
+
const DEFAULT_CHUNK = 1024 * 1024;
|
|
57
|
+
const MAX_CHUNK = 16 * 1024 * 1024;
|
|
58
|
+
|
|
59
|
+
/** Total header length in bytes. */
|
|
60
|
+
export const HEADER_BYTES =
|
|
61
|
+
ENCRYPTION_MAGIC.length + 4 + 4 + SALT_BYTES + IV_BYTES;
|
|
62
|
+
|
|
63
|
+
export type EncryptionHeader = {
|
|
64
|
+
log2N: number;
|
|
65
|
+
r: number;
|
|
66
|
+
p: number;
|
|
67
|
+
chunkSize: number;
|
|
68
|
+
salt: Buffer;
|
|
69
|
+
iv: Buffer;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
/** Knobs for tests; production always uses the defaults. */
|
|
73
|
+
export type EncryptOptions = { log2N?: number; chunkSize?: number };
|
|
74
|
+
|
|
75
|
+
function formatError(detail: string): TalonError {
|
|
76
|
+
return new TalonError(`Encrypted backup: ${detail}`, {
|
|
77
|
+
reason: "bad_request",
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function authError(): TalonError {
|
|
82
|
+
return formatError(
|
|
83
|
+
"authentication failed — wrong passphrase, or the file was modified",
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function encodeHeader(header: EncryptionHeader): Buffer {
|
|
88
|
+
const buf = Buffer.alloc(HEADER_BYTES);
|
|
89
|
+
let at = ENCRYPTION_MAGIC.copy(buf, 0);
|
|
90
|
+
buf[at++] = KDF_SCRYPT;
|
|
91
|
+
buf[at++] = header.log2N;
|
|
92
|
+
buf[at++] = header.r;
|
|
93
|
+
buf[at++] = header.p;
|
|
94
|
+
at = buf.writeUInt32BE(header.chunkSize, at);
|
|
95
|
+
at += header.salt.copy(buf, at);
|
|
96
|
+
header.iv.copy(buf, at);
|
|
97
|
+
return buf;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function hasEncryptionMagic(buf: Uint8Array): boolean {
|
|
101
|
+
return (
|
|
102
|
+
buf.length >= ENCRYPTION_MAGIC.length &&
|
|
103
|
+
Buffer.from(buf.subarray(0, ENCRYPTION_MAGIC.length)).equals(
|
|
104
|
+
ENCRYPTION_MAGIC,
|
|
105
|
+
)
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Parse and bound-check a header. Throws on anything unexpected. */
|
|
110
|
+
export function parseHeader(buf: Buffer): EncryptionHeader {
|
|
111
|
+
if (buf.length < HEADER_BYTES || !hasEncryptionMagic(buf)) {
|
|
112
|
+
throw formatError("not an encrypted Talon backup (bad header)");
|
|
113
|
+
}
|
|
114
|
+
let at = ENCRYPTION_MAGIC.length;
|
|
115
|
+
const kdf = buf[at++];
|
|
116
|
+
const [log2N, r, p] = [buf[at++], buf[at++], buf[at++]];
|
|
117
|
+
const chunkSize = buf.readUInt32BE(at);
|
|
118
|
+
at += 4;
|
|
119
|
+
if (kdf !== KDF_SCRYPT) throw formatError(`unknown key derivation ${kdf}`);
|
|
120
|
+
if (!scryptParamsInRange({ log2N, r, p })) {
|
|
121
|
+
throw formatError("scrypt parameters out of range");
|
|
122
|
+
}
|
|
123
|
+
if (chunkSize < 1 || chunkSize > MAX_CHUNK) {
|
|
124
|
+
throw formatError("chunk size out of range");
|
|
125
|
+
}
|
|
126
|
+
const salt = Buffer.from(buf.subarray(at, at + SALT_BYTES));
|
|
127
|
+
const iv = Buffer.from(
|
|
128
|
+
buf.subarray(at + SALT_BYTES, at + SALT_BYTES + IV_BYTES),
|
|
129
|
+
);
|
|
130
|
+
return { log2N, r, p, chunkSize, salt, iv };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** scrypt cost parameters, as recorded next to whatever they protect. */
|
|
134
|
+
export type ScryptParams = { log2N: number; r: number; p: number };
|
|
135
|
+
|
|
136
|
+
/** The scrypt parameters every new part and manifest is written with. */
|
|
137
|
+
export const DEFAULT_SCRYPT: ScryptParams = {
|
|
138
|
+
log2N: DEFAULT_LOG2N,
|
|
139
|
+
r: 8,
|
|
140
|
+
p: 1,
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
/** True when stored parameters are inside the range this build accepts. */
|
|
144
|
+
export function scryptParamsInRange({ log2N, r, p }: ScryptParams): boolean {
|
|
145
|
+
return log2N >= 10 && log2N <= 20 && r >= 1 && r <= 32 && p >= 1 && p <= 16;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** scrypt → 32-byte key, with enough maxmem for the given parameters. */
|
|
149
|
+
export function deriveScryptKey(
|
|
150
|
+
passphrase: string,
|
|
151
|
+
salt: Buffer,
|
|
152
|
+
params: ScryptParams,
|
|
153
|
+
): Promise<Buffer> {
|
|
154
|
+
const N = 2 ** params.log2N;
|
|
155
|
+
const maxmem = 2 * 128 * N * params.r * params.p + 1024 * 1024;
|
|
156
|
+
return new Promise((resolve, reject) => {
|
|
157
|
+
scrypt(
|
|
158
|
+
passphrase,
|
|
159
|
+
salt,
|
|
160
|
+
KEY_BYTES,
|
|
161
|
+
{ N, r: params.r, p: params.p, maxmem },
|
|
162
|
+
(err, key) => (err ? reject(err) : resolve(key)),
|
|
163
|
+
);
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function deriveKey(
|
|
168
|
+
passphrase: string,
|
|
169
|
+
header: EncryptionHeader,
|
|
170
|
+
): Promise<Buffer> {
|
|
171
|
+
return deriveScryptKey(passphrase, header.salt, header);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function nonceFor(iv: Buffer, counter: number): Buffer {
|
|
175
|
+
const nonce = Buffer.from(iv);
|
|
176
|
+
nonce.writeBigUInt64BE(nonce.readBigUInt64BE(4) ^ BigInt(counter), 4);
|
|
177
|
+
return nonce;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function aadFor(header: Buffer, counter: number, flag: number): Buffer {
|
|
181
|
+
const tail = Buffer.alloc(9);
|
|
182
|
+
tail.writeBigUInt64BE(BigInt(counter), 0);
|
|
183
|
+
tail[8] = flag;
|
|
184
|
+
return Buffer.concat([header, tail]);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** A FIFO of buffers that concatenates only when a caller takes bytes. */
|
|
188
|
+
class ByteQueue {
|
|
189
|
+
private parts: Buffer[] = [];
|
|
190
|
+
length = 0;
|
|
191
|
+
|
|
192
|
+
push(chunk: Buffer): void {
|
|
193
|
+
if (chunk.length === 0) return;
|
|
194
|
+
this.parts.push(chunk);
|
|
195
|
+
this.length += chunk.length;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
peek(n: number): Buffer {
|
|
199
|
+
const first = this.parts[0];
|
|
200
|
+
if (first && first.length >= n) return first.subarray(0, n);
|
|
201
|
+
return this.flatten().subarray(0, n);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
take(n: number): Buffer {
|
|
205
|
+
const all = this.flatten();
|
|
206
|
+
const rest = all.subarray(n);
|
|
207
|
+
this.parts = rest.length > 0 ? [rest] : [];
|
|
208
|
+
this.length = rest.length;
|
|
209
|
+
return all.subarray(0, n);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
private flatten(): Buffer {
|
|
213
|
+
if (this.parts.length !== 1) {
|
|
214
|
+
this.parts = [Buffer.concat(this.parts, this.length)];
|
|
215
|
+
}
|
|
216
|
+
return this.parts[0];
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Plaintext in, header + sealed records out. */
|
|
221
|
+
class EncryptStream extends Transform {
|
|
222
|
+
private readonly queue = new ByteQueue();
|
|
223
|
+
private readonly headerBytes: Buffer;
|
|
224
|
+
private counter = 0;
|
|
225
|
+
private started = false;
|
|
226
|
+
|
|
227
|
+
constructor(
|
|
228
|
+
private readonly key: Buffer,
|
|
229
|
+
private readonly header: EncryptionHeader,
|
|
230
|
+
) {
|
|
231
|
+
super();
|
|
232
|
+
this.headerBytes = encodeHeader(header);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
override _transform(
|
|
236
|
+
chunk: Buffer,
|
|
237
|
+
_encoding: BufferEncoding,
|
|
238
|
+
done: TransformCallback,
|
|
239
|
+
): void {
|
|
240
|
+
this.start();
|
|
241
|
+
this.queue.push(chunk);
|
|
242
|
+
// Strictly greater: at least one byte stays behind for the final record.
|
|
243
|
+
while (this.queue.length > this.header.chunkSize) {
|
|
244
|
+
this.seal(this.queue.take(this.header.chunkSize), 0);
|
|
245
|
+
}
|
|
246
|
+
done();
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
override _flush(done: TransformCallback): void {
|
|
250
|
+
this.start();
|
|
251
|
+
this.seal(this.queue.take(this.queue.length), FLAG_FINAL);
|
|
252
|
+
done();
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
private start(): void {
|
|
256
|
+
if (this.started) return;
|
|
257
|
+
this.started = true;
|
|
258
|
+
this.push(this.headerBytes);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
private seal(plain: Buffer, flag: number): void {
|
|
262
|
+
const cipher = createCipheriv(
|
|
263
|
+
"aes-256-gcm",
|
|
264
|
+
this.key,
|
|
265
|
+
nonceFor(this.header.iv, this.counter),
|
|
266
|
+
);
|
|
267
|
+
cipher.setAAD(aadFor(this.headerBytes, this.counter, flag));
|
|
268
|
+
const body = Buffer.concat([cipher.update(plain), cipher.final()]);
|
|
269
|
+
const head = Buffer.alloc(RECORD_HEAD);
|
|
270
|
+
head[0] = flag;
|
|
271
|
+
head.writeUInt32BE(body.length, 1);
|
|
272
|
+
this.push(Buffer.concat([head, body, cipher.getAuthTag()]));
|
|
273
|
+
this.counter += 1;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Records in (header already consumed), verified plaintext out. */
|
|
278
|
+
class DecryptStream extends Transform {
|
|
279
|
+
private readonly queue = new ByteQueue();
|
|
280
|
+
private readonly headerBytes: Buffer;
|
|
281
|
+
private counter = 0;
|
|
282
|
+
private finished = false;
|
|
283
|
+
|
|
284
|
+
constructor(
|
|
285
|
+
private readonly key: Buffer,
|
|
286
|
+
private readonly header: EncryptionHeader,
|
|
287
|
+
) {
|
|
288
|
+
super();
|
|
289
|
+
this.headerBytes = encodeHeader(header);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
override _transform(
|
|
293
|
+
chunk: Buffer,
|
|
294
|
+
_encoding: BufferEncoding,
|
|
295
|
+
done: TransformCallback,
|
|
296
|
+
): void {
|
|
297
|
+
this.queue.push(chunk);
|
|
298
|
+
try {
|
|
299
|
+
this.drain();
|
|
300
|
+
done();
|
|
301
|
+
} catch (err) {
|
|
302
|
+
done(err as Error);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
override _flush(done: TransformCallback): void {
|
|
307
|
+
if (!this.finished || this.queue.length > 0) {
|
|
308
|
+
done(formatError("truncated or has trailing bytes"));
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
done();
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
private drain(): void {
|
|
315
|
+
while (this.queue.length >= RECORD_HEAD) {
|
|
316
|
+
if (this.finished) throw formatError("data after the final record");
|
|
317
|
+
const head = this.queue.peek(RECORD_HEAD);
|
|
318
|
+
const flag = head[0];
|
|
319
|
+
const length = head.readUInt32BE(1);
|
|
320
|
+
if (flag > FLAG_FINAL || length > this.header.chunkSize) {
|
|
321
|
+
throw authError();
|
|
322
|
+
}
|
|
323
|
+
const total = RECORD_HEAD + length + TAG_BYTES;
|
|
324
|
+
if (this.queue.length < total) return;
|
|
325
|
+
const record = this.queue.take(total);
|
|
326
|
+
this.open(
|
|
327
|
+
record.subarray(RECORD_HEAD, RECORD_HEAD + length),
|
|
328
|
+
record.subarray(RECORD_HEAD + length),
|
|
329
|
+
flag,
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
private open(body: Buffer, tag: Buffer, flag: number): void {
|
|
335
|
+
const decipher = createDecipheriv(
|
|
336
|
+
"aes-256-gcm",
|
|
337
|
+
this.key,
|
|
338
|
+
nonceFor(this.header.iv, this.counter),
|
|
339
|
+
);
|
|
340
|
+
decipher.setAAD(aadFor(this.headerBytes, this.counter, flag));
|
|
341
|
+
decipher.setAuthTag(tag);
|
|
342
|
+
let plain: Buffer;
|
|
343
|
+
try {
|
|
344
|
+
plain = Buffer.concat([decipher.update(body), decipher.final()]);
|
|
345
|
+
} catch {
|
|
346
|
+
throw authError();
|
|
347
|
+
}
|
|
348
|
+
this.counter += 1;
|
|
349
|
+
if (flag === FLAG_FINAL) this.finished = true;
|
|
350
|
+
if (plain.length > 0) this.push(plain);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** An encrypting transform with a fresh salt, IV and derived key. */
|
|
355
|
+
export async function createEncryptor(
|
|
356
|
+
passphrase: string,
|
|
357
|
+
options: EncryptOptions = {},
|
|
358
|
+
): Promise<Transform> {
|
|
359
|
+
const header: EncryptionHeader = {
|
|
360
|
+
log2N: options.log2N ?? DEFAULT_LOG2N,
|
|
361
|
+
r: 8,
|
|
362
|
+
p: 1,
|
|
363
|
+
chunkSize: options.chunkSize ?? DEFAULT_CHUNK,
|
|
364
|
+
salt: randomBytes(SALT_BYTES),
|
|
365
|
+
iv: randomBytes(IV_BYTES),
|
|
366
|
+
};
|
|
367
|
+
return new EncryptStream(await deriveKey(passphrase, header), header);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/** The first bytes of a file — enough to recognise the format. */
|
|
371
|
+
async function readPrefix(path: string, bytes: number): Promise<Buffer> {
|
|
372
|
+
const handle = await open(path, "r");
|
|
373
|
+
try {
|
|
374
|
+
const buf = Buffer.alloc(bytes);
|
|
375
|
+
const { bytesRead } = await handle.read(buf, 0, bytes, 0);
|
|
376
|
+
return buf.subarray(0, bytesRead);
|
|
377
|
+
} finally {
|
|
378
|
+
await handle.close();
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** True when the file starts with the encryption signature. */
|
|
383
|
+
export async function isEncryptedFile(path: string): Promise<boolean> {
|
|
384
|
+
return hasEncryptionMagic(await readPrefix(path, ENCRYPTION_MAGIC.length));
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Stream the verified plaintext of an encrypted file. Errors (wrong
|
|
389
|
+
* passphrase, tampering, truncation) surface on the returned stream.
|
|
390
|
+
*/
|
|
391
|
+
export async function openDecrypted(
|
|
392
|
+
path: string,
|
|
393
|
+
passphrase: string,
|
|
394
|
+
): Promise<Readable> {
|
|
395
|
+
const header = parseHeader(await readPrefix(path, HEADER_BYTES));
|
|
396
|
+
const key = await deriveKey(passphrase, header);
|
|
397
|
+
return pipeline(
|
|
398
|
+
createReadStream(path, { start: HEADER_BYTES }),
|
|
399
|
+
new DecryptStream(key, header),
|
|
400
|
+
() => {
|
|
401
|
+
/* errors are delivered to whoever reads the returned stream */
|
|
402
|
+
},
|
|
403
|
+
);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** Decrypt the whole file and discard it: throws unless every record verifies. */
|
|
407
|
+
export async function verifyDecryptable(
|
|
408
|
+
path: string,
|
|
409
|
+
passphrase: string,
|
|
410
|
+
): Promise<void> {
|
|
411
|
+
const stream = await openDecrypted(path, passphrase);
|
|
412
|
+
for await (const _chunk of stream) {
|
|
413
|
+
/* authentication is the point; the bytes are not needed */
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/** Whether this passphrase opens the file's first record. */
|
|
418
|
+
export async function passphraseOpens(
|
|
419
|
+
path: string,
|
|
420
|
+
passphrase: string,
|
|
421
|
+
): Promise<boolean> {
|
|
422
|
+
try {
|
|
423
|
+
const stream = await openDecrypted(path, passphrase);
|
|
424
|
+
for await (const _chunk of stream) break;
|
|
425
|
+
return true;
|
|
426
|
+
} catch {
|
|
427
|
+
return false;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Manifest authentication — proof that a manifest was written by someone
|
|
3
|
+
* holding the backup passphrase.
|
|
4
|
+
*
|
|
5
|
+
* Every encrypted part authenticates itself, but the manifest decides
|
|
6
|
+
* which parts a restore extracts, where the `extra/<n>` trees land and
|
|
7
|
+
* which digests count as "correct". It travels with the parts to every
|
|
8
|
+
* remote target, so whoever controls the remote controls the manifest:
|
|
9
|
+
* without a MAC they could drop the state part, re-point an extra path,
|
|
10
|
+
* or swap an encrypted part for a plaintext one of their own and fix the
|
|
11
|
+
* digest to match.
|
|
12
|
+
*
|
|
13
|
+
* The MAC is HMAC-SHA256 under a scrypt-derived key (fresh salt per
|
|
14
|
+
* manifest, parameters stored beside it) over a canonical JSON encoding
|
|
15
|
+
* of every field except the ones that legitimately change after the
|
|
16
|
+
* snapshot is written — `pinned` and the per-target `remote` map — and
|
|
17
|
+
* the `auth` block itself.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
|
|
21
|
+
import type { Manifest, ManifestAuth } from "../types.js";
|
|
22
|
+
import {
|
|
23
|
+
DEFAULT_SCRYPT,
|
|
24
|
+
deriveScryptKey,
|
|
25
|
+
scryptParamsInRange,
|
|
26
|
+
type ScryptParams,
|
|
27
|
+
} from "./crypt.js";
|
|
28
|
+
|
|
29
|
+
const SALT_BYTES = 16;
|
|
30
|
+
/** Domain separation: this key signs manifests and nothing else. */
|
|
31
|
+
const CONTEXT = "talon-backup-manifest-v1\0";
|
|
32
|
+
|
|
33
|
+
/** JSON with object keys sorted at every depth and undefined dropped. */
|
|
34
|
+
export function canonicalJson(value: unknown): string {
|
|
35
|
+
if (Array.isArray(value)) {
|
|
36
|
+
return `[${value.map((item) => canonicalJson(item ?? null)).join(",")}]`;
|
|
37
|
+
}
|
|
38
|
+
if (value !== null && typeof value === "object") {
|
|
39
|
+
const entries = Object.entries(value as Record<string, unknown>)
|
|
40
|
+
.filter(([, v]) => v !== undefined)
|
|
41
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
42
|
+
.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`);
|
|
43
|
+
return `{${entries.join(",")}}`;
|
|
44
|
+
}
|
|
45
|
+
return JSON.stringify(value);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The bytes the MAC covers: everything that must not change after writing. */
|
|
49
|
+
function signedBytes(manifest: Manifest): Buffer {
|
|
50
|
+
const {
|
|
51
|
+
remote: _remote,
|
|
52
|
+
pinned: _pinned,
|
|
53
|
+
auth: _auth,
|
|
54
|
+
...covered
|
|
55
|
+
} = manifest;
|
|
56
|
+
return Buffer.from(CONTEXT + canonicalJson(covered), "utf8");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function mac(key: Buffer, manifest: Manifest): Buffer {
|
|
60
|
+
return createHmac("sha256", key).update(signedBytes(manifest)).digest();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** A fresh `auth` block for this manifest under this passphrase. */
|
|
64
|
+
export async function signManifest(
|
|
65
|
+
manifest: Manifest,
|
|
66
|
+
passphrase: string,
|
|
67
|
+
params: ScryptParams = DEFAULT_SCRYPT,
|
|
68
|
+
): Promise<ManifestAuth> {
|
|
69
|
+
const salt = randomBytes(SALT_BYTES);
|
|
70
|
+
const key = await deriveScryptKey(passphrase, salt, params);
|
|
71
|
+
return {
|
|
72
|
+
v: 1,
|
|
73
|
+
alg: "hmac-sha256",
|
|
74
|
+
kdf: "scrypt",
|
|
75
|
+
...params,
|
|
76
|
+
salt: salt.toString("base64"),
|
|
77
|
+
mac: mac(key, manifest).toString("base64"),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* True only when the manifest carries a well-formed `auth` block whose MAC
|
|
83
|
+
* verifies under this passphrase. Any malformed field is a failure, never
|
|
84
|
+
* an exception — the caller decides what an unauthenticated manifest means.
|
|
85
|
+
*/
|
|
86
|
+
export async function verifyManifest(
|
|
87
|
+
manifest: Manifest,
|
|
88
|
+
passphrase: string,
|
|
89
|
+
): Promise<boolean> {
|
|
90
|
+
const auth = manifest.auth;
|
|
91
|
+
if (!auth || auth.v !== 1 || auth.alg !== "hmac-sha256") return false;
|
|
92
|
+
if (auth.kdf !== "scrypt" || !scryptParamsInRange(auth)) return false;
|
|
93
|
+
const salt = Buffer.from(String(auth.salt), "base64");
|
|
94
|
+
const expected = Buffer.from(String(auth.mac), "base64");
|
|
95
|
+
if (salt.length !== SALT_BYTES || expected.length !== 32) return false;
|
|
96
|
+
const key = await deriveScryptKey(passphrase, salt, auth);
|
|
97
|
+
return timingSafeEqual(mac(key, manifest), expected);
|
|
98
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the backup passphrase comes from — and the rule that plaintext
|
|
3
|
+
* never leaves the machine.
|
|
4
|
+
*
|
|
5
|
+
* Two sources, first match wins:
|
|
6
|
+
*
|
|
7
|
+
* 1. `TALON_BACKUP_PASSPHRASE` in the environment.
|
|
8
|
+
* 2. `backup.encryption.passphraseFile` — a file holding the passphrase
|
|
9
|
+
* (one line; surrounding whitespace ignored). `talon backup keygen`
|
|
10
|
+
* writes one with mode 600.
|
|
11
|
+
*
|
|
12
|
+
* With neither, snapshots stay plaintext and local-only: `upload.ts`
|
|
13
|
+
* refuses to hand an unencrypted part to any remote target. When
|
|
14
|
+
* `backup.encryption` is present but yields no passphrase, a snapshot
|
|
15
|
+
* fails instead of silently falling back to plaintext.
|
|
16
|
+
*
|
|
17
|
+
* The passphrase file lives outside every snapshot root on purpose — a
|
|
18
|
+
* key stored inside the backup it unlocks is no key at all. Keep a copy
|
|
19
|
+
* somewhere off this machine (a password manager): without it, encrypted
|
|
20
|
+
* snapshots cannot be restored.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { randomBytes } from "node:crypto";
|
|
24
|
+
import { readFile, stat, writeFile, mkdir } from "node:fs/promises";
|
|
25
|
+
import { dirname } from "node:path";
|
|
26
|
+
import { logWarn } from "../../util/log.js";
|
|
27
|
+
import { TalonError } from "../errors.js";
|
|
28
|
+
import { expandUserPath } from "./plan.js";
|
|
29
|
+
import type { BackupSettings } from "./types.js";
|
|
30
|
+
|
|
31
|
+
export const PASSPHRASE_ENV = "TALON_BACKUP_PASSPHRASE";
|
|
32
|
+
/** Short enough to be a typo or an empty file, not a passphrase. */
|
|
33
|
+
const MIN_PASSPHRASE_LENGTH = 12;
|
|
34
|
+
|
|
35
|
+
function passphraseError(message: string): TalonError {
|
|
36
|
+
return new TalonError(message, { reason: "bad_request" });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function checked(passphrase: string, source: string): string {
|
|
40
|
+
if (passphrase.length < MIN_PASSPHRASE_LENGTH) {
|
|
41
|
+
throw passphraseError(
|
|
42
|
+
`Backup passphrase from ${source} is shorter than ${MIN_PASSPHRASE_LENGTH} characters`,
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
return passphrase;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function readPassphraseFile(raw: string): Promise<string> {
|
|
49
|
+
const path = expandUserPath(raw);
|
|
50
|
+
let text: string;
|
|
51
|
+
try {
|
|
52
|
+
text = await readFile(path, "utf8");
|
|
53
|
+
} catch (err) {
|
|
54
|
+
throw passphraseError(
|
|
55
|
+
`Cannot read backup.encryption.passphraseFile ${path}: ${(err as NodeJS.ErrnoException).code ?? String(err)}`,
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
const mode = (await stat(path)).mode;
|
|
59
|
+
if (process.platform !== "win32" && (mode & 0o077) !== 0) {
|
|
60
|
+
logWarn("backup", `${path} is readable by other users — chmod 600 it`);
|
|
61
|
+
}
|
|
62
|
+
return checked(text.trim(), path);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Absolute path of the configured passphrase file, or null. The snapshot
|
|
67
|
+
* builder uses it to keep the key out of every part.
|
|
68
|
+
*/
|
|
69
|
+
export function passphraseFilePath(
|
|
70
|
+
settings: Pick<BackupSettings, "encryption">,
|
|
71
|
+
): string | null {
|
|
72
|
+
const file = settings.encryption?.passphraseFile;
|
|
73
|
+
return file ? expandUserPath(file) : null;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The passphrase this deployment encrypts with, or null when encryption
|
|
78
|
+
* is not configured. Throws when it is configured but unusable.
|
|
79
|
+
*/
|
|
80
|
+
export async function resolvePassphrase(
|
|
81
|
+
settings: Pick<BackupSettings, "encryption">,
|
|
82
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
83
|
+
): Promise<string | null> {
|
|
84
|
+
const fromEnv = env[PASSPHRASE_ENV]?.trim();
|
|
85
|
+
if (fromEnv) return checked(fromEnv, PASSPHRASE_ENV);
|
|
86
|
+
const file = settings.encryption?.passphraseFile;
|
|
87
|
+
if (file) return readPassphraseFile(file);
|
|
88
|
+
if (settings.encryption) {
|
|
89
|
+
throw passphraseError(
|
|
90
|
+
`backup.encryption is set but no passphrase was found: set backup.encryption.passphraseFile or ${PASSPHRASE_ENV}`,
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Like `resolvePassphrase`, for callers that cannot go on without one. */
|
|
97
|
+
export async function requirePassphrase(
|
|
98
|
+
settings: Pick<BackupSettings, "encryption">,
|
|
99
|
+
what: string,
|
|
100
|
+
): Promise<string> {
|
|
101
|
+
const passphrase = await resolvePassphrase(settings);
|
|
102
|
+
if (passphrase) return passphrase;
|
|
103
|
+
throw passphraseError(
|
|
104
|
+
`${what} is encrypted — set backup.encryption.passphraseFile or ${PASSPHRASE_ENV} to decrypt it`,
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Write a new random passphrase (256 bits, base64url) to `rawPath` with
|
|
110
|
+
* mode 600. Refuses to overwrite: replacing a key orphans every snapshot
|
|
111
|
+
* encrypted with it. Returns the absolute path; never the passphrase.
|
|
112
|
+
*/
|
|
113
|
+
export async function generatePassphraseFile(rawPath: string): Promise<string> {
|
|
114
|
+
const path = expandUserPath(rawPath);
|
|
115
|
+
await mkdir(dirname(path), { recursive: true });
|
|
116
|
+
try {
|
|
117
|
+
await writeFile(path, `${randomBytes(32).toString("base64url")}\n`, {
|
|
118
|
+
mode: 0o600,
|
|
119
|
+
flag: "wx",
|
|
120
|
+
});
|
|
121
|
+
} catch (err) {
|
|
122
|
+
if ((err as NodeJS.ErrnoException).code === "EEXIST") {
|
|
123
|
+
throw passphraseError(
|
|
124
|
+
`${path} already exists — refusing to overwrite a backup key`,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
throw err;
|
|
128
|
+
}
|
|
129
|
+
return path;
|
|
130
|
+
}
|