talon-agent 5.26.1 → 5.26.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/package.json +1 -1
- package/src/app.ts +42 -1
- package/src/bootstrap.ts +171 -57
- package/src/cli/commands/backup.ts +4 -1
- package/src/cli/commands/history.ts +152 -0
- package/src/cli/index.ts +6 -0
- package/src/core/backup/boot/version-checkpoint.ts +199 -0
- package/src/core/backup/index.ts +3 -0
- package/src/core/backup/passphrase.ts +55 -2
- package/src/core/backup/restore.ts +32 -4
- package/src/core/backup/scheduler.ts +125 -13
- package/src/core/backup/status.ts +26 -3
- package/src/core/update/self-update.ts +144 -47
- package/src/frontend/discord/admin.ts +2 -2
- package/src/frontend/discord/callbacks/components/backend-select.ts +4 -4
- package/src/frontend/discord/commands/admin.ts +22 -3
- package/src/frontend/discord/commands/definitions.ts +7 -0
- package/src/frontend/native/chats/chat-lifecycle.ts +5 -2
- package/src/frontend/native/chats/chats.ts +14 -2
- package/src/frontend/native/chats/empty-chat-sweep.ts +4 -1
- package/src/frontend/native/chats/reset.ts +39 -17
- package/src/frontend/native/commands/session.ts +2 -2
- package/src/frontend/native/surface/models.ts +3 -3
- package/src/frontend/native/turn/turn-meta.ts +0 -6
- package/src/frontend/presentation/model-commands.ts +8 -11
- package/src/frontend/presentation/session-status.ts +12 -13
- package/src/frontend/presentation/text-commands.ts +6 -9
- package/src/frontend/telegram/admin/sessions.ts +2 -1
- package/src/frontend/telegram/callbacks/model/backend.ts +5 -6
- package/src/frontend/telegram/commands/admin.ts +21 -3
- package/src/frontend/whatsapp/commands.ts +5 -8
- package/src/storage/backup/index.ts +1 -1
- package/src/storage/chat-settings.ts +13 -0
- package/src/storage/db.ts +5 -0
- package/src/storage/history.ts +167 -17
- package/src/storage/media-index.ts +4 -3
- package/src/storage/repositories/history-repo.ts +75 -7
- package/src/storage/sessions.ts +60 -1
- package/src/storage/sql/history.sql +45 -4
- package/src/storage/sql/schema.sql +13 -0
- package/src/storage/sql/statements.generated.ts +49 -5
- package/src/storage/turn-meta.ts +1 -1
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The boot-time upgrade checkpoint: a pinned snapshot taken the first time
|
|
3
|
+
* a new version boots, before anything in that version touches the data.
|
|
4
|
+
*
|
|
5
|
+
* `/update` on a git checkout takes its own pre-update checkpoint, but
|
|
6
|
+
* that is the only install shape that does. A Docker or TrueNAS update is
|
|
7
|
+
* a new image started against the old volume; npm and binary installs are
|
|
8
|
+
* the same story. The first code of the new version to run is this boot,
|
|
9
|
+
* so this boot is where the safety net has to be.
|
|
10
|
+
*
|
|
11
|
+
* The last version that booted is kept in a small marker file beside the
|
|
12
|
+
* Talon home's config (not in the database: the check runs before the
|
|
13
|
+
* database is opened, and a restore that rolls the database back should
|
|
14
|
+
* not also roll back the fact that a newer version ran). The database is
|
|
15
|
+
* captured through a read-only handle, so the checkpoint holds it exactly
|
|
16
|
+
* as the previous version left it — before this boot's schema setup.
|
|
17
|
+
*
|
|
18
|
+
* Never throws and never blocks boot: a daemon that refuses to start is
|
|
19
|
+
* worse than one that starts carefully. A failure is logged loudly,
|
|
20
|
+
* raised as a critical operator alert, and reported to the caller so the
|
|
21
|
+
* boot can skip its destructive steps. The marker is only advanced on
|
|
22
|
+
* success, so the next boot tries again.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { existsSync } from "node:fs";
|
|
26
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
27
|
+
import { dirname, join } from "node:path";
|
|
28
|
+
import { dirs } from "../../../util/paths.js";
|
|
29
|
+
import { log, logError } from "../../../util/log.js";
|
|
30
|
+
import {
|
|
31
|
+
databasePath,
|
|
32
|
+
snapshotDatabase,
|
|
33
|
+
snapshotSqliteFile,
|
|
34
|
+
} from "../../../storage/backup/index.js";
|
|
35
|
+
import { raiseAlert } from "../../frontend-runtime/alerts.js";
|
|
36
|
+
import { buildSnapshot } from "../snapshot.js";
|
|
37
|
+
import type { BackupSettings } from "../types.js";
|
|
38
|
+
|
|
39
|
+
/** File (under the Talon home) recording the last version that booted. */
|
|
40
|
+
const BOOT_VERSION_MARKER = "last-boot-version.json";
|
|
41
|
+
|
|
42
|
+
/** Alert key for a failed upgrade checkpoint. */
|
|
43
|
+
export const UPGRADE_CHECKPOINT_ALERT = "backup.upgrade-checkpoint";
|
|
44
|
+
|
|
45
|
+
export type VersionCheckpointResult =
|
|
46
|
+
/** Same version as last boot — nothing to do. */
|
|
47
|
+
| { status: "unchanged"; version: string }
|
|
48
|
+
/** No marker and no database: a brand-new install, nothing to protect. */
|
|
49
|
+
| { status: "fresh-install"; version: string }
|
|
50
|
+
/** Version changed but `backup.checkpointBeforeUpdate` is off. */
|
|
51
|
+
| { status: "disabled"; from: string; to: string }
|
|
52
|
+
| { status: "taken"; id: string; from: string; to: string }
|
|
53
|
+
| { status: "failed"; from: string; to: string; error: string };
|
|
54
|
+
|
|
55
|
+
type MarkerFile = { version: string; bootedAt: string };
|
|
56
|
+
|
|
57
|
+
export type VersionCheckpointOptions = {
|
|
58
|
+
settings: BackupSettings;
|
|
59
|
+
/** The version now booting. */
|
|
60
|
+
version: string;
|
|
61
|
+
/** Talon home; tests point this at a scratch directory. */
|
|
62
|
+
home?: string;
|
|
63
|
+
/** The database file to capture; defaults to the daemon's. */
|
|
64
|
+
databaseFile?: string;
|
|
65
|
+
/** Snapshot builder — a test seam. */
|
|
66
|
+
build?: typeof buildSnapshot;
|
|
67
|
+
/** Operator alert — a test seam. */
|
|
68
|
+
alert?: typeof raiseAlert;
|
|
69
|
+
/** Clock, for tests. */
|
|
70
|
+
now?: Date;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
export function bootVersionMarkerPath(home: string = dirs.root): string {
|
|
74
|
+
return join(home, BOOT_VERSION_MARKER);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** The last version that booted, or null when none is recorded (or readable). */
|
|
78
|
+
export async function readLastBootVersion(
|
|
79
|
+
home: string = dirs.root,
|
|
80
|
+
): Promise<string | null> {
|
|
81
|
+
try {
|
|
82
|
+
const raw = await readFile(bootVersionMarkerPath(home), "utf8");
|
|
83
|
+
const parsed = JSON.parse(raw) as Partial<MarkerFile>;
|
|
84
|
+
return typeof parsed.version === "string" && parsed.version
|
|
85
|
+
? parsed.version
|
|
86
|
+
: null;
|
|
87
|
+
} catch {
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function writeLastBootVersion(
|
|
93
|
+
home: string,
|
|
94
|
+
version: string,
|
|
95
|
+
now: Date,
|
|
96
|
+
): Promise<void> {
|
|
97
|
+
const path = bootVersionMarkerPath(home);
|
|
98
|
+
const marker: MarkerFile = { version, bootedAt: now.toISOString() };
|
|
99
|
+
try {
|
|
100
|
+
await mkdir(dirname(path), { recursive: true });
|
|
101
|
+
const temp = `${path}.tmp`;
|
|
102
|
+
await writeFile(temp, `${JSON.stringify(marker, null, 2)}\n`, {
|
|
103
|
+
mode: 0o600,
|
|
104
|
+
});
|
|
105
|
+
await rename(temp, path);
|
|
106
|
+
} catch (err) {
|
|
107
|
+
// Worst case the next boot takes one more checkpoint than needed.
|
|
108
|
+
logError("backup", `Could not record boot version ${version}`, err);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Copy the database through a read-only handle; fall back to the regular
|
|
114
|
+
* (opening) copy when SQLite refuses a read-only open, e.g. a WAL file
|
|
115
|
+
* whose shared-memory index can't be created on a read-only handle.
|
|
116
|
+
*/
|
|
117
|
+
function copyDatabaseReadOnly(dbFile: string): (dest: string) => void {
|
|
118
|
+
return (dest) => {
|
|
119
|
+
try {
|
|
120
|
+
snapshotSqliteFile(dbFile, dest);
|
|
121
|
+
} catch (err) {
|
|
122
|
+
logError(
|
|
123
|
+
"backup",
|
|
124
|
+
"Read-only database copy failed; copying through the daemon handle",
|
|
125
|
+
err,
|
|
126
|
+
);
|
|
127
|
+
snapshotDatabase(dest);
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Take a pinned `pre-upgrade <old>→<new>` checkpoint when the version
|
|
134
|
+
* booting differs from the last one that did. See the module comment.
|
|
135
|
+
*/
|
|
136
|
+
export async function checkpointOnVersionChange(
|
|
137
|
+
options: VersionCheckpointOptions,
|
|
138
|
+
): Promise<VersionCheckpointResult> {
|
|
139
|
+
const home = options.home ?? dirs.root;
|
|
140
|
+
const now = options.now ?? new Date();
|
|
141
|
+
const to = options.version;
|
|
142
|
+
const previous = await readLastBootVersion(home);
|
|
143
|
+
if (previous === to) return { status: "unchanged", version: to };
|
|
144
|
+
|
|
145
|
+
const dbFile = options.databaseFile ?? databasePath();
|
|
146
|
+
const hasDatabase = existsSync(dbFile);
|
|
147
|
+
if (previous === null && !hasDatabase) {
|
|
148
|
+
await writeLastBootVersion(home, to, now);
|
|
149
|
+
return { status: "fresh-install", version: to };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// No marker but a database: an install from before the marker existed.
|
|
153
|
+
const from = previous ?? "unknown";
|
|
154
|
+
if (!options.settings.checkpointBeforeUpdate) {
|
|
155
|
+
log(
|
|
156
|
+
"backup",
|
|
157
|
+
`Version changed ${from}→${to}; pre-upgrade checkpoint disabled (backup.checkpointBeforeUpdate=false)`,
|
|
158
|
+
);
|
|
159
|
+
await writeLastBootVersion(home, to, now);
|
|
160
|
+
return { status: "disabled", from, to };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const build = options.build ?? buildSnapshot;
|
|
164
|
+
try {
|
|
165
|
+
const manifest = await build({
|
|
166
|
+
kind: "checkpoint",
|
|
167
|
+
label: `pre-upgrade ${from}→${to}`,
|
|
168
|
+
pinned: true,
|
|
169
|
+
settings: options.settings,
|
|
170
|
+
...(options.home === undefined ? {} : { home, userHome: null }),
|
|
171
|
+
// Read-only: the copy is the database exactly as the previous
|
|
172
|
+
// version left it, before this boot opens it and sets up its schema.
|
|
173
|
+
...(hasDatabase ? { copyDatabase: copyDatabaseReadOnly(dbFile) } : {}),
|
|
174
|
+
now,
|
|
175
|
+
});
|
|
176
|
+
await writeLastBootVersion(home, to, now);
|
|
177
|
+
log(
|
|
178
|
+
"backup",
|
|
179
|
+
`Pre-upgrade checkpoint ${manifest.id} taken (${from}→${to}, pinned)`,
|
|
180
|
+
);
|
|
181
|
+
return { status: "taken", id: manifest.id, from, to };
|
|
182
|
+
} catch (err) {
|
|
183
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
184
|
+
logError(
|
|
185
|
+
"backup",
|
|
186
|
+
`PRE-UPGRADE CHECKPOINT FAILED (${from}→${to}) — booting without a safety snapshot; destructive boot steps are skipped`,
|
|
187
|
+
err,
|
|
188
|
+
);
|
|
189
|
+
(options.alert ?? raiseAlert)(
|
|
190
|
+
UPGRADE_CHECKPOINT_ALERT,
|
|
191
|
+
`Talon upgraded ${from}→${to} but the pre-upgrade checkpoint failed: ${error}\n` +
|
|
192
|
+
"The daemon booted anyway and skipped its boot-time cleanup. " +
|
|
193
|
+
"Fix the backup setup (e.g. a missing backup key) and restart; " +
|
|
194
|
+
"the checkpoint is retried on every boot until it succeeds.",
|
|
195
|
+
{ severity: "critical" },
|
|
196
|
+
);
|
|
197
|
+
return { status: "failed", from, to, error };
|
|
198
|
+
}
|
|
199
|
+
}
|
package/src/core/backup/index.ts
CHANGED
|
@@ -25,6 +25,7 @@ export {
|
|
|
25
25
|
runBackup,
|
|
26
26
|
stopBackupScheduler,
|
|
27
27
|
checkpointBeforeUpdate,
|
|
28
|
+
type UpdateCheckpoint,
|
|
28
29
|
} from "./scheduler.js";
|
|
29
30
|
|
|
30
31
|
export {
|
|
@@ -41,6 +42,8 @@ export {
|
|
|
41
42
|
writeRestorePending,
|
|
42
43
|
} from "./restore.js";
|
|
43
44
|
|
|
45
|
+
export { checkpointOnVersionChange } from "./boot/version-checkpoint.js";
|
|
46
|
+
|
|
44
47
|
export { discoverTargets, type BackupTarget } from "./targets.js";
|
|
45
48
|
|
|
46
49
|
export {
|
|
@@ -45,7 +45,10 @@ function checked(passphrase: string, source: string): string {
|
|
|
45
45
|
return passphrase;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
async function readPassphraseFile(
|
|
48
|
+
async function readPassphraseFile(
|
|
49
|
+
raw: string,
|
|
50
|
+
warnOnMode = true,
|
|
51
|
+
): Promise<string> {
|
|
49
52
|
const path = expandUserPath(raw);
|
|
50
53
|
let text: string;
|
|
51
54
|
try {
|
|
@@ -56,7 +59,7 @@ async function readPassphraseFile(raw: string): Promise<string> {
|
|
|
56
59
|
);
|
|
57
60
|
}
|
|
58
61
|
const mode = (await stat(path)).mode;
|
|
59
|
-
if (process.platform !== "win32" && (mode & 0o077) !== 0) {
|
|
62
|
+
if (warnOnMode && process.platform !== "win32" && (mode & 0o077) !== 0) {
|
|
60
63
|
logWarn("backup", `${path} is readable by other users — chmod 600 it`);
|
|
61
64
|
}
|
|
62
65
|
return checked(text.trim(), path);
|
|
@@ -93,6 +96,56 @@ export async function resolvePassphrase(
|
|
|
93
96
|
return null;
|
|
94
97
|
}
|
|
95
98
|
|
|
99
|
+
/**
|
|
100
|
+
* What is wrong with the configured key, or null when nothing is. Unlike
|
|
101
|
+
* `resolvePassphrase` this looks at every configured source: a
|
|
102
|
+
* `passphraseFile` that has gone missing is reported even while
|
|
103
|
+
* `TALON_BACKUP_PASSPHRASE` keeps snapshots running, because restoring
|
|
104
|
+
* anywhere without that variable needs the file. `blocking` says whether
|
|
105
|
+
* snapshots fail because of it. Never throws.
|
|
106
|
+
*/
|
|
107
|
+
export async function passphraseProblem(
|
|
108
|
+
settings: Pick<BackupSettings, "encryption">,
|
|
109
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
110
|
+
): Promise<{ message: string; blocking: boolean } | null> {
|
|
111
|
+
const fromEnv = env[PASSPHRASE_ENV]?.trim();
|
|
112
|
+
let envProblem: string | null = null;
|
|
113
|
+
if (fromEnv) {
|
|
114
|
+
try {
|
|
115
|
+
checked(fromEnv, PASSPHRASE_ENV);
|
|
116
|
+
} catch (err) {
|
|
117
|
+
envProblem = (err as Error).message;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const file = settings.encryption?.passphraseFile;
|
|
121
|
+
if (file) {
|
|
122
|
+
try {
|
|
123
|
+
await readPassphraseFile(file, false);
|
|
124
|
+
} catch (err) {
|
|
125
|
+
const message = (err as Error).message;
|
|
126
|
+
// A usable environment passphrase wins, so snapshots still run.
|
|
127
|
+
if (fromEnv && !envProblem) {
|
|
128
|
+
return {
|
|
129
|
+
message: `${message} (snapshots still run on ${PASSPHRASE_ENV}, but a restore without it needs this file)`,
|
|
130
|
+
blocking: false,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
message: envProblem ? `${envProblem}; ${message}` : message,
|
|
135
|
+
blocking: true,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (envProblem) return { message: envProblem, blocking: true };
|
|
140
|
+
if (settings.encryption && !file && !fromEnv) {
|
|
141
|
+
return {
|
|
142
|
+
message: `backup.encryption is set but no passphrase was found: set backup.encryption.passphraseFile or ${PASSPHRASE_ENV}`,
|
|
143
|
+
blocking: true,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
|
|
96
149
|
/** Like `resolvePassphrase`, for callers that cannot go on without one. */
|
|
97
150
|
export async function requirePassphrase(
|
|
98
151
|
settings: Pick<BackupSettings, "encryption">,
|
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
copyFile,
|
|
38
38
|
} from "node:fs/promises";
|
|
39
39
|
import { homedir } from "node:os";
|
|
40
|
-
import { dirname, join } from "node:path";
|
|
40
|
+
import { dirname, join, resolve } from "node:path";
|
|
41
41
|
import writeFileAtomic from "write-file-atomic";
|
|
42
42
|
import { dirs } from "../../util/paths.js";
|
|
43
43
|
import { log, logWarn } from "../../util/log.js";
|
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
import { sha256File } from "./archive/digest.js";
|
|
51
51
|
import { extractTar } from "./archive/tar.js";
|
|
52
52
|
import { createDecompressor } from "./archive/zstd.js";
|
|
53
|
-
import { requirePassphrase } from "./passphrase.js";
|
|
53
|
+
import { passphraseFilePath, requirePassphrase } from "./passphrase.js";
|
|
54
54
|
import { collectTree, excludeForRoot } from "./plan.js";
|
|
55
55
|
import {
|
|
56
56
|
authenticateManifest,
|
|
@@ -348,20 +348,32 @@ export function destinationFor(
|
|
|
348
348
|
return join(home, ...segments);
|
|
349
349
|
}
|
|
350
350
|
|
|
351
|
+
/** True when `path` is the active backup passphrase file. */
|
|
352
|
+
function isKeyFile(path: string, keyFile: string | null): boolean {
|
|
353
|
+
return keyFile !== null && resolve(path) === keyFile;
|
|
354
|
+
}
|
|
355
|
+
|
|
351
356
|
/**
|
|
352
357
|
* Bring one include root to exactly the snapshot's state: remove what the
|
|
353
358
|
* snapshot rules would have captured, keep what they deliberately skip.
|
|
354
359
|
* Returns how many live files were removed.
|
|
360
|
+
*
|
|
361
|
+
* The active passphrase file is never removed. The snapshot builder leaves
|
|
362
|
+
* it out of every part (a key inside the backup it unlocks is no key), so
|
|
363
|
+
* when it sits inside an include root — `workspace/secrets/`, `keys/`, an
|
|
364
|
+
* extra path — nothing in the snapshot would ever put it back.
|
|
355
365
|
*/
|
|
356
366
|
async function clearCovered(
|
|
357
367
|
destRoot: string,
|
|
358
368
|
archiveRoot: string,
|
|
369
|
+
keyFile: string | null,
|
|
359
370
|
): Promise<number> {
|
|
360
371
|
const existing = await collectTree(destRoot, archiveRoot, {
|
|
361
372
|
exclude: excludeForRoot(archiveRoot),
|
|
362
373
|
});
|
|
363
374
|
let removed = 0;
|
|
364
375
|
for (const entry of [...existing].reverse()) {
|
|
376
|
+
if (isKeyFile(entry.source, keyFile)) continue;
|
|
365
377
|
try {
|
|
366
378
|
if (entry.type === "dir") await rmdir(entry.source).catch(() => {});
|
|
367
379
|
else {
|
|
@@ -410,6 +422,7 @@ async function applyStaged(
|
|
|
410
422
|
staging: string,
|
|
411
423
|
home: string,
|
|
412
424
|
external: readonly ExternalDestination[],
|
|
425
|
+
keyFile: string | null,
|
|
413
426
|
): Promise<RestoreReport> {
|
|
414
427
|
const extras = manifest.extras ?? [];
|
|
415
428
|
const report: RestoreReport = {
|
|
@@ -429,7 +442,7 @@ async function applyStaged(
|
|
|
429
442
|
logWarn("backup", `No destination for ${root} on this machine — skipped`);
|
|
430
443
|
continue;
|
|
431
444
|
}
|
|
432
|
-
report.removed += await clearCovered(destRoot, root);
|
|
445
|
+
report.removed += await clearCovered(destRoot, root, keyFile);
|
|
433
446
|
// A session database's sidecars describe the file being replaced.
|
|
434
447
|
if (external.some((entry) => entry.root === root && entry.sqlite)) {
|
|
435
448
|
await rm(`${destRoot}-wal`, { force: true });
|
|
@@ -439,6 +452,15 @@ async function applyStaged(
|
|
|
439
452
|
for (const entry of staged) {
|
|
440
453
|
const dest = destinationFor(entry.archivePath, home, extras, external);
|
|
441
454
|
if (!dest) continue;
|
|
455
|
+
// An older snapshot (or one from another machine) may carry a file
|
|
456
|
+
// where this machine keeps its key. The key path is never written.
|
|
457
|
+
if (entry.type !== "dir" && isKeyFile(dest, keyFile)) {
|
|
458
|
+
logWarn(
|
|
459
|
+
"backup",
|
|
460
|
+
`Left the backup passphrase file ${dest} untouched — the snapshot's copy was not applied`,
|
|
461
|
+
);
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
442
464
|
if (entry.type === "dir") await mkdir(dest, { recursive: true });
|
|
443
465
|
else {
|
|
444
466
|
await placeFile(entry.source, dest);
|
|
@@ -585,7 +607,13 @@ export async function restoreSnapshot(
|
|
|
585
607
|
const staging = join(snapshotDir(manifest.id, home), "restore-staging");
|
|
586
608
|
await extractParts(manifest, home, staging, options.settings, parts);
|
|
587
609
|
await options.beforeApply?.();
|
|
588
|
-
const report = await applyStaged(
|
|
610
|
+
const report = await applyStaged(
|
|
611
|
+
manifest,
|
|
612
|
+
staging,
|
|
613
|
+
home,
|
|
614
|
+
external,
|
|
615
|
+
passphraseFilePath(options.settings),
|
|
616
|
+
);
|
|
589
617
|
report.checkpointId = checkpointId;
|
|
590
618
|
if (options.clone && manifest.origin) {
|
|
591
619
|
report.configRewritten = await rewriteConfigForClone(
|
|
@@ -31,6 +31,7 @@ import { faultText } from "../engine/fault-text.js";
|
|
|
31
31
|
import { bus } from "../bus/index.js";
|
|
32
32
|
import { log, logError } from "../../util/log.js";
|
|
33
33
|
import { dirs } from "../../util/paths.js";
|
|
34
|
+
import { passphraseProblem } from "./passphrase.js";
|
|
34
35
|
import { buildSnapshot } from "./snapshot.js";
|
|
35
36
|
import { listLocalManifests, pruneLocal, reconcileIndex } from "./store.js";
|
|
36
37
|
import { discoverTargets, selectTargets } from "./targets.js";
|
|
@@ -54,6 +55,7 @@ export type RunRequest = {
|
|
|
54
55
|
/** Test seam: the expensive collaborators, swappable in unit tests. */
|
|
55
56
|
export const _backupDeps = {
|
|
56
57
|
build: buildSnapshot,
|
|
58
|
+
keyProblem: (settings: BackupSettings) => passphraseProblem(settings),
|
|
57
59
|
discover: discoverTargets,
|
|
58
60
|
upload: uploadSnapshot,
|
|
59
61
|
pruneLocal,
|
|
@@ -85,6 +87,10 @@ type SchedulerState = {
|
|
|
85
87
|
lastSnapshotId: string | undefined;
|
|
86
88
|
lastError: string | undefined;
|
|
87
89
|
nextRunAt: number | undefined;
|
|
90
|
+
/** The hourly passphrase check (see checkBackupKey). */
|
|
91
|
+
keyTimer: ReturnType<typeof setInterval> | null;
|
|
92
|
+
/** What was wrong with the key at the last check, if anything. */
|
|
93
|
+
keyProblem: string | undefined;
|
|
88
94
|
};
|
|
89
95
|
|
|
90
96
|
const state: SchedulerState = {
|
|
@@ -100,6 +106,8 @@ const state: SchedulerState = {
|
|
|
100
106
|
lastSnapshotId: undefined,
|
|
101
107
|
lastError: undefined,
|
|
102
108
|
nextRunAt: undefined,
|
|
109
|
+
keyTimer: null,
|
|
110
|
+
keyProblem: undefined,
|
|
103
111
|
};
|
|
104
112
|
|
|
105
113
|
/**
|
|
@@ -242,6 +250,7 @@ async function tick(generation: number): Promise<void> {
|
|
|
242
250
|
const intervalMs = settings.intervalHours * HOUR_MS;
|
|
243
251
|
// Recomputed every tick: a suspended machine or a stepped clock lands
|
|
244
252
|
// here late, and the answer is "run now", not "run N missed times".
|
|
253
|
+
await checkBackupKey();
|
|
245
254
|
if (backoff.active()) {
|
|
246
255
|
schedule(Math.min(intervalMs, BOOT_DELAY_MS), generation);
|
|
247
256
|
return;
|
|
@@ -254,6 +263,82 @@ async function tick(generation: number): Promise<void> {
|
|
|
254
263
|
schedule(intervalMs, generation);
|
|
255
264
|
}
|
|
256
265
|
|
|
266
|
+
// ── The key check ───────────────────────────────────────────────────────────
|
|
267
|
+
|
|
268
|
+
const KEY_ALERT = "backup.key";
|
|
269
|
+
const KEY_CHECK_MS = HOUR_MS;
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Is the configured passphrase still there? A key that disappears breaks
|
|
273
|
+
* every snapshot and the pre-update checkpoint, and a run only notices at
|
|
274
|
+
* its next window, hours later. So this runs at boot, every hour, before
|
|
275
|
+
* each scheduled run and on every status request.
|
|
276
|
+
*
|
|
277
|
+
* It tells the admin once when the key goes bad and once when it is back,
|
|
278
|
+
* never on every check. Returns the current problem, or null. Never throws.
|
|
279
|
+
*/
|
|
280
|
+
export async function checkBackupKey(): Promise<string | null> {
|
|
281
|
+
const settings = state.settings;
|
|
282
|
+
if (!settings) return null;
|
|
283
|
+
let problem: { message: string; blocking: boolean } | null;
|
|
284
|
+
try {
|
|
285
|
+
problem = await _backupDeps.keyProblem(settings);
|
|
286
|
+
} catch (err) {
|
|
287
|
+
problem = {
|
|
288
|
+
message: err instanceof Error ? err.message : String(err),
|
|
289
|
+
blocking: true,
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
// A check that finished after a reconfiguration describes old settings.
|
|
293
|
+
if (state.settings !== settings) return state.keyProblem ?? null;
|
|
294
|
+
const previous = state.keyProblem;
|
|
295
|
+
state.keyProblem = problem?.message;
|
|
296
|
+
if (problem && previous === undefined) {
|
|
297
|
+
await reportKeyProblem(problem);
|
|
298
|
+
} else if (!problem && previous !== undefined) {
|
|
299
|
+
await reportKeyRecovered();
|
|
300
|
+
}
|
|
301
|
+
return problem?.message ?? null;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
async function reportKeyProblem(problem: {
|
|
305
|
+
message: string;
|
|
306
|
+
blocking: boolean;
|
|
307
|
+
}): Promise<void> {
|
|
308
|
+
const consequence = problem.blocking
|
|
309
|
+
? "Backups and the pre-update checkpoint will fail until it is fixed, and /update is refused unless forced."
|
|
310
|
+
: "Backups still run for now.";
|
|
311
|
+
const text = `Backup key problem: ${faultText(problem.message, 300)}. ${consequence}`;
|
|
312
|
+
logError("backup", text);
|
|
313
|
+
if (state.notify === notifyAdmin) {
|
|
314
|
+
raiseAlert(KEY_ALERT, text, {
|
|
315
|
+
severity: problem.blocking ? "error" : "warn",
|
|
316
|
+
});
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
await state.notify(`⚠️ ${text}`).catch(() => {
|
|
320
|
+
/* the notifier logs its own failures */
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
async function reportKeyRecovered(): Promise<void> {
|
|
325
|
+
const text = "The backup passphrase is readable again.";
|
|
326
|
+
log("backup", text);
|
|
327
|
+
if (state.notify === notifyAdmin) {
|
|
328
|
+
resolveAlert(KEY_ALERT, text);
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
await state.notify(`✅ ${text}`).catch(() => {
|
|
332
|
+
/* the notifier logs its own failures */
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function armKeyCheck(): void {
|
|
337
|
+
if (state.keyTimer) clearInterval(state.keyTimer);
|
|
338
|
+
state.keyTimer = setInterval(() => void checkBackupKey(), KEY_CHECK_MS);
|
|
339
|
+
state.keyTimer.unref?.();
|
|
340
|
+
}
|
|
341
|
+
|
|
257
342
|
/**
|
|
258
343
|
* Wire the subsystem and arm the timer. Safe to call with backups
|
|
259
344
|
* disabled — it reconciles the index either way, so the listing surfaces
|
|
@@ -274,6 +359,10 @@ export async function initBackup(options: {
|
|
|
274
359
|
const newest = (await listLocalManifests(state.home))[0];
|
|
275
360
|
state.lastRunAt = newest?.createdAt ?? 0;
|
|
276
361
|
state.lastSnapshotId = newest?.id;
|
|
362
|
+
// The key is checked whether or not the schedule runs: manual
|
|
363
|
+
// checkpoints and the pre-update checkpoint need it too.
|
|
364
|
+
await checkBackupKey();
|
|
365
|
+
armKeyCheck();
|
|
277
366
|
if (!options.settings.enabled) {
|
|
278
367
|
log(
|
|
279
368
|
"backup",
|
|
@@ -301,6 +390,8 @@ export function stopBackupScheduler(): void {
|
|
|
301
390
|
if (state.timer) clearTimeout(state.timer);
|
|
302
391
|
state.timer = null;
|
|
303
392
|
state.nextRunAt = undefined;
|
|
393
|
+
if (state.keyTimer) clearInterval(state.keyTimer);
|
|
394
|
+
state.keyTimer = null;
|
|
304
395
|
}
|
|
305
396
|
|
|
306
397
|
/** What the status surfaces report about the schedule itself. */
|
|
@@ -326,18 +417,39 @@ export function schedulerStatus(): {
|
|
|
326
417
|
};
|
|
327
418
|
}
|
|
328
419
|
|
|
420
|
+
/**
|
|
421
|
+
* What the pre-update hook managed. `disabled` is the operator's explicit
|
|
422
|
+
* opt-out (`backup.checkpointBeforeUpdate: false`); `failed` covers both a
|
|
423
|
+
* run that threw and a process with no backup subsystem to ask.
|
|
424
|
+
*/
|
|
425
|
+
export type UpdateCheckpoint =
|
|
426
|
+
| { status: "taken"; id: string }
|
|
427
|
+
| { status: "disabled" }
|
|
428
|
+
| { status: "failed"; error: string };
|
|
429
|
+
|
|
329
430
|
/**
|
|
330
431
|
* The self-update hook: a pinned checkpoint before the tree moves, so a
|
|
331
|
-
* bad update is one `talon backup restore` away from undone.
|
|
332
|
-
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
432
|
+
* bad update is one `talon backup restore` away from undone.
|
|
433
|
+
*
|
|
434
|
+
* Never throws — it reports. The caller decides: `/update` refuses to go
|
|
435
|
+
* on after a `failed` checkpoint unless the operator forces it, because
|
|
436
|
+
* an update with no way back is exactly when data goes missing.
|
|
437
|
+
*
|
|
438
|
+
* `backup.enabled: false` only stops the schedule; manual checkpoints
|
|
439
|
+
* still work, so this one is still taken.
|
|
335
440
|
*/
|
|
336
441
|
export async function checkpointBeforeUpdate(
|
|
337
442
|
fromVersion: string,
|
|
338
443
|
toVersion: string,
|
|
339
|
-
): Promise<
|
|
340
|
-
|
|
444
|
+
): Promise<UpdateCheckpoint> {
|
|
445
|
+
const settings = state.settings;
|
|
446
|
+
if (!settings) {
|
|
447
|
+
return {
|
|
448
|
+
status: "failed",
|
|
449
|
+
error: "the backup subsystem is not running in this process",
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
if (!settings.checkpointBeforeUpdate) return { status: "disabled" };
|
|
341
453
|
try {
|
|
342
454
|
const manifest = await runBackup({
|
|
343
455
|
kind: "checkpoint",
|
|
@@ -345,14 +457,13 @@ export async function checkpointBeforeUpdate(
|
|
|
345
457
|
pinned: true,
|
|
346
458
|
trigger: "pre-update",
|
|
347
459
|
});
|
|
348
|
-
return manifest.id;
|
|
460
|
+
return { status: "taken", id: manifest.id };
|
|
349
461
|
} catch (err) {
|
|
350
|
-
logError(
|
|
351
|
-
|
|
352
|
-
"
|
|
353
|
-
err,
|
|
354
|
-
|
|
355
|
-
return null;
|
|
462
|
+
logError("backup", "Pre-update checkpoint failed", err);
|
|
463
|
+
return {
|
|
464
|
+
status: "failed",
|
|
465
|
+
error: err instanceof Error ? err.message : String(err),
|
|
466
|
+
};
|
|
356
467
|
}
|
|
357
468
|
}
|
|
358
469
|
|
|
@@ -372,5 +483,6 @@ export function _resetBackupScheduler(): void {
|
|
|
372
483
|
state.lastRunAt = 0;
|
|
373
484
|
state.lastSnapshotId = undefined;
|
|
374
485
|
state.lastError = undefined;
|
|
486
|
+
state.keyProblem = undefined;
|
|
375
487
|
backoff.succeed();
|
|
376
488
|
}
|
|
@@ -15,8 +15,12 @@ import {
|
|
|
15
15
|
selectTargets,
|
|
16
16
|
type BackupTarget,
|
|
17
17
|
} from "./targets.js";
|
|
18
|
-
import {
|
|
19
|
-
|
|
18
|
+
import {
|
|
19
|
+
backupSettings,
|
|
20
|
+
checkBackupKey,
|
|
21
|
+
schedulerStatus,
|
|
22
|
+
} from "./scheduler.js";
|
|
23
|
+
import { PASSPHRASE_ENV, passphraseProblem } from "./passphrase.js";
|
|
20
24
|
import type { BackupSettings, SnapshotSummary } from "./types.js";
|
|
21
25
|
|
|
22
26
|
type TargetStatus = {
|
|
@@ -41,6 +45,8 @@ export type BackupStatus = {
|
|
|
41
45
|
snapshots: SnapshotSummary[];
|
|
42
46
|
/** Retention and encryption, when the subsystem is initialised. */
|
|
43
47
|
policy?: BackupPolicy;
|
|
48
|
+
/** What is wrong with the configured passphrase, if anything. */
|
|
49
|
+
keyProblem?: string;
|
|
44
50
|
};
|
|
45
51
|
|
|
46
52
|
/** The settings a status panel renders alongside the numbers. */
|
|
@@ -125,12 +131,25 @@ export async function collectBackupStatus(
|
|
|
125
131
|
options: {
|
|
126
132
|
home?: string;
|
|
127
133
|
withTargets?: boolean;
|
|
134
|
+
/**
|
|
135
|
+
* Settings to describe when this process runs no backup subsystem
|
|
136
|
+
* (the CLI with the daemon down). The daemon's own settings win.
|
|
137
|
+
*/
|
|
138
|
+
settings?: BackupSettings;
|
|
128
139
|
} = {},
|
|
129
140
|
): Promise<BackupStatus> {
|
|
130
141
|
const home = options.home ?? dirs.root;
|
|
131
142
|
const snapshots = await listSnapshots(home);
|
|
132
143
|
const local = snapshots.filter((snapshot) => snapshot.local);
|
|
133
|
-
const
|
|
144
|
+
const live = backupSettings();
|
|
145
|
+
const settings = live ?? options.settings ?? null;
|
|
146
|
+
// In the daemon the check also raises (or clears) the key alert; outside
|
|
147
|
+
// it there is nobody to alert, so it only reports.
|
|
148
|
+
const keyProblem = live
|
|
149
|
+
? await checkBackupKey()
|
|
150
|
+
: settings
|
|
151
|
+
? ((await passphraseProblem(settings))?.message ?? null)
|
|
152
|
+
: null;
|
|
134
153
|
const targets =
|
|
135
154
|
options.withTargets === false
|
|
136
155
|
? []
|
|
@@ -148,6 +167,7 @@ export async function collectBackupStatus(
|
|
|
148
167
|
targets,
|
|
149
168
|
snapshots,
|
|
150
169
|
policy: describePolicy(settings),
|
|
170
|
+
...(keyProblem ? { keyProblem } : {}),
|
|
151
171
|
};
|
|
152
172
|
}
|
|
153
173
|
|
|
@@ -198,6 +218,9 @@ export function formatBackupStatus(
|
|
|
198
218
|
`Last run: ${formatRelative(schedule.lastRunAt, now)}` +
|
|
199
219
|
(schedule.lastSnapshotId ? ` — ${schedule.lastSnapshotId}` : ""),
|
|
200
220
|
);
|
|
221
|
+
if (status.keyProblem) {
|
|
222
|
+
lines.push(`Key: PROBLEM — ${status.keyProblem}`);
|
|
223
|
+
}
|
|
201
224
|
if (schedule.consecutiveFailures > 0) {
|
|
202
225
|
lines.push(
|
|
203
226
|
`Failing: ${schedule.consecutiveFailures} consecutive — ${schedule.lastError ?? "unknown error"}`,
|