talon-agent 5.26.3 → 5.26.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/backend/remote-server/sessions.ts +82 -13
- package/src/backend/runtime/turn/handle-retry.ts +6 -5
- package/src/cli/commands/backup.ts +7 -3
- package/src/core/backup/archive/verify.ts +53 -0
- package/src/core/backup/plan.ts +3 -0
- package/src/core/backup/retention/policy.ts +195 -0
- package/src/core/backup/scheduler.ts +10 -4
- package/src/core/backup/snapshot.ts +3 -0
- package/src/core/backup/status.ts +7 -0
- package/src/core/backup/store.ts +20 -25
- package/src/core/backup/types.ts +14 -0
- package/src/core/backup/upload.ts +37 -20
- package/src/core/config/index.ts +26 -1
- package/src/core/errors.ts +13 -2
- package/src/frontend/presentation/backup-panel.ts +7 -1
package/package.json
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
setSessionId,
|
|
16
16
|
} from "../../storage/sessions.js";
|
|
17
17
|
import { log, logWarn } from "../../util/log.js";
|
|
18
|
+
import { TalonError } from "../../core/errors.js";
|
|
18
19
|
import type { RemoteAgentClient, RemotePermissionRule } from "./client.js";
|
|
19
20
|
import type { RemoteServerState } from "./state.js";
|
|
20
21
|
import {
|
|
@@ -73,32 +74,100 @@ export function buildPermissionRuleset(chatId: string): RemotePermissionRule[] {
|
|
|
73
74
|
];
|
|
74
75
|
}
|
|
75
76
|
|
|
77
|
+
/** Pull an HTTP status out of whatever shape the SDK client threw. */
|
|
78
|
+
function errorStatus(err: unknown): number | undefined {
|
|
79
|
+
if (typeof err !== "object" || err === null) return undefined;
|
|
80
|
+
const e = err as Record<string, unknown>;
|
|
81
|
+
const cause = e.cause as Record<string, unknown> | undefined;
|
|
82
|
+
const response = e.response as Record<string, unknown> | undefined;
|
|
83
|
+
for (const candidate of [
|
|
84
|
+
e.status,
|
|
85
|
+
e.statusCode,
|
|
86
|
+
response?.status,
|
|
87
|
+
cause?.status,
|
|
88
|
+
]) {
|
|
89
|
+
if (typeof candidate === "number") return candidate;
|
|
90
|
+
}
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** The error's `name`, from the thrown object or the parsed body behind it. */
|
|
95
|
+
function errorNames(err: unknown): string[] {
|
|
96
|
+
if (typeof err !== "object" || err === null) return [];
|
|
97
|
+
const e = err as Record<string, unknown>;
|
|
98
|
+
const cause = e.cause as Record<string, unknown> | undefined;
|
|
99
|
+
const body = (cause?.body ?? e.body) as Record<string, unknown> | undefined;
|
|
100
|
+
return [e.name, body?.name, (e.data as Record<string, unknown>)?.name].filter(
|
|
101
|
+
(name): name is string => typeof name === "string",
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* True only when the server definitely says the session does not exist:
|
|
107
|
+
* an HTTP 404, or the server's `NotFoundError`. Everything else — a
|
|
108
|
+
* refused connection while the server is still starting, a 5xx, a
|
|
109
|
+
* timeout, a body that would not parse — is not proof the session is
|
|
110
|
+
* gone, and must not cost the chat its session.
|
|
111
|
+
*/
|
|
112
|
+
export function isRemoteSessionNotFound(err: unknown): boolean {
|
|
113
|
+
if (errorStatus(err) === 404) return true;
|
|
114
|
+
return errorNames(err).includes("NotFoundError");
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Waits between `session.get` attempts on a transient failure. */
|
|
118
|
+
const RESUME_RETRY_DELAYS_MS: readonly number[] = [500, 1_500];
|
|
119
|
+
|
|
76
120
|
/**
|
|
77
121
|
* Ensure a session exists for this chat on the remote agent server.
|
|
78
122
|
*
|
|
79
123
|
* Resumes the stored session id if `session.get` confirms it's still
|
|
80
|
-
* alive.
|
|
81
|
-
* resets
|
|
82
|
-
*
|
|
83
|
-
*
|
|
124
|
+
* alive. Only a definite not-found (see {@link isRemoteSessionNotFound})
|
|
125
|
+
* resets the chat — archiving the old id — and creates a fresh session
|
|
126
|
+
* with Talon's standard permission ruleset (see
|
|
127
|
+
* {@link buildPermissionRuleset}). Any other failure is retried a couple
|
|
128
|
+
* of times and then fails the turn with the session left in place: a
|
|
129
|
+
* server that is still starting after an update must not wipe every
|
|
130
|
+
* chat's session mapping.
|
|
84
131
|
*/
|
|
85
132
|
export async function ensureRemoteSession<TClient extends RemoteAgentClient>(
|
|
86
133
|
client: TClient,
|
|
87
134
|
state: RemoteServerState<TClient>,
|
|
88
135
|
chatId: string,
|
|
136
|
+
retryDelaysMs: readonly number[] = RESUME_RETRY_DELAYS_MS,
|
|
89
137
|
): Promise<string> {
|
|
90
138
|
const session = getSession(chatId);
|
|
91
139
|
|
|
92
140
|
if (session.sessionId) {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
141
|
+
const sessionId = session.sessionId;
|
|
142
|
+
for (let attempt = 0; ; attempt++) {
|
|
143
|
+
try {
|
|
144
|
+
await client.session.get({ sessionID: sessionId });
|
|
145
|
+
return sessionId;
|
|
146
|
+
} catch (err) {
|
|
147
|
+
if (isRemoteSessionNotFound(err)) {
|
|
148
|
+
logWarn(
|
|
149
|
+
"agent",
|
|
150
|
+
`[${chatId}] ${state.label} session ${sessionId} not found on the server, creating new`,
|
|
151
|
+
);
|
|
152
|
+
resetSession(chatId, "remote_session_not_found");
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
const delay = retryDelaysMs[attempt];
|
|
156
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
157
|
+
if (delay === undefined) {
|
|
158
|
+
throw new TalonError(
|
|
159
|
+
`Could not check ${state.label} session ${sessionId} (${detail}); ` +
|
|
160
|
+
`kept it — try again once the server is up`,
|
|
161
|
+
{ reason: "network", retryable: true, cause: err },
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
logWarn(
|
|
165
|
+
"agent",
|
|
166
|
+
`[${chatId}] ${state.label} session check failed (${detail}); ` +
|
|
167
|
+
`retrying in ${delay}ms, session kept`,
|
|
168
|
+
);
|
|
169
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
170
|
+
}
|
|
102
171
|
}
|
|
103
172
|
}
|
|
104
173
|
|
|
@@ -111,7 +111,8 @@ export interface ApplyRetryDecisionResult {
|
|
|
111
111
|
*
|
|
112
112
|
* Side effects (when a retry fires):
|
|
113
113
|
* - `incrementCounter('errors.<reason>')` exactly once per call.
|
|
114
|
-
* - `resetSession(chatId)` before recursion
|
|
114
|
+
* - `resetSession(chatId, reason)` before recursion (the old id is
|
|
115
|
+
* archived under that reason).
|
|
115
116
|
* - For `fallback_model`: the fallback model id is spread into the
|
|
116
117
|
* recursion's params (`params.model` outranks chat settings).
|
|
117
118
|
*
|
|
@@ -156,7 +157,7 @@ export async function applyRetryDecision(
|
|
|
156
157
|
"agent",
|
|
157
158
|
`[${chatId}] ${prefix}${decision.reason}, resetting ${resetNoun} and retrying`,
|
|
158
159
|
);
|
|
159
|
-
resetSession(chatId);
|
|
160
|
+
resetSession(chatId, decision.reason);
|
|
160
161
|
return { retry: await recurseWithRetried(params), classified };
|
|
161
162
|
}
|
|
162
163
|
|
|
@@ -165,7 +166,7 @@ export async function applyRetryDecision(
|
|
|
165
166
|
"agent",
|
|
166
167
|
`[${chatId}] ${classified.reason}, falling back to ${decision.fallbackModelId}`,
|
|
167
168
|
);
|
|
168
|
-
resetSession(chatId);
|
|
169
|
+
resetSession(chatId, `fallback_model:${decision.fallbackModelId}`);
|
|
169
170
|
return {
|
|
170
171
|
retry: await recurseWithRetried({
|
|
171
172
|
...params,
|
|
@@ -270,7 +271,7 @@ export async function* applyRetryDecisionStream(
|
|
|
270
271
|
"agent",
|
|
271
272
|
`[${chatId}] ${prefix}${decision.reason}, resetting ${resetNoun} and retrying`,
|
|
272
273
|
);
|
|
273
|
-
resetSession(chatId);
|
|
274
|
+
resetSession(chatId, decision.reason);
|
|
274
275
|
yield* buildRetryStream();
|
|
275
276
|
return { retried: true, classified };
|
|
276
277
|
}
|
|
@@ -280,7 +281,7 @@ export async function* applyRetryDecisionStream(
|
|
|
280
281
|
"agent",
|
|
281
282
|
`[${chatId}] ${classified.reason}, falling back to ${decision.fallbackModelId}`,
|
|
282
283
|
);
|
|
283
|
-
resetSession(chatId);
|
|
284
|
+
resetSession(chatId, `fallback_model:${decision.fallbackModelId}`);
|
|
284
285
|
yield* buildRetryStream(decision.fallbackModelId);
|
|
285
286
|
return { retried: true, classified };
|
|
286
287
|
}
|
|
@@ -46,6 +46,10 @@ import {
|
|
|
46
46
|
import { resolveBackupSettings } from "../../core/backup/plan.js";
|
|
47
47
|
import { buildSnapshot } from "../../core/backup/snapshot.js";
|
|
48
48
|
import { pruneLocal, reconcileIndex } from "../../core/backup/store.js";
|
|
49
|
+
import {
|
|
50
|
+
describeRetention,
|
|
51
|
+
localRetention,
|
|
52
|
+
} from "../../core/backup/retention/policy.js";
|
|
49
53
|
import { generatePassphraseFile } from "../../core/backup/passphrase.js";
|
|
50
54
|
import { dirs } from "../../util/paths.js";
|
|
51
55
|
import { fetchGateway } from "../daemon-api.js";
|
|
@@ -170,7 +174,7 @@ async function backupNow(flags: Flags): Promise<void> {
|
|
|
170
174
|
pinned: pin,
|
|
171
175
|
settings,
|
|
172
176
|
});
|
|
173
|
-
await pruneLocal(settings
|
|
177
|
+
await pruneLocal(localRetention(settings));
|
|
174
178
|
console.log(
|
|
175
179
|
` ${pc.green("●")} ${manifest.id} — ${manifest.parts.length} part(s), ` +
|
|
176
180
|
`${formatBytes(manifest.sizeBytes)}\n`,
|
|
@@ -246,10 +250,10 @@ async function backupPin(id: string, pinned: boolean): Promise<void> {
|
|
|
246
250
|
|
|
247
251
|
async function backupPrune(): Promise<void> {
|
|
248
252
|
const settings = resolveBackupSettings(loadConfig().backup);
|
|
249
|
-
const removed = await pruneLocal(settings
|
|
253
|
+
const removed = await pruneLocal(localRetention(settings));
|
|
250
254
|
console.log(
|
|
251
255
|
removed.length === 0
|
|
252
|
-
? ` ${pc.dim(`Nothing to prune (
|
|
256
|
+
? ` ${pc.dim(`Nothing to prune (${describeRetention(localRetention(settings))}).`)}\n`
|
|
253
257
|
: ` ${pc.green("●")} Pruned ${removed.length}: ${removed.join(", ")}\n`,
|
|
254
258
|
);
|
|
255
259
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-back verification of a freshly written snapshot.
|
|
3
|
+
*
|
|
4
|
+
* A digest taken off the bytes on their way to disk says what was sent,
|
|
5
|
+
* not what landed. Before a snapshot is allowed to count as good, every
|
|
6
|
+
* part is read back from disk and re-hashed, and an encrypted part is
|
|
7
|
+
* decrypted end to end (and discarded) so a broken encryptor cannot
|
|
8
|
+
* produce a snapshot nobody can open. The result is `verifiedAt` in the
|
|
9
|
+
* manifest, and retention never prunes the newest snapshot that has one
|
|
10
|
+
* (see retention/policy.ts).
|
|
11
|
+
*
|
|
12
|
+
* Runs before the manifest is signed, so `verifiedAt` is covered by the
|
|
13
|
+
* MAC like every other field written at build time.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { TalonError } from "../../errors.js";
|
|
18
|
+
import { isEncryptedFile, verifyDecryptable } from "./crypt.js";
|
|
19
|
+
import { sha256File } from "./digest.js";
|
|
20
|
+
import type { SnapshotPart } from "../types.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Throws a TalonError naming the first part that does not read back.
|
|
24
|
+
* Returns the time verification finished.
|
|
25
|
+
*/
|
|
26
|
+
export async function verifyWrittenParts(
|
|
27
|
+
dir: string,
|
|
28
|
+
parts: readonly SnapshotPart[],
|
|
29
|
+
passphrase: string | null,
|
|
30
|
+
now: () => number = Date.now,
|
|
31
|
+
): Promise<number> {
|
|
32
|
+
for (const part of parts) {
|
|
33
|
+
const path = join(dir, part.name);
|
|
34
|
+
const actual = await sha256File(path);
|
|
35
|
+
if (actual !== part.sha256) {
|
|
36
|
+
throw new TalonError(
|
|
37
|
+
`Part ${part.name} does not read back (sha256 mismatch)`,
|
|
38
|
+
{ reason: "unknown" },
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
if (passphrase && (await isEncryptedFile(path))) {
|
|
42
|
+
try {
|
|
43
|
+
await verifyDecryptable(path, passphrase);
|
|
44
|
+
} catch (err) {
|
|
45
|
+
throw new TalonError(
|
|
46
|
+
`Part ${part.name} cannot be decrypted after writing: ${err instanceof Error ? err.message : String(err)}`,
|
|
47
|
+
{ reason: "unknown", cause: err },
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return now();
|
|
53
|
+
}
|
package/src/core/backup/plan.ts
CHANGED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Snapshot retention — which snapshots a prune pass may delete.
|
|
3
|
+
*
|
|
4
|
+
* Keeping only the newest N is not a safety net: if something silently
|
|
5
|
+
* wipes memory, the next N scheduled snapshots faithfully back up the
|
|
6
|
+
* damage and every good copy ages out within N × intervalHours (three
|
|
7
|
+
* days at the defaults). So retention is tiered, the way restic and
|
|
8
|
+
* borg do it:
|
|
9
|
+
*
|
|
10
|
+
* - `keepLast` the newest N scheduled snapshots (`keepLocal` /
|
|
11
|
+
* `keepRemote` in config — their old meaning).
|
|
12
|
+
* - `keepDaily` the newest snapshot of each of the last D days that
|
|
13
|
+
* have one.
|
|
14
|
+
* - `keepWeekly` the newest snapshot of each of the last W ISO weeks
|
|
15
|
+
* that have one.
|
|
16
|
+
*
|
|
17
|
+
* Days and weeks are counted over snapshots that exist, not over the
|
|
18
|
+
* calendar, so a machine that was off for a month does not lose its
|
|
19
|
+
* history to the clock.
|
|
20
|
+
*
|
|
21
|
+
* Around the tiers sit three guard rails:
|
|
22
|
+
*
|
|
23
|
+
* - Pinned snapshots are kept and count against nothing.
|
|
24
|
+
* - Checkpoints (manual, pre-update, pre-upgrade, pre-restore) have
|
|
25
|
+
* their own cap, `keepCheckpoints`, and never displace a scheduled
|
|
26
|
+
* snapshot — nor are they displaced by one.
|
|
27
|
+
* - The newest snapshot that passed verification is always kept, so a
|
|
28
|
+
* run of corrupt snapshots can never prune the last known-good one.
|
|
29
|
+
*
|
|
30
|
+
* Entries whose `createdAt` is not a real timestamp (an unreadable or
|
|
31
|
+
* partial remote manifest) are never pruned: without an age there is no
|
|
32
|
+
* honest way to rank them, and deleting what you cannot read is how a
|
|
33
|
+
* retention pass becomes data loss. They are reported as `skipped` so
|
|
34
|
+
* the caller can warn.
|
|
35
|
+
*
|
|
36
|
+
* Pure: no clock, no filesystem.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/** The tiers one prune pass applies. */
|
|
40
|
+
export type RetentionPolicy = {
|
|
41
|
+
/** Newest N scheduled snapshots. */
|
|
42
|
+
keepLast: number;
|
|
43
|
+
/** Newest snapshot per day, for the last N days that have one. 0 = off. */
|
|
44
|
+
keepDaily: number;
|
|
45
|
+
/** Newest snapshot per ISO week, for the last N weeks that have one. 0 = off. */
|
|
46
|
+
keepWeekly: number;
|
|
47
|
+
/** Newest N unpinned checkpoints, counted apart from scheduled snapshots. */
|
|
48
|
+
keepCheckpoints: number;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/** What retention needs to know about one snapshot. */
|
|
52
|
+
export type RetentionEntry = {
|
|
53
|
+
id: string;
|
|
54
|
+
createdAt: number;
|
|
55
|
+
pinned: boolean;
|
|
56
|
+
/** "backup" | "checkpoint"; anything else is treated as "backup". */
|
|
57
|
+
kind?: string;
|
|
58
|
+
/** Epoch ms of a successful post-write verification, if there was one. */
|
|
59
|
+
verifiedAt?: number;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
/** Why a snapshot survived, for logs and tests. */
|
|
63
|
+
type KeepReason =
|
|
64
|
+
"pinned" | "last" | "daily" | "weekly" | "checkpoint" | "verified";
|
|
65
|
+
|
|
66
|
+
export type RetentionPlan<T> = {
|
|
67
|
+
/** Snapshots to delete, newest first. */
|
|
68
|
+
prune: T[];
|
|
69
|
+
/** id → every reason it was kept. */
|
|
70
|
+
keep: Map<string, KeepReason[]>;
|
|
71
|
+
/** Entries without a usable createdAt — kept, and worth a warning. */
|
|
72
|
+
skipped: T[];
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/** Settings shape the policy is derived from (a subset of BackupSettings). */
|
|
76
|
+
type RetentionSettings = {
|
|
77
|
+
keepLocal: number;
|
|
78
|
+
keepRemote: number;
|
|
79
|
+
keepDaily: number;
|
|
80
|
+
keepWeekly: number;
|
|
81
|
+
keepCheckpoints: number;
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
export function localRetention(settings: RetentionSettings): RetentionPolicy {
|
|
85
|
+
return {
|
|
86
|
+
keepLast: settings.keepLocal,
|
|
87
|
+
keepDaily: settings.keepDaily,
|
|
88
|
+
keepWeekly: settings.keepWeekly,
|
|
89
|
+
keepCheckpoints: settings.keepCheckpoints,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function remoteRetention(settings: RetentionSettings): RetentionPolicy {
|
|
94
|
+
return { ...localRetention(settings), keepLast: settings.keepRemote };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** One-line description for logs: `last=12 daily=7 weekly=8 checkpoints=10`. */
|
|
98
|
+
export function describeRetention(policy: RetentionPolicy): string {
|
|
99
|
+
return (
|
|
100
|
+
`last=${policy.keepLast} daily=${policy.keepDaily} ` +
|
|
101
|
+
`weekly=${policy.keepWeekly} checkpoints=${policy.keepCheckpoints}`
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** A createdAt a snapshot can be ranked by. */
|
|
106
|
+
function hasUsableTimestamp(entry: { createdAt: unknown }): boolean {
|
|
107
|
+
return (
|
|
108
|
+
typeof entry.createdAt === "number" &&
|
|
109
|
+
Number.isFinite(entry.createdAt) &&
|
|
110
|
+
entry.createdAt > 0
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** `2026-09-30` in UTC. */
|
|
115
|
+
function dayKey(at: number): string {
|
|
116
|
+
return new Date(at).toISOString().slice(0, 10);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** ISO-8601 week, UTC: `2026-W40`. */
|
|
120
|
+
function weekKey(at: number): string {
|
|
121
|
+
const date = new Date(at);
|
|
122
|
+
const day = date.getUTCDay() || 7; // Mon=1 … Sun=7
|
|
123
|
+
// The Thursday of this week decides which year the week belongs to.
|
|
124
|
+
const thursday = Date.UTC(
|
|
125
|
+
date.getUTCFullYear(),
|
|
126
|
+
date.getUTCMonth(),
|
|
127
|
+
date.getUTCDate() + 4 - day,
|
|
128
|
+
);
|
|
129
|
+
const year = new Date(thursday).getUTCFullYear();
|
|
130
|
+
const week = Math.ceil(
|
|
131
|
+
((thursday - Date.UTC(year, 0, 1)) / 86_400_000 + 1) / 7,
|
|
132
|
+
);
|
|
133
|
+
return `${year}-W${String(week).padStart(2, "0")}`;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Keep the newest entry of each of the first `limit` distinct buckets. */
|
|
137
|
+
function keepPerBucket<T extends RetentionEntry>(
|
|
138
|
+
ordered: readonly T[],
|
|
139
|
+
limit: number,
|
|
140
|
+
bucketOf: (at: number) => string,
|
|
141
|
+
reason: KeepReason,
|
|
142
|
+
mark: (entry: T, reason: KeepReason) => void,
|
|
143
|
+
): void {
|
|
144
|
+
if (limit <= 0) return;
|
|
145
|
+
const seen = new Set<string>();
|
|
146
|
+
for (const entry of ordered) {
|
|
147
|
+
const bucket = bucketOf(entry.createdAt);
|
|
148
|
+
if (seen.has(bucket)) continue;
|
|
149
|
+
seen.add(bucket);
|
|
150
|
+
mark(entry, reason);
|
|
151
|
+
if (seen.size >= limit) return;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Decide what survives. See the module comment for the rules. */
|
|
156
|
+
export function planRetention<T extends RetentionEntry>(
|
|
157
|
+
snapshots: readonly T[],
|
|
158
|
+
policy: RetentionPolicy,
|
|
159
|
+
): RetentionPlan<T> {
|
|
160
|
+
const keep = new Map<string, KeepReason[]>();
|
|
161
|
+
const mark = (entry: T, reason: KeepReason): void => {
|
|
162
|
+
const reasons = keep.get(entry.id);
|
|
163
|
+
if (reasons) reasons.push(reason);
|
|
164
|
+
else keep.set(entry.id, [reason]);
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
const skipped = snapshots.filter((entry) => !hasUsableTimestamp(entry));
|
|
168
|
+
const ranked = snapshots
|
|
169
|
+
.filter(hasUsableTimestamp)
|
|
170
|
+
.sort((a, b) => b.createdAt - a.createdAt);
|
|
171
|
+
|
|
172
|
+
for (const entry of ranked) if (entry.pinned) mark(entry, "pinned");
|
|
173
|
+
|
|
174
|
+
const unpinned = ranked.filter((entry) => !entry.pinned);
|
|
175
|
+
const checkpoints = unpinned.filter((entry) => entry.kind === "checkpoint");
|
|
176
|
+
const scheduled = unpinned.filter((entry) => entry.kind !== "checkpoint");
|
|
177
|
+
|
|
178
|
+
for (const entry of checkpoints.slice(0, Math.max(0, policy.keepCheckpoints)))
|
|
179
|
+
mark(entry, "checkpoint");
|
|
180
|
+
for (const entry of scheduled.slice(0, Math.max(0, policy.keepLast)))
|
|
181
|
+
mark(entry, "last");
|
|
182
|
+
keepPerBucket(scheduled, policy.keepDaily, dayKey, "daily", mark);
|
|
183
|
+
keepPerBucket(scheduled, policy.keepWeekly, weekKey, "weekly", mark);
|
|
184
|
+
|
|
185
|
+
const lastVerified = ranked.find(
|
|
186
|
+
(entry) => typeof entry.verifiedAt === "number" && entry.verifiedAt > 0,
|
|
187
|
+
);
|
|
188
|
+
if (lastVerified) mark(lastVerified, "verified");
|
|
189
|
+
|
|
190
|
+
return {
|
|
191
|
+
prune: ranked.filter((entry) => !keep.has(entry.id)),
|
|
192
|
+
keep,
|
|
193
|
+
skipped,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
@@ -34,6 +34,11 @@ import { dirs } from "../../util/paths.js";
|
|
|
34
34
|
import { passphraseProblem } from "./passphrase.js";
|
|
35
35
|
import { buildSnapshot } from "./snapshot.js";
|
|
36
36
|
import { listLocalManifests, pruneLocal, reconcileIndex } from "./store.js";
|
|
37
|
+
import {
|
|
38
|
+
describeRetention,
|
|
39
|
+
localRetention,
|
|
40
|
+
remoteRetention,
|
|
41
|
+
} from "./retention/policy.js";
|
|
37
42
|
import { discoverTargets, selectTargets } from "./targets.js";
|
|
38
43
|
import { pruneRemote, uploadSnapshot } from "./upload.js";
|
|
39
44
|
import type { BackupSettings, Manifest, SnapshotKind } from "./types.js";
|
|
@@ -162,7 +167,7 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
|
|
|
162
167
|
parts: manifest.parts.length,
|
|
163
168
|
durationMs: Date.now() - started,
|
|
164
169
|
});
|
|
165
|
-
await _backupDeps.pruneLocal(settings
|
|
170
|
+
await _backupDeps.pruneLocal(localRetention(settings), state.home);
|
|
166
171
|
if (!request.localOnly) {
|
|
167
172
|
const targets = selectTargets(
|
|
168
173
|
await _backupDeps.discover(),
|
|
@@ -170,7 +175,7 @@ async function executeRun(request: RunRequest): Promise<Manifest> {
|
|
|
170
175
|
);
|
|
171
176
|
if (targets.length > 0) {
|
|
172
177
|
await _backupDeps.upload(manifest, targets, state.home);
|
|
173
|
-
await _backupDeps.pruneRemote(targets, settings
|
|
178
|
+
await _backupDeps.pruneRemote(targets, remoteRetention(settings));
|
|
174
179
|
}
|
|
175
180
|
}
|
|
176
181
|
backoff.succeed();
|
|
@@ -379,8 +384,9 @@ export async function initBackup(options: {
|
|
|
379
384
|
log(
|
|
380
385
|
"backup",
|
|
381
386
|
`Scheduled every ${options.settings.intervalHours}h; first run in ` +
|
|
382
|
-
`${Math.round(delay / 60_000)}m (
|
|
383
|
-
`${options.settings
|
|
387
|
+
`${Math.round(delay / 60_000)}m (retention ` +
|
|
388
|
+
`${describeRetention(localRetention(options.settings))}; ` +
|
|
389
|
+
`${options.settings.keepRemote} newest per remote target)`,
|
|
384
390
|
);
|
|
385
391
|
}
|
|
386
392
|
|
|
@@ -54,6 +54,7 @@ import {
|
|
|
54
54
|
import { signManifest } from "./archive/manifest-auth.js";
|
|
55
55
|
import { TarWriter } from "./archive/tar.js";
|
|
56
56
|
import { createCompressor } from "./archive/zstd.js";
|
|
57
|
+
import { verifyWrittenParts } from "./archive/verify.js";
|
|
57
58
|
import {
|
|
58
59
|
collectTree,
|
|
59
60
|
excludeForRoot,
|
|
@@ -755,6 +756,7 @@ export async function buildSnapshot(options: BuildOptions): Promise<Manifest> {
|
|
|
755
756
|
}
|
|
756
757
|
}
|
|
757
758
|
await removeScratch(dir);
|
|
759
|
+
const verifiedAt = await verifyWrittenParts(dir, parts, passphrase);
|
|
758
760
|
|
|
759
761
|
const gitHead = await readGitHead(process.cwd());
|
|
760
762
|
const manifest: Manifest = {
|
|
@@ -776,6 +778,7 @@ export async function buildSnapshot(options: BuildOptions): Promise<Manifest> {
|
|
|
776
778
|
...(palaceHash ? { palaceHash } : {}),
|
|
777
779
|
sizeBytes: parts.reduce((sum, part) => sum + part.bytes, 0),
|
|
778
780
|
remote: {},
|
|
781
|
+
verifiedAt,
|
|
779
782
|
};
|
|
780
783
|
if (passphrase) manifest.auth = await signManifest(manifest, passphrase);
|
|
781
784
|
await writeManifest(manifest, home);
|
|
@@ -53,6 +53,10 @@ export type BackupStatus = {
|
|
|
53
53
|
type BackupPolicy = {
|
|
54
54
|
keepLocal: number;
|
|
55
55
|
keepRemote: number;
|
|
56
|
+
/** Absent on a policy built before tiered retention (older daemons). */
|
|
57
|
+
keepDaily?: number;
|
|
58
|
+
keepWeekly?: number;
|
|
59
|
+
keepCheckpoints?: number;
|
|
56
60
|
/**
|
|
57
61
|
* Snapshots are written encrypted: `backup.encryption` is configured or
|
|
58
62
|
* the passphrase comes from the environment (see passphrase.ts). Off
|
|
@@ -69,6 +73,9 @@ function describePolicy(
|
|
|
69
73
|
return {
|
|
70
74
|
keepLocal: settings.keepLocal,
|
|
71
75
|
keepRemote: settings.keepRemote,
|
|
76
|
+
keepDaily: settings.keepDaily,
|
|
77
|
+
keepWeekly: settings.keepWeekly,
|
|
78
|
+
keepCheckpoints: settings.keepCheckpoints,
|
|
72
79
|
encrypted:
|
|
73
80
|
settings.encryption !== undefined || Boolean(env[PASSPHRASE_ENV]?.trim()),
|
|
74
81
|
};
|
package/src/core/backup/store.ts
CHANGED
|
@@ -33,6 +33,11 @@ import {
|
|
|
33
33
|
updateBackupManifest,
|
|
34
34
|
} from "../../storage/backup/index.js";
|
|
35
35
|
import { pathExists } from "./sources/sessions.js";
|
|
36
|
+
import {
|
|
37
|
+
describeRetention,
|
|
38
|
+
planRetention,
|
|
39
|
+
type RetentionPolicy,
|
|
40
|
+
} from "./retention/policy.js";
|
|
36
41
|
import type {
|
|
37
42
|
Manifest,
|
|
38
43
|
RemoteState,
|
|
@@ -236,26 +241,6 @@ export async function setSnapshotPinned(
|
|
|
236
241
|
|
|
237
242
|
// ── Retention ───────────────────────────────────────────────────────────────
|
|
238
243
|
|
|
239
|
-
/**
|
|
240
|
-
* Which snapshots to drop so that at most `keep` unpinned ones remain.
|
|
241
|
-
* Pure, and the order is the one the caller sees: newest first, pinned
|
|
242
|
-
* snapshots kept unconditionally and not counted against the budget —
|
|
243
|
-
* pinning is the user saying "this one outlives the policy".
|
|
244
|
-
*/
|
|
245
|
-
export function selectPrunable<
|
|
246
|
-
T extends { id: string; createdAt: number; pinned: boolean },
|
|
247
|
-
>(snapshots: readonly T[], keep: number): T[] {
|
|
248
|
-
const ordered = [...snapshots].sort((a, b) => b.createdAt - a.createdAt);
|
|
249
|
-
const doomed: T[] = [];
|
|
250
|
-
let kept = 0;
|
|
251
|
-
for (const snapshot of ordered) {
|
|
252
|
-
if (snapshot.pinned) continue;
|
|
253
|
-
kept += 1;
|
|
254
|
-
if (kept > keep) doomed.push(snapshot);
|
|
255
|
-
}
|
|
256
|
-
return doomed;
|
|
257
|
-
}
|
|
258
|
-
|
|
259
244
|
/** Delete a snapshot directory and its index rows. */
|
|
260
245
|
async function removeSnapshot(
|
|
261
246
|
id: string,
|
|
@@ -266,14 +251,24 @@ async function removeSnapshot(
|
|
|
266
251
|
deleteBackup(id);
|
|
267
252
|
}
|
|
268
253
|
|
|
269
|
-
/**
|
|
254
|
+
/**
|
|
255
|
+
* Apply the local retention policy (see retention/policy.ts). Returns the ids
|
|
256
|
+
* removed. A directory whose manifest cannot be read never reaches the
|
|
257
|
+
* listing, so it is never pruned.
|
|
258
|
+
*/
|
|
270
259
|
export async function pruneLocal(
|
|
271
|
-
|
|
260
|
+
policy: RetentionPolicy,
|
|
272
261
|
home: string = dirs.root,
|
|
273
262
|
): Promise<string[]> {
|
|
274
|
-
const
|
|
263
|
+
const plan = planRetention(await listLocalManifests(home), policy);
|
|
264
|
+
for (const manifest of plan.skipped) {
|
|
265
|
+
logWarn(
|
|
266
|
+
"backup",
|
|
267
|
+
`Not pruning ${manifest.id}: its manifest has no usable createdAt`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
275
270
|
const removed: string[] = [];
|
|
276
|
-
for (const manifest of
|
|
271
|
+
for (const manifest of plan.prune) {
|
|
277
272
|
try {
|
|
278
273
|
await removeSnapshot(manifest.id, home);
|
|
279
274
|
removed.push(manifest.id);
|
|
@@ -284,7 +279,7 @@ export async function pruneLocal(
|
|
|
284
279
|
if (removed.length > 0) {
|
|
285
280
|
log(
|
|
286
281
|
"backup",
|
|
287
|
-
`Pruned ${removed.length} local snapshot(s) (
|
|
282
|
+
`Pruned ${removed.length} local snapshot(s) (${describeRetention(policy)})`,
|
|
288
283
|
);
|
|
289
284
|
}
|
|
290
285
|
return removed;
|
package/src/core/backup/types.ts
CHANGED
|
@@ -123,6 +123,14 @@ export type Manifest = {
|
|
|
123
123
|
/** Total bytes of all parts. */
|
|
124
124
|
sizeBytes: number;
|
|
125
125
|
remote: Record<string, RemoteState>;
|
|
126
|
+
/**
|
|
127
|
+
* Epoch ms at which every part was read back and matched its digest
|
|
128
|
+
* (and, when encrypted, decrypted end to end) — see archive/verify.ts. Set
|
|
129
|
+
* before signing, so the MAC covers it. Absent on snapshots written
|
|
130
|
+
* before verification existed. Retention never prunes the newest
|
|
131
|
+
* verified snapshot.
|
|
132
|
+
*/
|
|
133
|
+
verifiedAt?: number;
|
|
126
134
|
/** MAC under the backup passphrase; absent on plaintext/legacy snapshots. */
|
|
127
135
|
auth?: ManifestAuth;
|
|
128
136
|
};
|
|
@@ -146,6 +154,12 @@ export type BackupSettings = {
|
|
|
146
154
|
intervalHours: number;
|
|
147
155
|
keepLocal: number;
|
|
148
156
|
keepRemote: number;
|
|
157
|
+
/** Newest snapshot per day, for this many days (0 = off). */
|
|
158
|
+
keepDaily: number;
|
|
159
|
+
/** Newest snapshot per ISO week, for this many weeks (0 = off). */
|
|
160
|
+
keepWeekly: number;
|
|
161
|
+
/** Unpinned checkpoints kept, apart from the scheduled snapshots. */
|
|
162
|
+
keepCheckpoints: number;
|
|
149
163
|
includePalace: boolean;
|
|
150
164
|
/** WhatsApp auth + userbot session: see {@link LoginSessionsPolicy}. */
|
|
151
165
|
loginSessions: LoginSessionsPolicy;
|
|
@@ -30,11 +30,11 @@ import {
|
|
|
30
30
|
} from "../../storage/backup/index.js";
|
|
31
31
|
import { isEncryptedFile } from "./archive/crypt.js";
|
|
32
32
|
import {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
} from "./store.js";
|
|
33
|
+
describeRetention,
|
|
34
|
+
planRetention,
|
|
35
|
+
type RetentionPolicy,
|
|
36
|
+
} from "./retention/policy.js";
|
|
37
|
+
import { partPath, reindexSnapshot, writeManifest } from "./store.js";
|
|
38
38
|
import type { BackupTarget } from "./targets.js";
|
|
39
39
|
import type { Manifest, RemoteState, SnapshotPart } from "./types.js";
|
|
40
40
|
|
|
@@ -182,14 +182,19 @@ async function uploadToAll(
|
|
|
182
182
|
}
|
|
183
183
|
|
|
184
184
|
/**
|
|
185
|
-
* Apply the remote retention policy on each target
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
*
|
|
185
|
+
* Apply the remote retention policy on each target — the same tiers as
|
|
186
|
+
* local (see retention/policy.ts), with `keepRemote` as the newest-N tier.
|
|
187
|
+
*
|
|
188
|
+
* Two refusals keep a retention pass from turning into data loss:
|
|
189
|
+
* - A target that cannot list is skipped: deleting on a partial
|
|
190
|
+
* listing is how that happens.
|
|
191
|
+
* - An entry whose manifest is missing, unreadable, or has no
|
|
192
|
+
* createdAt is never pruned. Without an age it cannot be ranked,
|
|
193
|
+
* and treating it as the oldest would delete it first.
|
|
189
194
|
*/
|
|
190
195
|
export async function pruneRemote(
|
|
191
196
|
targets: readonly BackupTarget[],
|
|
192
|
-
|
|
197
|
+
policy: RetentionPolicy,
|
|
193
198
|
): Promise<void> {
|
|
194
199
|
for (const target of targets) {
|
|
195
200
|
if (!target.ready) continue;
|
|
@@ -203,15 +208,27 @@ export async function pruneRemote(
|
|
|
203
208
|
);
|
|
204
209
|
continue;
|
|
205
210
|
}
|
|
206
|
-
const
|
|
207
|
-
snapshots.map((entry) =>
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
211
|
+
const plan = planRetention(
|
|
212
|
+
snapshots.map((entry) => {
|
|
213
|
+
const manifest = entry.manifest as Partial<Manifest> | undefined;
|
|
214
|
+
return {
|
|
215
|
+
id: entry.snapshotId,
|
|
216
|
+
createdAt: manifest?.createdAt as number,
|
|
217
|
+
pinned: manifest?.pinned === true,
|
|
218
|
+
kind: manifest?.kind,
|
|
219
|
+
verifiedAt: manifest?.verifiedAt,
|
|
220
|
+
};
|
|
221
|
+
}),
|
|
222
|
+
policy,
|
|
213
223
|
);
|
|
214
|
-
|
|
224
|
+
if (plan.skipped.length > 0) {
|
|
225
|
+
logWarn(
|
|
226
|
+
"backup",
|
|
227
|
+
`Not pruning ${plan.skipped.length} snapshot(s) on ${target.id} whose manifest ` +
|
|
228
|
+
`is unreadable or has no createdAt: ${plan.skipped.map((s) => s.id).join(", ")}`,
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
for (const victim of plan.prune) {
|
|
215
232
|
try {
|
|
216
233
|
await target.remove(victim.id);
|
|
217
234
|
deleteBackupRemote(victim.id, target.id);
|
|
@@ -222,10 +239,10 @@ export async function pruneRemote(
|
|
|
222
239
|
);
|
|
223
240
|
}
|
|
224
241
|
}
|
|
225
|
-
if (
|
|
242
|
+
if (plan.prune.length > 0) {
|
|
226
243
|
log(
|
|
227
244
|
"backup",
|
|
228
|
-
`Pruned ${
|
|
245
|
+
`Pruned ${plan.prune.length} snapshot(s) from ${target.id} (${describeRetention(policy)})`,
|
|
229
246
|
);
|
|
230
247
|
}
|
|
231
248
|
}
|
package/src/core/config/index.ts
CHANGED
|
@@ -679,7 +679,8 @@ const configSchema = z.object({
|
|
|
679
679
|
* Backups & checkpoints (docs/backups.md). Talon's only safety net, so
|
|
680
680
|
* it is on by default: every `intervalHours` it writes a snapshot of
|
|
681
681
|
* the identity, state, database and memory under ~/.talon/backups/,
|
|
682
|
-
* keeps
|
|
682
|
+
* keeps them on a tiered schedule (newest `keepLocal`, one a day for
|
|
683
|
+
* `keepDaily` days, one a week for `keepWeekly` weeks), and uploads to whatever remote targets
|
|
683
684
|
* are registered (`targets: []` keeps everything local).
|
|
684
685
|
*
|
|
685
686
|
* - `workspaceInclude` — the workspace is mostly bulk that can be
|
|
@@ -716,6 +717,30 @@ const configSchema = z.object({
|
|
|
716
717
|
.min(1)
|
|
717
718
|
.max(1000)
|
|
718
719
|
.default(DEFAULT_BACKUP_SETTINGS.keepRemote),
|
|
720
|
+
/**
|
|
721
|
+
* Tiered retention on top of the newest `keepLocal`/`keepRemote`
|
|
722
|
+
* (docs/backups.md#retention): the newest snapshot of each of the
|
|
723
|
+
* last `keepDaily` days and `keepWeekly` weeks, plus up to
|
|
724
|
+
* `keepCheckpoints` unpinned checkpoints counted on their own.
|
|
725
|
+
*/
|
|
726
|
+
keepDaily: z
|
|
727
|
+
.number()
|
|
728
|
+
.int()
|
|
729
|
+
.min(0)
|
|
730
|
+
.max(1000)
|
|
731
|
+
.default(DEFAULT_BACKUP_SETTINGS.keepDaily),
|
|
732
|
+
keepWeekly: z
|
|
733
|
+
.number()
|
|
734
|
+
.int()
|
|
735
|
+
.min(0)
|
|
736
|
+
.max(1000)
|
|
737
|
+
.default(DEFAULT_BACKUP_SETTINGS.keepWeekly),
|
|
738
|
+
keepCheckpoints: z
|
|
739
|
+
.number()
|
|
740
|
+
.int()
|
|
741
|
+
.min(1)
|
|
742
|
+
.max(1000)
|
|
743
|
+
.default(DEFAULT_BACKUP_SETTINGS.keepCheckpoints),
|
|
719
744
|
includePalace: z.boolean().default(DEFAULT_BACKUP_SETTINGS.includePalace),
|
|
720
745
|
/**
|
|
721
746
|
* WhatsApp auth + the userbot's Telegram login. "local" (default)
|
package/src/core/errors.ts
CHANGED
|
@@ -139,6 +139,17 @@ function errorCodes(err: unknown, depth = 0): string[] {
|
|
|
139
139
|
|
|
140
140
|
// ── Classify any error ──────────────────────────────────────────────────────
|
|
141
141
|
|
|
142
|
+
/**
|
|
143
|
+
* A backend session that expired or no longer exists. Claude's CLI answers
|
|
144
|
+
* a resume of a session whose transcript is missing with "No conversation
|
|
145
|
+
* found with session ID: …" — a definite not-found; without the match
|
|
146
|
+
* every later turn in the chat fails the same way. The reset that follows
|
|
147
|
+
* archives the old id (resetSession), so a transcript that turns up again
|
|
148
|
+
* can be re-attached.
|
|
149
|
+
*/
|
|
150
|
+
const SESSION_EXPIRED_RE =
|
|
151
|
+
/session.*expired|expired.*session|invalid.*resume|no conversation found/i;
|
|
152
|
+
|
|
142
153
|
/**
|
|
143
154
|
* Wrap or classify any thrown value into a TalonError.
|
|
144
155
|
* Call this at module boundaries (backend catch, bridge catch) to convert
|
|
@@ -259,8 +270,8 @@ export function classify(err: unknown): TalonError {
|
|
|
259
270
|
});
|
|
260
271
|
}
|
|
261
272
|
|
|
262
|
-
// Session expired
|
|
263
|
-
if (
|
|
273
|
+
// Session expired (or gone — see SESSION_EXPIRED_RE)
|
|
274
|
+
if (SESSION_EXPIRED_RE.test(msg)) {
|
|
264
275
|
return new TalonError(msg, {
|
|
265
276
|
reason: "session_expired",
|
|
266
277
|
retryable: false,
|
|
@@ -180,7 +180,13 @@ function storageLines(f: ReportFormatter, status: BackupStatus): string[] {
|
|
|
180
180
|
if (policy) {
|
|
181
181
|
lines.push(
|
|
182
182
|
`${f.bold("Retention:")} newest ${policy.keepLocal} kept here, ` +
|
|
183
|
-
`${policy.keepRemote} per target
|
|
183
|
+
`${policy.keepRemote} per target` +
|
|
184
|
+
(policy.keepDaily ? ` · 1/day for ${policy.keepDaily}d` : "") +
|
|
185
|
+
(policy.keepWeekly ? ` · 1/week for ${policy.keepWeekly}w` : "") +
|
|
186
|
+
(policy.keepCheckpoints
|
|
187
|
+
? ` · ${policy.keepCheckpoints} checkpoints`
|
|
188
|
+
: "") +
|
|
189
|
+
` · pinned ones and the last verified are never pruned`,
|
|
184
190
|
);
|
|
185
191
|
lines.push(
|
|
186
192
|
policy.encrypted
|