sealkeep 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +20 -0
- package/CHANGELOG.md +146 -0
- package/README.md +117 -3
- package/dist/site/index.html +23 -1
- package/dist/site.zip +0 -0
- package/dist/src/adapters.js +12 -5
- package/dist/src/agent-context.js +169 -18
- package/dist/src/agent-memory.d.ts +52 -0
- package/dist/src/agent-memory.js +128 -0
- package/dist/src/cli.js +64 -4
- package/dist/src/daemon.d.ts +3 -0
- package/dist/src/daemon.js +52 -7
- package/dist/src/doctor.js +23 -0
- package/dist/src/fork.d.ts +78 -0
- package/dist/src/fork.js +197 -0
- package/dist/src/heartbeat.js +19 -1
- package/dist/src/local-api.js +10 -0
- package/dist/src/machine-settings.d.ts +30 -1
- package/dist/src/machine-settings.js +40 -0
- package/dist/src/mcp.js +9 -0
- package/dist/src/meter.d.ts +75 -0
- package/dist/src/meter.js +241 -0
- package/dist/src/search.d.ts +2 -1
- package/dist/src/search.js +60 -21
- package/dist/src/team-presence.js +15 -3
- package/dist/src/vault.d.ts +14 -0
- package/dist/src/vault.js +16 -0
- package/package.json +2 -2
- package/web/app.js +21 -0
- package/web/index.html +21 -0
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The context meter — what this session is costing you, per step.
|
|
3
|
+
*
|
|
4
|
+
* An AI coding session charges you for its own history on every request. A
|
|
5
|
+
* conversation that has been alive for months carries its whole transcript
|
|
6
|
+
* into each step, so a one-line question costs whatever the transcript costs,
|
|
7
|
+
* and an agent loop pays that once per command. Nothing tells you until the
|
|
8
|
+
* limit is gone.
|
|
9
|
+
*
|
|
10
|
+
* Sealkeep already holds the session files, so it can answer the question
|
|
11
|
+
* nobody else can: what is each live session costing per step, how much of
|
|
12
|
+
* that is re-read history, and what would the same work cost in a fresh
|
|
13
|
+
* session carrying a small context pack instead of a transcript.
|
|
14
|
+
*
|
|
15
|
+
* Everything here reads local files only. No key, no network, nothing leaves
|
|
16
|
+
* the machine to produce a number.
|
|
17
|
+
*/
|
|
18
|
+
import { createReadStream } from "node:fs";
|
|
19
|
+
import { readdir, stat } from "node:fs/promises";
|
|
20
|
+
import { homedir } from "node:os";
|
|
21
|
+
import { basename, dirname, join } from "node:path";
|
|
22
|
+
import { createInterface } from "node:readline";
|
|
23
|
+
/** A fork carries rules, a statement of what is true now, and the last handoff. */
|
|
24
|
+
export const CONTEXT_PACK_TOKENS = 3_000;
|
|
25
|
+
/**
|
|
26
|
+
* Which sessions are worth saying something about. Thresholds are deliberately
|
|
27
|
+
* boring: each one was true of a real session that ended a weekly allowance.
|
|
28
|
+
*/
|
|
29
|
+
export function signalsFor(input) {
|
|
30
|
+
const signals = [];
|
|
31
|
+
if (input.historyShare > 0.9 && input.perStep > 50_000)
|
|
32
|
+
signals.push("BLOATED");
|
|
33
|
+
if (input.ageDays > 7 || input.sizeMB > 200)
|
|
34
|
+
signals.push("STALE");
|
|
35
|
+
if (input.sleeps >= 5)
|
|
36
|
+
signals.push("POLLING");
|
|
37
|
+
return signals;
|
|
38
|
+
}
|
|
39
|
+
/** Turns raw counters into the numbers a person can act on. */
|
|
40
|
+
export function scoreSession(raw) {
|
|
41
|
+
const steps = Math.max(raw.steps, 1);
|
|
42
|
+
const perStep = Math.round(raw.input / steps);
|
|
43
|
+
const historyShare = raw.input > 0 ? raw.cached / raw.input : 0;
|
|
44
|
+
// A fresh session still pays for the new work; what it stops paying for is
|
|
45
|
+
// the transcript. The pack is the small thing carried in its place.
|
|
46
|
+
const freshCost = steps * (CONTEXT_PACK_TOKENS + Math.round(raw.fresh / steps));
|
|
47
|
+
const scored = { ...raw, perStep, historyShare, freshCost, saving: Math.max(0, raw.input - freshCost), signals: [] };
|
|
48
|
+
scored.signals = signalsFor(scored);
|
|
49
|
+
return scored;
|
|
50
|
+
}
|
|
51
|
+
async function* jsonlFiles(root) {
|
|
52
|
+
let entries;
|
|
53
|
+
try {
|
|
54
|
+
entries = await readdir(root, { withFileTypes: true });
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
for (const entry of entries) {
|
|
60
|
+
const path = join(root, entry.name);
|
|
61
|
+
if (entry.isDirectory())
|
|
62
|
+
yield* jsonlFiles(path);
|
|
63
|
+
else if (entry.name.endsWith(".jsonl"))
|
|
64
|
+
yield path;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async function* lines(file) {
|
|
68
|
+
// Streamed, because a session worth metering is often gigabytes.
|
|
69
|
+
const stream = createReadStream(file, { encoding: "utf8" });
|
|
70
|
+
try {
|
|
71
|
+
for await (const line of createInterface({ input: stream, crlfDelay: Infinity }))
|
|
72
|
+
yield line;
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
stream.destroy();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Claude Code transcripts report usage per message, so a window is a sum. */
|
|
79
|
+
async function claudeSession(file, since) {
|
|
80
|
+
const info = await stat(file).catch(() => null);
|
|
81
|
+
if (!info || info.mtimeMs < since)
|
|
82
|
+
return null;
|
|
83
|
+
let steps = 0, cached = 0, fresh = 0, cacheWrite = 0, output = 0;
|
|
84
|
+
for await (const line of lines(file)) {
|
|
85
|
+
if (!line.includes('"usage"'))
|
|
86
|
+
continue;
|
|
87
|
+
let row;
|
|
88
|
+
try {
|
|
89
|
+
row = JSON.parse(line);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
if (!(Date.parse(row.timestamp ?? "") >= since))
|
|
95
|
+
continue;
|
|
96
|
+
const usage = row.message?.usage ?? {};
|
|
97
|
+
steps += 1;
|
|
98
|
+
cached += usage.cache_read_input_tokens ?? 0;
|
|
99
|
+
fresh += usage.input_tokens ?? 0;
|
|
100
|
+
cacheWrite += usage.cache_creation_input_tokens ?? 0;
|
|
101
|
+
output += usage.output_tokens ?? 0;
|
|
102
|
+
}
|
|
103
|
+
if (!steps)
|
|
104
|
+
return null;
|
|
105
|
+
return scoreSession({
|
|
106
|
+
tool: "claude", file,
|
|
107
|
+
project: basename(dirname(file)).replace(/^-/, "").replace(/-/g, "/"),
|
|
108
|
+
sizeMB: Math.round(info.size / 1e6),
|
|
109
|
+
ageDays: (Date.now() - info.birthtimeMs) / 864e5,
|
|
110
|
+
steps, tools: 0, sleeps: 0,
|
|
111
|
+
input: cached + fresh + cacheWrite, cached, fresh: fresh + cacheWrite, output,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
/** Codex rollouts report cumulative counters, so a window is a delta. */
|
|
115
|
+
async function codexSession(file, since) {
|
|
116
|
+
const info = await stat(file).catch(() => null);
|
|
117
|
+
if (!info || info.mtimeMs < since)
|
|
118
|
+
return null;
|
|
119
|
+
let cwd = "", before = null, last = null, steps = 0, tools = 0, sleeps = 0, seen = 0;
|
|
120
|
+
for await (const line of lines(file)) {
|
|
121
|
+
if (!line)
|
|
122
|
+
continue;
|
|
123
|
+
let row;
|
|
124
|
+
try {
|
|
125
|
+
row = JSON.parse(line);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
const payload = row.payload ?? {};
|
|
131
|
+
if (seen < 5) {
|
|
132
|
+
seen += 1;
|
|
133
|
+
const meta = payload.type === "session_meta" ? (payload.payload ?? payload) : (row.type === "session_meta" ? payload : null);
|
|
134
|
+
if (meta?.cwd)
|
|
135
|
+
cwd = meta.cwd;
|
|
136
|
+
}
|
|
137
|
+
const at = Date.parse(row.timestamp ?? "");
|
|
138
|
+
if (payload.type === "token_count") {
|
|
139
|
+
const usage = payload.info?.total_token_usage;
|
|
140
|
+
if (!usage)
|
|
141
|
+
continue;
|
|
142
|
+
if (!(at >= since)) {
|
|
143
|
+
before = usage;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
before ??= usage;
|
|
147
|
+
last = usage;
|
|
148
|
+
steps += 1;
|
|
149
|
+
}
|
|
150
|
+
else if (at >= since && (payload.type === "custom_tool_call" || payload.type === "function_call")) {
|
|
151
|
+
tools += 1;
|
|
152
|
+
if (/\bsleep\b|"name"\s*:\s*"sleep"/.test(line))
|
|
153
|
+
sleeps += 1;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (!last || !before || !steps)
|
|
157
|
+
return null;
|
|
158
|
+
const input = last.input_tokens - before.input_tokens;
|
|
159
|
+
const cached = last.cached_input_tokens - before.cached_input_tokens;
|
|
160
|
+
return scoreSession({
|
|
161
|
+
tool: "codex", file, project: cwd || "?",
|
|
162
|
+
sizeMB: Math.round(info.size / 1e6),
|
|
163
|
+
ageDays: (Date.now() - info.birthtimeMs) / 864e5,
|
|
164
|
+
steps, tools, sleeps,
|
|
165
|
+
input, cached, fresh: input - cached, output: last.output_tokens - before.output_tokens,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
/** Reads local session files and scores every session touched in the window. */
|
|
169
|
+
export async function meterSessions(options = {}) {
|
|
170
|
+
const hours = options.hours ?? 24;
|
|
171
|
+
const since = (options.now ?? Date.now()) - hours * 36e5;
|
|
172
|
+
const claudeRoot = options.roots?.claude ?? join(homedir(), ".claude", "projects");
|
|
173
|
+
const codexRoot = options.roots?.codex ?? join(homedir(), ".codex", "sessions");
|
|
174
|
+
const sessions = [];
|
|
175
|
+
for await (const file of jsonlFiles(claudeRoot)) {
|
|
176
|
+
const scored = await claudeSession(file, since).catch(() => null);
|
|
177
|
+
if (scored)
|
|
178
|
+
sessions.push(scored);
|
|
179
|
+
}
|
|
180
|
+
for await (const file of jsonlFiles(codexRoot)) {
|
|
181
|
+
const scored = await codexSession(file, since).catch(() => null);
|
|
182
|
+
if (scored)
|
|
183
|
+
sessions.push(scored);
|
|
184
|
+
}
|
|
185
|
+
sessions.sort((a, b) => b.input - a.input);
|
|
186
|
+
const totalInput = sessions.reduce((n, s) => n + s.input, 0);
|
|
187
|
+
const totalSaving = sessions.reduce((n, s) => n + s.saving, 0);
|
|
188
|
+
return {
|
|
189
|
+
hours, sessions, totalInput,
|
|
190
|
+
totalFreshCost: totalInput - totalSaving,
|
|
191
|
+
totalSaving,
|
|
192
|
+
savingShare: totalInput ? totalSaving / totalInput : 0,
|
|
193
|
+
worst: sessions.find((s) => s.signals.includes("BLOATED")) ?? sessions[0],
|
|
194
|
+
polling: sessions.filter((s) => s.signals.includes("POLLING")),
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
const tokens = (n) => n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : n >= 1e3 ? `${Math.round(n / 1e3)}k` : String(Math.round(n));
|
|
198
|
+
const percent = (share) => `${Math.round(share * 100)}%`;
|
|
199
|
+
/** The receipt for one session, in the words a person would use. */
|
|
200
|
+
export function renderReceipt(session) {
|
|
201
|
+
const lines = [
|
|
202
|
+
`${session.tool} session in ${basename(session.project) || session.project}`,
|
|
203
|
+
` ${tokens(session.perStep)} tokens per step, ${percent(session.historyShare)} of it re-read history.`,
|
|
204
|
+
` ${session.steps} steps cost ${tokens(session.input)}. The same work in a fresh session: about ${tokens(session.freshCost)}.`,
|
|
205
|
+
];
|
|
206
|
+
if (session.signals.includes("STALE")) {
|
|
207
|
+
lines.push(` ${session.sizeMB} MB on disk, ${session.ageDays.toFixed(0)} days old.`);
|
|
208
|
+
}
|
|
209
|
+
if (session.saving > 0)
|
|
210
|
+
lines.push(` You are paying about ${tokens(session.saving)} for history you did not ask for.`);
|
|
211
|
+
return lines.join("\n");
|
|
212
|
+
}
|
|
213
|
+
/** The whole picture, for `sealkeep meter`. */
|
|
214
|
+
export function renderMeter(summary) {
|
|
215
|
+
if (!summary.sessions.length) {
|
|
216
|
+
return `No agent sessions were touched in the last ${summary.hours}h, so there is nothing to meter.`;
|
|
217
|
+
}
|
|
218
|
+
const out = [
|
|
219
|
+
`Context cost, last ${summary.hours}h, across ${summary.sessions.length} session${summary.sessions.length === 1 ? "" : "s"}.`,
|
|
220
|
+
"Read from local session files. Nothing left this machine.",
|
|
221
|
+
"",
|
|
222
|
+
`${"tool".padEnd(7)}${"where".padEnd(22)}${"steps".padStart(6)}${"per step".padStart(10)}${"history".padStart(9)}${"total in".padStart(10)} signals`,
|
|
223
|
+
];
|
|
224
|
+
for (const s of summary.sessions.slice(0, 12)) {
|
|
225
|
+
out.push(`${s.tool.padEnd(7)}${(basename(s.project) || "?").slice(0, 21).padEnd(22)}${String(s.steps).padStart(6)}`
|
|
226
|
+
+ `${tokens(s.perStep).padStart(10)}${percent(s.historyShare).padStart(9)}${tokens(s.input).padStart(10)} ${s.signals.join(" ")}`);
|
|
227
|
+
}
|
|
228
|
+
out.push("", `Total sent: ${tokens(summary.totalInput)}. The same work from fresh sessions carrying a ${tokens(CONTEXT_PACK_TOKENS)}-token pack: about ${tokens(summary.totalFreshCost)}.`);
|
|
229
|
+
if (summary.totalSaving > 0) {
|
|
230
|
+
out.push(`That is roughly ${tokens(summary.totalSaving)} spent re-reading, ${percent(summary.savingShare)} of everything you sent.`);
|
|
231
|
+
}
|
|
232
|
+
if (summary.worst?.signals.includes("BLOATED")) {
|
|
233
|
+
out.push("", "Worst offender:", renderReceipt(summary.worst));
|
|
234
|
+
out.push(" Fork it: write what is true now into a short state note, start fresh, and search the rest on demand.");
|
|
235
|
+
}
|
|
236
|
+
if (summary.polling.length) {
|
|
237
|
+
out.push("", "Polling: an agent is sleeping and re-checking inside its own turn, paying the full context on every tick.");
|
|
238
|
+
out.push(" Move the wait into a shell command that returns once.");
|
|
239
|
+
}
|
|
240
|
+
return out.join("\n");
|
|
241
|
+
}
|
package/dist/src/search.d.ts
CHANGED
|
@@ -246,7 +246,8 @@ export declare function searchMetadata(dataDir: string, query: string, options?:
|
|
|
246
246
|
*
|
|
247
247
|
* This decrypts every archive locally, so it requires the recovery phrase. The
|
|
248
248
|
* resulting index is itself encrypted with the same envelope before it touches the
|
|
249
|
-
* disk, and
|
|
249
|
+
* disk, and only that ciphertext ever leaves the machine — synced to the account
|
|
250
|
+
* so your other machines can search it (`pushed` in the result says whether it was).
|
|
250
251
|
*/
|
|
251
252
|
export declare function buildContentIndex(dataDir: string, rawPhrase: string, options?: {
|
|
252
253
|
sync?: boolean;
|
package/dist/src/search.js
CHANGED
|
@@ -8,7 +8,7 @@ import { createGunzip, gunzipSync } from "node:zlib";
|
|
|
8
8
|
import { decryptArchive as openEnvelope, decryptArchiveBodyChunks, encryptArchive as sealEnvelope, openArchiveToFile, sealArchiveToFile, unsqueezeStream, unsqueezeSync, validateEnvelope, rawPublicKey, x25519PrivateKeyFromRaw, x25519PublicKeyFromRaw } from "../packages/sealkeep-crypto/src/index.js";
|
|
9
9
|
import { fail, isSealkeepError } from "./errors.js";
|
|
10
10
|
import { canonicalPhrase } from "./mnemonic.js";
|
|
11
|
-
import { configuredRecipients, decryptRecord, deltaOf, listArchives, readConfig, resolveDeltaChain, restoreRecordToFile, teamSpaceOf } from "./vault.js";
|
|
11
|
+
import { configuredRecipients, decryptRecord, deltaOf, listArchives, listArchivesWithIntegrity, readConfig, resolveDeltaChain, restoreRecordToFile, teamSpaceOf } from "./vault.js";
|
|
12
12
|
import { chunkWindowForRange, openChunkWindow, supportsChunkAccess } from "../packages/sealkeep-crypto/src/chunk-access.js";
|
|
13
13
|
import { isV2, teamCustodyRecordIsValid } from "./types.js";
|
|
14
14
|
import { archiveWrapsEveryMember } from "./upload.js";
|
|
@@ -397,12 +397,27 @@ export async function contentIndexNeedsPolicyUpgrade(dataDir) {
|
|
|
397
397
|
if (!Array.isArray(raw.indexed) || raw.extractionPolicy !== INDEX_EXTRACTION_POLICY)
|
|
398
398
|
return true;
|
|
399
399
|
const indexed = new Set(raw.indexed.filter((id) => typeof id === "string"));
|
|
400
|
-
return (await listArchives(dataDir)).
|
|
400
|
+
return coverageGaps(await listArchives(dataDir), indexed).length > 0;
|
|
401
401
|
}
|
|
402
402
|
catch {
|
|
403
403
|
return stat(indexPath(dataDir)).then(() => true, () => false);
|
|
404
404
|
}
|
|
405
405
|
}
|
|
406
|
+
/**
|
|
407
|
+
* The archives the index still owes: sealed here, not superseded, not covered.
|
|
408
|
+
*
|
|
409
|
+
* One definition, because the daemon's "is the index finished" check kept its
|
|
410
|
+
* own and counted superseded snapshots too. Those are never indexed on their
|
|
411
|
+
* own — the snapshot that superseded one carries every byte of it, and the
|
|
412
|
+
* index retires its id — so they can never appear in coverage, and the check
|
|
413
|
+
* could never pass on a vault that had ever superseded one: the daemon retried
|
|
414
|
+
* `index_incomplete` forever, and disk reclaim waits for exactly that check.
|
|
415
|
+
* `sealkeep index status` and doctor read the same list.
|
|
416
|
+
*/
|
|
417
|
+
function coverageGaps(records, indexed) {
|
|
418
|
+
const superseded = supersededArchiveIds(records);
|
|
419
|
+
return records.filter((record) => !indexed.has(record.id) && !superseded.has(record.id));
|
|
420
|
+
}
|
|
406
421
|
const MIN_TOKEN = 3;
|
|
407
422
|
const MAX_TOKENS_PER_ARCHIVE = 4000;
|
|
408
423
|
/**
|
|
@@ -442,6 +457,12 @@ export function isRecalledMemoryLine(line) {
|
|
|
442
457
|
return true;
|
|
443
458
|
if (line.includes("Sealkeep automatically loaded preserved context for project"))
|
|
444
459
|
return true;
|
|
460
|
+
if (line.includes("preserved by Sealkeep from previous sessions"))
|
|
461
|
+
return true;
|
|
462
|
+
// Pull mode says only that recall exists; it carries no history, but it is
|
|
463
|
+
// still Sealkeep's own words rather than anything the session produced.
|
|
464
|
+
if (line.includes("Nothing has been loaded into this conversation."))
|
|
465
|
+
return true;
|
|
445
466
|
try {
|
|
446
467
|
const row = JSON.parse(line);
|
|
447
468
|
const payload = row.payload && typeof row.payload === "object" && !Array.isArray(row.payload)
|
|
@@ -785,7 +806,8 @@ export async function searchMetadata(dataDir, query, options = {}) {
|
|
|
785
806
|
*
|
|
786
807
|
* This decrypts every archive locally, so it requires the recovery phrase. The
|
|
787
808
|
* resulting index is itself encrypted with the same envelope before it touches the
|
|
788
|
-
* disk, and
|
|
809
|
+
* disk, and only that ciphertext ever leaves the machine — synced to the account
|
|
810
|
+
* so your other machines can search it (`pushed` in the result says whether it was).
|
|
789
811
|
*/
|
|
790
812
|
export async function buildContentIndex(dataDir, rawPhrase, options = {}) {
|
|
791
813
|
const phrase = canonicalPhrase(rawPhrase);
|
|
@@ -827,7 +849,9 @@ export async function buildContentIndex(dataDir, rawPhrase, options = {}) {
|
|
|
827
849
|
await readConfig(dataDir);
|
|
828
850
|
await cleanupStaleContentIndexTemps(dataDir);
|
|
829
851
|
await adoptLegacyIndexFile(dataDir);
|
|
830
|
-
|
|
852
|
+
// Absence is only evidence of deletion when every sidecar could be read.
|
|
853
|
+
const { records, unreadable } = await listArchivesWithIntegrity(dataDir);
|
|
854
|
+
const absenceMeansGone = unreadable.length === 0;
|
|
831
855
|
// Segments mode is decided once, up front: a vault either has a manifest
|
|
832
856
|
// with at least one segment (sealkeep index migrate created it) or it
|
|
833
857
|
// stays on the legacy blob path below, unchanged.
|
|
@@ -948,12 +972,19 @@ export async function buildContentIndex(dataDir, rawPhrase, options = {}) {
|
|
|
948
972
|
// file's index entry, the newer snapshot named by its `supersedes` field
|
|
949
973
|
// is — and every superseded id is named here even one this machine never
|
|
950
974
|
// got around to covering, so the manifest need not remember it later.
|
|
951
|
-
|
|
975
|
+
// Retiring an id from a segment is permanent: it disappears from every
|
|
976
|
+
// later lookup and compaction, and no build ever indexes it again. So it
|
|
977
|
+
// may only follow proof. A superseded snapshot is proof. Absence is not,
|
|
978
|
+
// unless the scan read every sidecar — see listArchivesWithIntegrity.
|
|
979
|
+
const stale = [...new Set([...mine, ...superseded])]
|
|
980
|
+
.filter((id) => superseded.has(id) || (absenceMeansGone && !liveIds.has(id)));
|
|
952
981
|
if (stale.length)
|
|
953
982
|
await retireArchiveIds(dataDir, stale);
|
|
954
983
|
}
|
|
955
984
|
else {
|
|
956
985
|
for (const id of Object.keys(index.archives)) {
|
|
986
|
+
if (!absenceMeansGone)
|
|
987
|
+
break; // a short scan is not a deletion
|
|
957
988
|
if (liveIds.has(id))
|
|
958
989
|
continue;
|
|
959
990
|
if (!mine.has(id))
|
|
@@ -991,10 +1022,6 @@ export async function buildContentIndex(dataDir, rawPhrase, options = {}) {
|
|
|
991
1022
|
}
|
|
992
1023
|
}
|
|
993
1024
|
const recordById = new Map(records.map((record) => [record.id, record]));
|
|
994
|
-
const deltaBases = new Set(records.flatMap((record) => {
|
|
995
|
-
const delta = deltaOf(record);
|
|
996
|
-
return delta ? [delta.baseArchiveId] : [];
|
|
997
|
-
}));
|
|
998
1025
|
// Once a verified child is present, its base is a dependency rather than a
|
|
999
1026
|
// separate stale search result. Purge those old logical postings in one
|
|
1000
1027
|
// token-map pass. Physical archive/catalog entries stay covered so a later
|
|
@@ -1061,8 +1088,9 @@ export async function buildContentIndex(dataDir, rawPhrase, options = {}) {
|
|
|
1061
1088
|
// start with them rather than making today's session wait behind months of
|
|
1062
1089
|
// history. Posting order no longer depends on completion order; the
|
|
1063
1090
|
// serializer sorts by sealed session metadata before applying its cap.
|
|
1064
|
-
// A linear delta chain is one logical session. Only
|
|
1065
|
-
// scheduled
|
|
1091
|
+
// A linear delta chain is one logical session. Only the newest link that
|
|
1092
|
+
// still needs indexing is scheduled — normally the leaf (see `awaitedBases`
|
|
1093
|
+
// below); indexOneRecord authenticates the chain once and records its
|
|
1066
1094
|
// ancestors as covered dependencies. This turns N watcher snapshots from
|
|
1067
1095
|
// 1+2+...+N physical reads into N reads.
|
|
1068
1096
|
// A whale goes after every ordinary session, newest-first within each
|
|
@@ -1077,8 +1105,23 @@ export async function buildContentIndex(dataDir, rawPhrase, options = {}) {
|
|
|
1077
1105
|
// coverage.json (this machine's own view of the manifest, already read
|
|
1078
1106
|
// above as `mine`) and the manifest's tombstones answer the same question.
|
|
1079
1107
|
const segmentsTombstones = segmentsMode ? new Set((await readManifest(dataDir)).tombstones) : null;
|
|
1080
|
-
const
|
|
1081
|
-
&& (segmentsMode ? !mine.has(record.id) && !segmentsTombstones.has(record.id) : !index.archives[record.id])
|
|
1108
|
+
const uncovered = (record) => !superseded.has(record.id)
|
|
1109
|
+
&& (segmentsMode ? !mine.has(record.id) && !segmentsTombstones.has(record.id) : !index.archives[record.id]);
|
|
1110
|
+
// A delta base is skipped only while one of its own children is still
|
|
1111
|
+
// waiting: the newest waiting link is scheduled instead, and walking its
|
|
1112
|
+
// chain to the root covers this base on the way. Skipping every base
|
|
1113
|
+
// unconditionally assumed the leaf would always be walked — but a leaf
|
|
1114
|
+
// indexed at seal time is never walked, and covers only the bytes it
|
|
1115
|
+
// appended. Every link sealed before it, the whole beginning of the
|
|
1116
|
+
// session, then sat outside every build forever while each run reported
|
|
1117
|
+
// success. Measured on a real vault: 400 such links across 76 sessions;
|
|
1118
|
+
// words found only in those early ranges came back in 0 of 20 searches,
|
|
1119
|
+
// words from the covered tail of the same sessions in 10.
|
|
1120
|
+
const awaitedBases = new Set(records.flatMap((record) => {
|
|
1121
|
+
const delta = deltaOf(record);
|
|
1122
|
+
return delta && uncovered(record) ? [delta.baseArchiveId] : [];
|
|
1123
|
+
}));
|
|
1124
|
+
const fresh = records.filter((record) => uncovered(record) && !awaitedBases.has(record.id))
|
|
1082
1125
|
.sort((a, b) => Number(whale(a)) - Number(whale(b)) || b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id));
|
|
1083
1126
|
let indexedNow = 0;
|
|
1084
1127
|
let skipped = 0;
|
|
@@ -6329,14 +6372,10 @@ export async function indexCoverage(dataDir, options = {}) {
|
|
|
6329
6372
|
.then((raw) => JSON.parse(raw))
|
|
6330
6373
|
.catch(() => null);
|
|
6331
6374
|
const indexedIds = new Set(coverage?.indexed ?? []);
|
|
6332
|
-
// A superseded snapshot is intentionally never indexed on its own
|
|
6333
|
-
//
|
|
6334
|
-
//
|
|
6335
|
-
|
|
6336
|
-
// vault whose only unindexed records are superseded snapshots.
|
|
6337
|
-
const superseded = supersededArchiveIds(records);
|
|
6338
|
-
const missing = records
|
|
6339
|
-
.filter((record) => !indexedIds.has(record.id) && !superseded.has(record.id))
|
|
6375
|
+
// A superseded snapshot is intentionally never indexed on its own, so it is
|
|
6376
|
+
// not a gap (see coverageGaps). Doctor's `search-index` check reads `missing`
|
|
6377
|
+
// and must go green on a vault whose only unindexed records are superseded.
|
|
6378
|
+
const missing = coverageGaps(records, indexedIds)
|
|
6340
6379
|
.map((record) => ({ id: record.id, path: record.source.path, agent: record.source.agent }));
|
|
6341
6380
|
const indexBytes = await import("node:fs/promises").then(({ stat }) => stat(indexPath(dataDir))).then((info) => info.size).catch(() => 0);
|
|
6342
6381
|
let searchable = null;
|
|
@@ -454,7 +454,13 @@ export async function reconcileCloudTeamAccess(dataDir, phrase, preferred) {
|
|
|
454
454
|
// seals it, binds the local project, then acknowledges the cloud request.
|
|
455
455
|
// If the response from createTeamSpace was lost, the next list observes the
|
|
456
456
|
// already-created owner space and completes the same request idempotently.
|
|
457
|
+
//
|
|
458
|
+
// The space list is read once and re-read only after this pass itself
|
|
459
|
+
// created, bound or re-listed a space. It used to be fetched three times
|
|
460
|
+
// per pass unconditionally; on an idle vault that was half of every pass's
|
|
461
|
+
// requests, and the three answers were always the same.
|
|
457
462
|
let spaces = await listTeamSpaces(dataDir, preferred);
|
|
463
|
+
let spacesStale = false;
|
|
458
464
|
const projectRequests = await listTeamProjectRequests(dataDir, preferred);
|
|
459
465
|
for (const request of projectRequests) {
|
|
460
466
|
if (request.state !== "requested")
|
|
@@ -464,6 +470,7 @@ export async function reconcileCloudTeamAccess(dataDir, phrase, preferred) {
|
|
|
464
470
|
continue;
|
|
465
471
|
let space = spaces.find((item) => item.role === "owner" && item.project_name?.trim() === project);
|
|
466
472
|
if (!space) {
|
|
473
|
+
spacesStale = true;
|
|
467
474
|
try {
|
|
468
475
|
const made = await createProjectTeamSpace(dataDir, project, phrase, preferred);
|
|
469
476
|
space = { space_key: made.spaceKey, role: "owner", key_version: made.keyVersion, realtime_key: made.realtimeKey, project_name: project };
|
|
@@ -486,8 +493,12 @@ export async function reconcileCloudTeamAccess(dataDir, phrase, preferred) {
|
|
|
486
493
|
failures.push({ operation: "complete_project", code: accessFailureCode(error) });
|
|
487
494
|
}
|
|
488
495
|
}
|
|
489
|
-
|
|
490
|
-
|
|
496
|
+
if (spacesStale)
|
|
497
|
+
spaces = await listTeamSpaces(dataDir, preferred);
|
|
498
|
+
const draftsBound = await reconcileTeamProjectDrafts(dataDir, phrase, spaces, preferred);
|
|
499
|
+
bound += draftsBound;
|
|
500
|
+
if (draftsBound > 0)
|
|
501
|
+
spacesStale = true;
|
|
491
502
|
const config = await readConfig(dataDir);
|
|
492
503
|
const localBySpace = new Map(Object.entries(config.teamSpaces ?? {}).map(([project, binding]) => [binding.spaceKey, { project, binding }]));
|
|
493
504
|
const locallyDisconnected = new Set(config.teamSpaceOptOuts ?? []);
|
|
@@ -532,7 +543,8 @@ export async function reconcileCloudTeamAccess(dataDir, phrase, preferred) {
|
|
|
532
543
|
// An accepted invitation is authoritative cloud membership. On every fresh
|
|
533
544
|
// machine, recover its wrapped feed key and bind the human project label
|
|
534
545
|
// without asking for an opaque space id or a local "join" command.
|
|
535
|
-
|
|
546
|
+
if (spacesStale)
|
|
547
|
+
spaces = await listTeamSpaces(dataDir, preferred);
|
|
536
548
|
for (const space of spaces) {
|
|
537
549
|
if (!SPACE_RE.test(space.space_key) || localBySpace.has(space.space_key) || locallyDisconnected.has(space.space_key))
|
|
538
550
|
continue;
|
package/dist/src/vault.d.ts
CHANGED
|
@@ -181,6 +181,20 @@ export declare function teamRecipientsForProject(dataDir: string, project: strin
|
|
|
181
181
|
* problem into a missing one. The archives that do parse still answer.
|
|
182
182
|
*/
|
|
183
183
|
export declare function listArchives(dataDir: string): Promise<ArchiveRecord[]>;
|
|
184
|
+
/**
|
|
185
|
+
* The same list, plus whether the scan could read every sidecar.
|
|
186
|
+
*
|
|
187
|
+
* A caller that concludes "this archive no longer exists" from absence needs
|
|
188
|
+
* this: one sidecar that could not be read — a partial write being replaced, a
|
|
189
|
+
* transient EIO — silently shortens the list. The index build drew exactly that
|
|
190
|
+
* conclusion and retired the archive from search permanently. Two of a real
|
|
191
|
+
* vault's sessions became unfindable that way; words in one returned no hits
|
|
192
|
+
* from it while matching hundreds of other sessions.
|
|
193
|
+
*/
|
|
194
|
+
export declare function listArchivesWithIntegrity(dataDir: string): Promise<{
|
|
195
|
+
records: ArchiveRecord[];
|
|
196
|
+
unreadable: string[];
|
|
197
|
+
}>;
|
|
184
198
|
/**
|
|
185
199
|
* Registers an X25519 public key that may open future archives. Only the public
|
|
186
200
|
* key is stored; the matching private key stays on its own device.
|
package/dist/src/vault.js
CHANGED
|
@@ -1373,6 +1373,22 @@ export async function listArchives(dataDir) {
|
|
|
1373
1373
|
// dashboard requests shared the underlying filesystem scan.
|
|
1374
1374
|
return scan.records.slice();
|
|
1375
1375
|
}
|
|
1376
|
+
/**
|
|
1377
|
+
* The same list, plus whether the scan could read every sidecar.
|
|
1378
|
+
*
|
|
1379
|
+
* A caller that concludes "this archive no longer exists" from absence needs
|
|
1380
|
+
* this: one sidecar that could not be read — a partial write being replaced, a
|
|
1381
|
+
* transient EIO — silently shortens the list. The index build drew exactly that
|
|
1382
|
+
* conclusion and retired the archive from search permanently. Two of a real
|
|
1383
|
+
* vault's sessions became unfindable that way; words in one returned no hits
|
|
1384
|
+
* from it while matching hundreds of other sessions.
|
|
1385
|
+
*/
|
|
1386
|
+
export async function listArchivesWithIntegrity(dataDir) {
|
|
1387
|
+
const config = await readConfig(dataDir);
|
|
1388
|
+
const scan = await currentArchiveLedgerScan(config.storage.root);
|
|
1389
|
+
reportUnreadableArchiveRecords(config.storage.root, scan.unreadable);
|
|
1390
|
+
return { records: scan.records.slice(), unreadable: scan.unreadable.slice() };
|
|
1391
|
+
}
|
|
1376
1392
|
// The dashboard loads several archive-derived endpoints together. A per-call
|
|
1377
1393
|
// batch bound alone still multiplied 32 opens by every endpoint. Share both an
|
|
1378
1394
|
// in-flight scan and a completed, generation-validated scan. Archive record
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sealkeep",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.4",
|
|
4
4
|
"type": "module",
|
|
5
|
-
"description": "Sealkeep by SPALA AI
|
|
5
|
+
"description": "Sealkeep by SPALA AI — your AI coding-agent history, sealed, searchable, and shared across your machines.",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "git+https://github.com/spala-ai/sealkeep.git"
|
package/web/app.js
CHANGED
|
@@ -4485,6 +4485,15 @@ function renderSettingsForm(local, _resolvedTrashStrategy) {
|
|
|
4485
4485
|
$("f-interval").value = local.schedule.intervalMinutes;
|
|
4486
4486
|
$("f-bandwidth").value = local.bandwidth?.uploadLimitMbps ?? "";
|
|
4487
4487
|
$("f-reserve").value = local.diskReserveMb ?? "";
|
|
4488
|
+
// Undefined means a settings file written before these existed; both default
|
|
4489
|
+
// on. Guarded because a shell that predates these controls must still load
|
|
4490
|
+
// the rest of the form rather than stop at the first missing element.
|
|
4491
|
+
const check = (id, on) => { const el = $(id); if (el) el.checked = on; };
|
|
4492
|
+
const fill = (id, value) => { const el = $(id); if (el) el.value = value; };
|
|
4493
|
+
check("f-auto-inject", local.recall?.autoInject !== false);
|
|
4494
|
+
check("f-mcp-enabled", local.mcp?.enabled !== false);
|
|
4495
|
+
fill("f-cpu", local.limits?.cpuPercent ?? "");
|
|
4496
|
+
fill("f-memory", local.limits?.memoryMb ?? "");
|
|
4488
4497
|
const cacheDays = local.localArchiveCacheDays === undefined ? 7 : local.localArchiveCacheDays;
|
|
4489
4498
|
$("f-local-archive-cache-all").checked = cacheDays === null;
|
|
4490
4499
|
$("f-local-archive-cache-days").value = cacheDays === null ? "7" : String(cacheDays);
|
|
@@ -4512,6 +4521,18 @@ function collectSettingsPatch() {
|
|
|
4512
4521
|
const raw = $("f-reserve").value.trim();
|
|
4513
4522
|
return raw === "" ? null : Math.min(102_400, Math.max(200, Math.round(Number(raw) || 0)));
|
|
4514
4523
|
})(),
|
|
4524
|
+
recall: { autoInject: $("f-auto-inject")?.checked !== false },
|
|
4525
|
+
mcp: { enabled: $("f-mcp-enabled")?.checked !== false },
|
|
4526
|
+
limits: {
|
|
4527
|
+
cpuPercent: (() => {
|
|
4528
|
+
const raw = ($("f-cpu")?.value ?? "").trim();
|
|
4529
|
+
return raw === "" ? null : Math.min(100, Math.max(5, Math.round(Number(raw) || 30)));
|
|
4530
|
+
})(),
|
|
4531
|
+
memoryMb: (() => {
|
|
4532
|
+
const raw = ($("f-memory")?.value ?? "").trim();
|
|
4533
|
+
return raw === "" ? null : Math.min(65_536, Math.max(128, Math.round(Number(raw) || 1536)));
|
|
4534
|
+
})(),
|
|
4535
|
+
},
|
|
4515
4536
|
localArchiveCacheDays: $("f-local-archive-cache-all").checked
|
|
4516
4537
|
? null
|
|
4517
4538
|
: Math.min(3650, Math.max(0, Math.round(Number($("f-local-archive-cache-days").value) || 0)))
|
package/web/index.html
CHANGED
|
@@ -323,6 +323,17 @@
|
|
|
323
323
|
<p class="note">Off by default. When on, this machine permanently frees only originals whose encrypted archive is verified in storage, searchable, unchanged, idle, and past the grace period. Codex keeps a tiny same-session resume pointer. Policy and timing stay in the <a href="#rules">Free up disk rules</a>.</p>
|
|
324
324
|
</div>
|
|
325
325
|
|
|
326
|
+
<h3 id="machine-agent">What reaches your agent</h3>
|
|
327
|
+
<div class="field">
|
|
328
|
+
<label class="opt"><input id="f-auto-inject" type="checkbox"> Hand preserved history to an agent when a session starts</label>
|
|
329
|
+
<p class="note">On by default. Your agent opens a project and already knows where you left off, without being asked.</p>
|
|
330
|
+
<p class="note"><b>What it costs.</b> Anything placed in a conversation is re-read by the model on every later step, so a block added at the start is paid for again on every request that follows. Sealkeep sends each thing once and never repeats itself, which on a measured session cut this from 86 million tokens to 11 million, under one percent of what that session sent. Turn this off and preserved history is still written as notes your agent reads for itself, and is still searchable — it simply is not volunteered.</p>
|
|
331
|
+
</div>
|
|
332
|
+
<div class="field">
|
|
333
|
+
<label class="opt"><input id="f-mcp-enabled" type="checkbox"> Let agents search this vault themselves</label>
|
|
334
|
+
<p class="note">On by default. Runs a local tool server so Claude Code and Codex can search your sealed history on demand. Nothing is exposed to the network. Turn it off and agents keep the notes and the automatic recall, but cannot go looking for more.</p>
|
|
335
|
+
</div>
|
|
336
|
+
|
|
326
337
|
<h3 id="machine-resources">Speed and resource limits</h3>
|
|
327
338
|
<div class="row">
|
|
328
339
|
<div class="field"><label for="f-interval">Check every (minutes)</label><input id="f-interval" type="number" min="1" max="1440"></div>
|
|
@@ -332,6 +343,16 @@
|
|
|
332
343
|
<p class="note">Automatic shares one 8 Mbps background ceiling across Sealkeep jobs so agent calls, meetings, and normal browsing keep network headroom. This applies to unattended archive and shared-memory transfers on this machine.</p>
|
|
333
344
|
<p class="note">Background sealing, indexing, and memory preparation share a low-priority CPU budget targeting 30% of one CPU core. Short bursts can exceed this target. Large index workers have a 1.5 GB RSS safety limit. Foreground MCP searches are not throttled.</p>
|
|
334
345
|
</div>
|
|
346
|
+
<div class="field">
|
|
347
|
+
<label for="f-cpu">Background CPU ceiling, % of one core (blank for automatic)</label>
|
|
348
|
+
<input id="f-cpu" type="number" min="5" max="100" step="5" inputmode="numeric" placeholder="automatic · 30%">
|
|
349
|
+
<p class="note">Automatic targets 30% of one core for sealing, indexing, and memory preparation. A number here only ever makes Sealkeep gentler, never greedier, and below 5% it would stall rather than pace.</p>
|
|
350
|
+
</div>
|
|
351
|
+
<div class="field">
|
|
352
|
+
<label for="f-memory">Background memory ceiling, MB (blank for automatic)</label>
|
|
353
|
+
<input id="f-memory" type="number" min="128" max="65536" step="128" inputmode="numeric" placeholder="automatic · 1536 MB">
|
|
354
|
+
<p class="note">The ceiling a background worker may reach before it stops and retries smaller. Lower it on a small machine; it only ever tightens the automatic value.</p>
|
|
355
|
+
</div>
|
|
335
356
|
<div class="field">
|
|
336
357
|
<label for="f-reserve">Always keep free, MB (blank for automatic)</label>
|
|
337
358
|
<input id="f-reserve" type="number" min="200" placeholder="automatic">
|