session-steward 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +16 -0
- package/bin/session-steward-cli.mjs +4 -0
- package/dist/assets/index-C94A1O5c.js +9 -0
- package/dist/assets/index-CXq8Tw8T.css +2 -0
- package/dist/index.html +2 -2
- package/lib/cli.mjs +119 -2
- package/lib/providers/claude-code/events.mjs +66 -1
- package/lib/providers/claude-code/index.mjs +2 -0
- package/lib/providers/claude-code/store.mjs +2 -2
- package/lib/providers/claude-code/tokens.mjs +174 -0
- package/lib/providers/codex/database-families.mjs +18 -2
- package/lib/providers/codex/events.mjs +54 -1
- package/lib/providers/codex/index.mjs +2 -0
- package/lib/providers/codex/store.mjs +101 -1
- package/lib/providers/codex/tokens.mjs +317 -0
- package/lib/server.mjs +30 -0
- package/lib/session-events.mjs +76 -0
- package/lib/session-token-cache.mjs +76 -0
- package/lib/session-tokens.mjs +73 -0
- package/lib/storage/jsonl.mjs +10 -1
- package/package.json +2 -1
- package/dist/assets/index-6OPqaZRp.js +0 -9
- package/dist/assets/index-Cn_JrqDr.css +0 -2
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { addTokenTotals, createTokenTotals, summarizeSessionTokens } from "../../session-tokens.mjs";
|
|
2
|
+
import { readCachedTokens, readFileStamp, writeCachedTokens } from "../../session-token-cache.mjs";
|
|
3
|
+
import { visitJsonlSnapshotEntries } from "../../storage/jsonl.mjs";
|
|
4
|
+
import { getSessionRecord } from "./store.mjs";
|
|
5
|
+
|
|
6
|
+
const SYNTHETIC_MODEL = "<synthetic>";
|
|
7
|
+
|
|
8
|
+
// Claude Code reports usage per request rather than as a running total, so there
|
|
9
|
+
// is no counter to reconcile. The catch is the opposite one: a single response is
|
|
10
|
+
// written as several records, one per content block, and every copy repeats the
|
|
11
|
+
// same usage in full. Summing records instead of requests inflates a session by
|
|
12
|
+
// roughly 2.3x.
|
|
13
|
+
function readCount(value) {
|
|
14
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Anthropic reports cached tokens *beside* the input count, where Codex reports
|
|
18
|
+
// them inside it. `input_tokens` is already the uncached remainder, so nothing is
|
|
19
|
+
// subtracted here — doing so is what would drive the fresh-input slice negative.
|
|
20
|
+
function normalizeUsage(value) {
|
|
21
|
+
if (!value || typeof value !== "object") return null;
|
|
22
|
+
|
|
23
|
+
const cachedInput = readCount(value.cache_read_input_tokens);
|
|
24
|
+
const cacheWrites = readCount(value.cache_creation_input_tokens);
|
|
25
|
+
const freshInput = readCount(value.input_tokens);
|
|
26
|
+
const output = readCount(value.output_tokens);
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
cachedInput,
|
|
30
|
+
cacheWrites,
|
|
31
|
+
freshInput,
|
|
32
|
+
output,
|
|
33
|
+
// Claude Code does not report a reasoning figure; thinking is billed as
|
|
34
|
+
// ordinary output and cannot be separated from it.
|
|
35
|
+
reasoning: 0,
|
|
36
|
+
total: cachedInput + cacheWrites + freshInput + output,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function createClaudeTokenCollector() {
|
|
41
|
+
// One entry per request, not per record. Repeats collapse onto the largest
|
|
42
|
+
// copy: a retried or superseded write reports zeros, and a streaming partial
|
|
43
|
+
// reports less than the finished response.
|
|
44
|
+
const requests = new Map();
|
|
45
|
+
let compactions = 0;
|
|
46
|
+
let observedRecords = 0;
|
|
47
|
+
let sessionId = null;
|
|
48
|
+
let sidechainRequests = 0;
|
|
49
|
+
let syntheticRecords = 0;
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
record(recordValue) {
|
|
53
|
+
if (!recordValue || typeof recordValue !== "object") return;
|
|
54
|
+
|
|
55
|
+
if (recordValue.subtype === "compact_boundary") compactions += 1;
|
|
56
|
+
if (sessionId === null && typeof recordValue.sessionId === "string") {
|
|
57
|
+
sessionId = recordValue.sessionId;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const message = recordValue.message;
|
|
61
|
+
if (!message || typeof message !== "object") return;
|
|
62
|
+
|
|
63
|
+
const usage = normalizeUsage(message.usage);
|
|
64
|
+
if (!usage) return;
|
|
65
|
+
observedRecords += 1;
|
|
66
|
+
|
|
67
|
+
// Synthetic entries stand in for locally generated messages. They carry no
|
|
68
|
+
// usage and no request id, and would otherwise open a model row of zeros.
|
|
69
|
+
if (message.model === SYNTHETIC_MODEL) {
|
|
70
|
+
syntheticRecords += 1;
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const key = recordValue.requestId ?? message.id;
|
|
75
|
+
if (typeof key !== "string") return;
|
|
76
|
+
|
|
77
|
+
const previous = requests.get(key);
|
|
78
|
+
if (previous && previous.usage.total >= usage.total) return;
|
|
79
|
+
if (!previous && recordValue.isSidechain) sidechainRequests += 1;
|
|
80
|
+
|
|
81
|
+
requests.set(key, { model: message.model ?? "Unknown", sidechain: Boolean(recordValue.isSidechain), usage });
|
|
82
|
+
},
|
|
83
|
+
|
|
84
|
+
result() {
|
|
85
|
+
const totals = createTokenTotals();
|
|
86
|
+
const modelTotals = new Map();
|
|
87
|
+
|
|
88
|
+
for (const { model, usage } of requests.values()) {
|
|
89
|
+
addTokenTotals(totals, usage);
|
|
90
|
+
if (!modelTotals.has(model)) modelTotals.set(model, createTokenTotals());
|
|
91
|
+
addTokenTotals(modelTotals.get(model), usage);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return {
|
|
95
|
+
available: requests.size > 0,
|
|
96
|
+
byModel: [...modelTotals]
|
|
97
|
+
.map(([model, modelSpend]) => ({ model, totals: modelSpend }))
|
|
98
|
+
.sort((left, right) => right.totals.total - left.totals.total),
|
|
99
|
+
cacheWriteUnderflow: false,
|
|
100
|
+
compactions,
|
|
101
|
+
countedRequests: requests.size,
|
|
102
|
+
observedRecords,
|
|
103
|
+
sessionId,
|
|
104
|
+
sidechainRequests,
|
|
105
|
+
syntheticRecords,
|
|
106
|
+
totals,
|
|
107
|
+
};
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export async function collectClaudeSessionTokens(filePath, { maxLineBytes, signal } = {}) {
|
|
113
|
+
const collector = createClaudeTokenCollector();
|
|
114
|
+
const { complete, snapshotBytes } = await visitJsonlSnapshotEntries(
|
|
115
|
+
filePath,
|
|
116
|
+
({ parsed }) => {
|
|
117
|
+
// A closed panel should not leave a large transcript being scanned.
|
|
118
|
+
if (signal?.aborted) return false;
|
|
119
|
+
if (parsed && typeof parsed === "object") collector.record(parsed);
|
|
120
|
+
return true;
|
|
121
|
+
},
|
|
122
|
+
maxLineBytes === undefined ? {} : { maxLineBytes },
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
return { ...collector.result(), complete, snapshotBytes };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Reading the timeline already streams every record of the same file, so the
|
|
129
|
+
// count rides along with that pass instead of paying for a second one.
|
|
130
|
+
export async function createSessionTokenScan({ record, signal }) {
|
|
131
|
+
if (!record?.rolloutPath) return null;
|
|
132
|
+
const stamp = await readFileStamp(record.rolloutPath);
|
|
133
|
+
const cached = readCachedTokens("summary", record.rolloutPath, stamp);
|
|
134
|
+
if (cached !== undefined) return { cached, record() {}, summarize: () => cached };
|
|
135
|
+
|
|
136
|
+
const collector = createClaudeTokenCollector();
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
cached: null,
|
|
140
|
+
record(value) {
|
|
141
|
+
collector.record(value);
|
|
142
|
+
},
|
|
143
|
+
summarize({ complete }) {
|
|
144
|
+
const summary = summarizeSessionTokens({ ...collector.result(), complete });
|
|
145
|
+
if (signal?.aborted || complete === false) return summary;
|
|
146
|
+
return writeCachedTokens("summary", record.rolloutPath, stamp, summary);
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export async function readSessionTokens({ claudeHome, desktopDataHome, id, maxLineBytes, signal }) {
|
|
152
|
+
const record = await getSessionRecord({ claudeHome, desktopDataHome, id });
|
|
153
|
+
if (!record) return null;
|
|
154
|
+
if (!record.rolloutPath) return { available: false, reason: "no-transcript-path" };
|
|
155
|
+
|
|
156
|
+
const stamp = await readFileStamp(record.rolloutPath);
|
|
157
|
+
const cached = readCachedTokens("summary", record.rolloutPath, stamp);
|
|
158
|
+
if (cached !== undefined) return cached;
|
|
159
|
+
|
|
160
|
+
let collected;
|
|
161
|
+
try {
|
|
162
|
+
collected = await collectClaudeSessionTokens(record.rolloutPath, { maxLineBytes, signal });
|
|
163
|
+
} catch (error) {
|
|
164
|
+
// A transcript can be deleted between the listing and the read. That is an
|
|
165
|
+
// answer about the session, not a failure of the server, and the timeline
|
|
166
|
+
// reader already treats it as one.
|
|
167
|
+
if (error?.code === "ENOENT") return { available: false, reason: "transcript-missing" };
|
|
168
|
+
throw error;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const summary = summarizeSessionTokens(collected);
|
|
172
|
+
if (signal?.aborted || collected.complete === false) return summary;
|
|
173
|
+
return writeCachedTokens("summary", record.rolloutPath, stamp, summary);
|
|
174
|
+
}
|
|
@@ -8,8 +8,8 @@ const CACHE_TTL_MS = 2_000;
|
|
|
8
8
|
export const CODEX_DATABASE_PROFILE = Object.freeze({
|
|
9
9
|
id: "codex-local-store-2026-08",
|
|
10
10
|
builtFor: {
|
|
11
|
-
chatgptDesktop: ["26.727.40816", "26.803.61601"],
|
|
12
|
-
codexCli: ["0.144.1", "0.146.0", "0.147.0"],
|
|
11
|
+
chatgptDesktop: ["26.727.40816", "26.803.61601", "26.818.21641", "26.818.22352"],
|
|
12
|
+
codexCli: ["0.144.1", "0.146.0", "0.147.0", "0.148.0"],
|
|
13
13
|
},
|
|
14
14
|
});
|
|
15
15
|
|
|
@@ -41,6 +41,22 @@ const SCHEMA_REQUIREMENTS = Object.freeze({
|
|
|
41
41
|
{ name: "thread_goal_continuation_deferrals", requiredColumns: ["thread_id"] },
|
|
42
42
|
],
|
|
43
43
|
},
|
|
44
|
+
queue: {
|
|
45
|
+
fallback: "queue_1.sqlite",
|
|
46
|
+
pattern: /^queue_(\d+)\.sqlite$/u,
|
|
47
|
+
required: false,
|
|
48
|
+
tables: [{ name: "queued_items", requiredColumns: ["thread_id"] }],
|
|
49
|
+
},
|
|
50
|
+
threadHistory: {
|
|
51
|
+
fallback: "thread_history_1.sqlite",
|
|
52
|
+
pattern: /^thread_history_(\d+)\.sqlite$/u,
|
|
53
|
+
required: false,
|
|
54
|
+
tables: [
|
|
55
|
+
{ name: "thread_items", requiredColumns: ["thread_id"] },
|
|
56
|
+
{ name: "thread_turns", requiredColumns: ["thread_id"] },
|
|
57
|
+
{ name: "thread_history_projection_state", requiredColumns: ["thread_id"] },
|
|
58
|
+
],
|
|
59
|
+
},
|
|
44
60
|
});
|
|
45
61
|
|
|
46
62
|
const resolutionCache = new Map();
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createSessionEvent,
|
|
3
|
+
createSessionEventComposition,
|
|
3
4
|
createSessionEventCoverage,
|
|
4
5
|
createSessionEventHeader,
|
|
6
|
+
createSessionEventSummary,
|
|
5
7
|
createSessionEventsResult,
|
|
8
|
+
finalizeSessionEventComposition,
|
|
6
9
|
SESSION_EVENT_KIND,
|
|
10
|
+
SESSION_EVENT_READ_MODE,
|
|
7
11
|
SESSION_EVENT_REASON,
|
|
8
12
|
} from "../../session-events.mjs";
|
|
9
13
|
import {
|
|
@@ -13,6 +17,7 @@ import {
|
|
|
13
17
|
} from "../../session-event-reader.mjs";
|
|
14
18
|
import { visitJsonlSnapshotEntries } from "../../storage/jsonl.mjs";
|
|
15
19
|
import { getSessionRecord, loadSessionStore } from "./store.mjs";
|
|
20
|
+
import { createSessionTokenScan } from "./tokens.mjs";
|
|
16
21
|
|
|
17
22
|
const PROVIDER_ID = "codex";
|
|
18
23
|
const RECORD_CLASSIFICATION = Object.freeze({
|
|
@@ -53,6 +58,30 @@ function createDuplicateTextTracker() {
|
|
|
53
58
|
};
|
|
54
59
|
}
|
|
55
60
|
|
|
61
|
+
const COMPOSITION_TOOL_OUTPUT = new Set([
|
|
62
|
+
"custom_tool_call_output",
|
|
63
|
+
"exec_command_end",
|
|
64
|
+
"function_call_output",
|
|
65
|
+
"mcp_tool_call_end",
|
|
66
|
+
"web_search_end",
|
|
67
|
+
]);
|
|
68
|
+
const COMPOSITION_MESSAGES = new Set(["agent_message", "message", "user_message"]);
|
|
69
|
+
const COMPOSITION_EDITS = new Set(["patch_apply_begin", "patch_apply_end"]);
|
|
70
|
+
const COMPOSITION_REASONING = new Set(["agent_reasoning", "reasoning"]);
|
|
71
|
+
|
|
72
|
+
function compositionSegment(parsed) {
|
|
73
|
+
const type = typeof parsed?.type === "string" ? parsed.type : "";
|
|
74
|
+
const payloadType = typeof parsed?.payload?.type === "string" ? parsed.payload.type : type;
|
|
75
|
+
if (type === "compacted" || payloadType === "compacted" || payloadType === "context_compacted") {
|
|
76
|
+
return "compaction";
|
|
77
|
+
}
|
|
78
|
+
if (COMPOSITION_TOOL_OUTPUT.has(payloadType)) return "toolOutput";
|
|
79
|
+
if (COMPOSITION_EDITS.has(payloadType)) return "edits";
|
|
80
|
+
if (COMPOSITION_REASONING.has(payloadType)) return "reasoning";
|
|
81
|
+
if (COMPOSITION_MESSAGES.has(payloadType)) return "messages";
|
|
82
|
+
return "other";
|
|
83
|
+
}
|
|
84
|
+
|
|
56
85
|
function asTimestamp(value) {
|
|
57
86
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
58
87
|
if (typeof value !== "string") return null;
|
|
@@ -298,12 +327,15 @@ function outputText(payload) {
|
|
|
298
327
|
return contentText(payload.output ?? payload.content ?? payload.stderr ?? payload.stdout).trim() || null;
|
|
299
328
|
}
|
|
300
329
|
|
|
301
|
-
function emptyResult({ cwd = null, origin = null, reason }) {
|
|
330
|
+
function emptyResult({ counted = false, cwd = null, origin = null, reason }) {
|
|
302
331
|
return createSessionEventsResult({
|
|
303
332
|
coverage: createSessionEventCoverage(),
|
|
304
333
|
events: [],
|
|
305
334
|
header: createSessionEventHeader({ cwd, origin, provider: PROVIDER_ID }),
|
|
306
335
|
reason,
|
|
336
|
+
// The reason the timeline is empty is the same reason there is no count:
|
|
337
|
+
// no transcript to read. Saying so beats leaving the field to guess.
|
|
338
|
+
tokens: counted ? { available: false, reason } : null,
|
|
307
339
|
window: {
|
|
308
340
|
complete: true,
|
|
309
341
|
end: null,
|
|
@@ -325,6 +357,7 @@ export async function readSessionEvents({
|
|
|
325
357
|
maxLineBytes,
|
|
326
358
|
mode,
|
|
327
359
|
signal,
|
|
360
|
+
tokens = false,
|
|
328
361
|
}) {
|
|
329
362
|
const record = await findSessionRecord({ codexHome, id });
|
|
330
363
|
if (!record) return null;
|
|
@@ -332,13 +365,22 @@ export async function readSessionEvents({
|
|
|
332
365
|
const origin = record.recordSource ?? null;
|
|
333
366
|
if (!record.rolloutPath) {
|
|
334
367
|
return emptyResult({
|
|
368
|
+
counted: tokens,
|
|
335
369
|
cwd: record.cwd || null,
|
|
336
370
|
origin,
|
|
337
371
|
reason: SESSION_EVENT_REASON.NO_TRANSCRIPT_PATH,
|
|
338
372
|
});
|
|
339
373
|
}
|
|
340
374
|
|
|
375
|
+
// A preview stops as soon as it has enough events, so a count taken from it
|
|
376
|
+
// would be of part of the file while reading as the whole.
|
|
377
|
+
const tokenScan = tokens && mode !== SESSION_EVENT_READ_MODE.PREVIEW
|
|
378
|
+
? await createSessionTokenScan({ codexHome, maxLineBytes, record, signal })
|
|
379
|
+
: null;
|
|
380
|
+
|
|
341
381
|
const coverage = createSessionEventCoverage();
|
|
382
|
+
const summary = createSessionEventSummary();
|
|
383
|
+
const composition = createSessionEventComposition();
|
|
342
384
|
const readState = createSessionEventReadState({ limit, mode });
|
|
343
385
|
const unmappedTypes = createUnmappedSessionEventTracker();
|
|
344
386
|
const duplicateTexts = createDuplicateTextTracker();
|
|
@@ -350,6 +392,9 @@ export async function readSessionEvents({
|
|
|
350
392
|
});
|
|
351
393
|
|
|
352
394
|
function addEvent(event, pendingId = null) {
|
|
395
|
+
if (event.kind === SESSION_EVENT_KIND.ASK && !event.injected) summary.asks += 1;
|
|
396
|
+
if (event.kind === SESSION_EVENT_KIND.EDIT) summary.edits += 1;
|
|
397
|
+
if (event.kind === SESSION_EVENT_KIND.RAN) summary.commands += 1;
|
|
353
398
|
return readState.add(event, { pendingId });
|
|
354
399
|
}
|
|
355
400
|
|
|
@@ -735,14 +780,18 @@ export async function readSessionEvents({
|
|
|
735
780
|
|
|
736
781
|
if (entry.oversized) {
|
|
737
782
|
coverage.oversized += 1;
|
|
783
|
+
composition.largeRecords += entry.bytes;
|
|
738
784
|
return true;
|
|
739
785
|
}
|
|
740
786
|
|
|
741
787
|
if (!entry.parsed || typeof entry.parsed !== "object") {
|
|
742
788
|
coverage.unparseable += 1;
|
|
789
|
+
composition.other += entry.bytes;
|
|
743
790
|
return true;
|
|
744
791
|
}
|
|
745
792
|
|
|
793
|
+
composition[compositionSegment(entry.parsed)] += entry.bytes;
|
|
794
|
+
tokenScan?.record(entry.parsed);
|
|
746
795
|
const result = handleRecord(entry.parsed, entry.index);
|
|
747
796
|
coverage[result.classification] += 1;
|
|
748
797
|
if (result.duplicate) coverage.duplicates += 1;
|
|
@@ -756,6 +805,7 @@ export async function readSessionEvents({
|
|
|
756
805
|
} catch (error) {
|
|
757
806
|
if (error?.code === "ENOENT") {
|
|
758
807
|
return emptyResult({
|
|
808
|
+
counted: tokens,
|
|
759
809
|
cwd: record.cwd || null,
|
|
760
810
|
origin,
|
|
761
811
|
reason: SESSION_EVENT_REASON.TRANSCRIPT_MISSING,
|
|
@@ -774,6 +824,9 @@ export async function readSessionEvents({
|
|
|
774
824
|
reason: events.length === 0 && read.complete
|
|
775
825
|
? SESSION_EVENT_REASON.NO_RECOGNIZED_EVENTS
|
|
776
826
|
: null,
|
|
827
|
+
composition: finalizeSessionEventComposition(composition, read.snapshotBytes),
|
|
828
|
+
summary,
|
|
829
|
+
tokens: tokenScan ? tokenScan.summarize(read) : null,
|
|
777
830
|
window: readState.window(read),
|
|
778
831
|
});
|
|
779
832
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readSessionEvents } from "./events.mjs";
|
|
2
|
+
import { readSessionTokens } from "./tokens.mjs";
|
|
2
3
|
import {
|
|
3
4
|
assertDeepCleanupSupported,
|
|
4
5
|
diagnoseStorageCompatibility,
|
|
@@ -40,6 +41,7 @@ export const codexProvider = Object.freeze({
|
|
|
40
41
|
planSessionDeletion,
|
|
41
42
|
preflightSessionDeletion,
|
|
42
43
|
readSessionEvents,
|
|
44
|
+
readSessionTokens,
|
|
43
45
|
restoreSessionDeletionBackup,
|
|
44
46
|
verifySessionDeletion,
|
|
45
47
|
});
|
|
@@ -440,11 +440,13 @@ export function getCodexPaths(codexHomeInput, { refresh = false } = {}) {
|
|
|
440
440
|
const archivedSessionsDirectory = path.join(codexHome, "archived_sessions");
|
|
441
441
|
const sessionsDirectory = path.join(codexHome, "sessions");
|
|
442
442
|
const resolution = resolveCodexDatabases(codexHome, { refresh });
|
|
443
|
-
const { goals, logs, memories, state } = resolution.families;
|
|
443
|
+
const { goals, logs, memories, queue, state, threadHistory } = resolution.families;
|
|
444
444
|
const stateDatabasePath = state.primary?.path ?? path.join(codexHome, "state_5.sqlite");
|
|
445
445
|
const logsDatabasePath = logs.primary?.path ?? path.join(codexHome, "logs_2.sqlite");
|
|
446
446
|
const memoryDatabasePath = memories.primary?.path ?? path.join(codexHome, "memories_1.sqlite");
|
|
447
447
|
const goalsDatabasePath = goals.primary?.path ?? path.join(codexHome, "goals_1.sqlite");
|
|
448
|
+
const queueDatabasePath = queue.primary?.path ?? path.join(codexHome, "queue_1.sqlite");
|
|
449
|
+
const threadHistoryDatabasePath = threadHistory.primary?.path ?? path.join(codexHome, "thread_history_1.sqlite");
|
|
448
450
|
|
|
449
451
|
return {
|
|
450
452
|
archivedSessionsDirectory,
|
|
@@ -460,6 +462,8 @@ export function getCodexPaths(codexHomeInput, { refresh = false } = {}) {
|
|
|
460
462
|
logsDatabasePaths: allValidDatabases(logs).map((database) => database.path),
|
|
461
463
|
memoryDatabasePath,
|
|
462
464
|
memoryDatabasePaths: allValidDatabases(memories).map((database) => database.path),
|
|
465
|
+
queueDatabasePath,
|
|
466
|
+
queueDatabasePaths: allValidDatabases(queue).map((database) => database.path),
|
|
463
467
|
resolvedDatabases: databaseFamilySummary(resolution),
|
|
464
468
|
sessionIndexPath: path.join(codexHome, "session_index.jsonl"),
|
|
465
469
|
sessionsDirectory,
|
|
@@ -467,6 +471,8 @@ export function getCodexPaths(codexHomeInput, { refresh = false } = {}) {
|
|
|
467
471
|
stateDatabases: allValidDatabases(state),
|
|
468
472
|
stateDatabasePaths: allValidDatabases(state).map((database) => database.path),
|
|
469
473
|
threadWriterLocksDirectory: path.join(codexHome, "thread-writer-locks"),
|
|
474
|
+
threadHistoryDatabasePath,
|
|
475
|
+
threadHistoryDatabasePaths: allValidDatabases(threadHistory).map((database) => database.path),
|
|
470
476
|
transcriptDirectories: [sessionsDirectory, archivedSessionsDirectory],
|
|
471
477
|
};
|
|
472
478
|
}
|
|
@@ -569,12 +575,16 @@ export async function loadSessionStore({ codexHome }) {
|
|
|
569
575
|
hasGoalsDatabase,
|
|
570
576
|
hasLogsDatabase,
|
|
571
577
|
hasMemoryDatabase,
|
|
578
|
+
hasQueueDatabase,
|
|
579
|
+
hasThreadHistoryDatabase,
|
|
572
580
|
] = await Promise.all([
|
|
573
581
|
pathExists(paths.desktopStatePath),
|
|
574
582
|
pathExists(paths.desktopStateBackupPath),
|
|
575
583
|
pathExists(paths.goalsDatabasePath),
|
|
576
584
|
pathExists(paths.logsDatabasePath),
|
|
577
585
|
pathExists(paths.memoryDatabasePath),
|
|
586
|
+
pathExists(paths.queueDatabasePath),
|
|
587
|
+
pathExists(paths.threadHistoryDatabasePath),
|
|
578
588
|
]);
|
|
579
589
|
const threadRowsById = new Map();
|
|
580
590
|
const spawnEdgesByKey = new Map();
|
|
@@ -768,9 +778,12 @@ export async function loadSessionStore({ codexHome }) {
|
|
|
768
778
|
hasGoalsDatabase,
|
|
769
779
|
hasLogsDatabase,
|
|
770
780
|
hasMemoryDatabase,
|
|
781
|
+
hasQueueDatabase,
|
|
782
|
+
hasThreadHistoryDatabase,
|
|
771
783
|
goalsDatabasePath: paths.goalsDatabasePath,
|
|
772
784
|
logsDatabasePath: paths.logsDatabasePath,
|
|
773
785
|
memoryDatabasePath: paths.memoryDatabasePath,
|
|
786
|
+
queueDatabasePath: paths.queueDatabasePath,
|
|
774
787
|
records,
|
|
775
788
|
recordsById: new Map(records.map((record) => [record.id, record])),
|
|
776
789
|
sessionIndexPath: paths.sessionIndexPath,
|
|
@@ -782,6 +795,9 @@ export async function loadSessionStore({ codexHome }) {
|
|
|
782
795
|
resolvedDatabases: paths.resolvedDatabases,
|
|
783
796
|
logsDatabasePaths: paths.logsDatabasePaths,
|
|
784
797
|
memoryDatabasePaths: paths.memoryDatabasePaths,
|
|
798
|
+
queueDatabasePaths: paths.queueDatabasePaths,
|
|
799
|
+
threadHistoryDatabasePath: paths.threadHistoryDatabasePath,
|
|
800
|
+
threadHistoryDatabasePaths: paths.threadHistoryDatabasePaths,
|
|
785
801
|
goalsDatabasePaths: paths.goalsDatabasePaths,
|
|
786
802
|
transcriptHeaders,
|
|
787
803
|
historyPath: paths.historyPath,
|
|
@@ -1217,12 +1233,16 @@ async function getStoreAvailability(paths) {
|
|
|
1217
1233
|
hasGoalsDatabase,
|
|
1218
1234
|
hasLogsDatabase,
|
|
1219
1235
|
hasMemoryDatabase,
|
|
1236
|
+
hasQueueDatabase,
|
|
1237
|
+
hasThreadHistoryDatabase,
|
|
1220
1238
|
] = await Promise.all([
|
|
1221
1239
|
pathExists(paths.desktopStatePath),
|
|
1222
1240
|
pathExists(paths.desktopStateBackupPath),
|
|
1223
1241
|
Promise.resolve(paths.goalsDatabasePaths.length > 0),
|
|
1224
1242
|
Promise.resolve(paths.logsDatabasePaths.length > 0),
|
|
1225
1243
|
Promise.resolve(paths.memoryDatabasePaths.length > 0),
|
|
1244
|
+
Promise.resolve(paths.queueDatabasePaths.length > 0),
|
|
1245
|
+
Promise.resolve(paths.threadHistoryDatabasePaths.length > 0),
|
|
1226
1246
|
]);
|
|
1227
1247
|
|
|
1228
1248
|
return {
|
|
@@ -1231,6 +1251,8 @@ async function getStoreAvailability(paths) {
|
|
|
1231
1251
|
hasGoalsDatabase,
|
|
1232
1252
|
hasLogsDatabase,
|
|
1233
1253
|
hasMemoryDatabase,
|
|
1254
|
+
hasQueueDatabase,
|
|
1255
|
+
hasThreadHistoryDatabase,
|
|
1234
1256
|
};
|
|
1235
1257
|
}
|
|
1236
1258
|
|
|
@@ -1485,6 +1507,7 @@ export async function loadDeletionStore({ codexHome, recordIds }) {
|
|
|
1485
1507
|
historyPath: paths.historyPath,
|
|
1486
1508
|
logsDatabasePath: paths.logsDatabasePath,
|
|
1487
1509
|
memoryDatabasePath: paths.memoryDatabasePath,
|
|
1510
|
+
queueDatabasePath: paths.queueDatabasePath,
|
|
1488
1511
|
records,
|
|
1489
1512
|
recordsById: new Map(records.map((record) => [record.id, record])),
|
|
1490
1513
|
sessionIndexPath: paths.sessionIndexPath,
|
|
@@ -1495,6 +1518,9 @@ export async function loadDeletionStore({ codexHome, recordIds }) {
|
|
|
1495
1518
|
resolvedDatabases: paths.resolvedDatabases,
|
|
1496
1519
|
logsDatabasePaths: paths.logsDatabasePaths,
|
|
1497
1520
|
memoryDatabasePaths: paths.memoryDatabasePaths,
|
|
1521
|
+
queueDatabasePaths: paths.queueDatabasePaths,
|
|
1522
|
+
threadHistoryDatabasePath: paths.threadHistoryDatabasePath,
|
|
1523
|
+
threadHistoryDatabasePaths: paths.threadHistoryDatabasePaths,
|
|
1498
1524
|
goalsDatabasePaths: paths.goalsDatabasePaths,
|
|
1499
1525
|
transcriptHeaders: new Map(
|
|
1500
1526
|
[...transcriptHeaders].filter(([id]) => relevantIds.has(id)),
|
|
@@ -1954,6 +1980,19 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
1954
1980
|
.reduce((total, databasePath) => total + countRowsForIds(databasePath, "stage1_outputs", "thread_id", deletionIds), 0);
|
|
1955
1981
|
const goalRowCount = (store.goalsDatabasePaths ?? [store.goalsDatabasePath].filter(Boolean))
|
|
1956
1982
|
.reduce((total, databasePath) => total + countRowsForIds(databasePath, "thread_goals", "thread_id", deletionIds), 0);
|
|
1983
|
+
const queueRowCount = (store.queueDatabasePaths ?? [store.queueDatabasePath].filter(Boolean))
|
|
1984
|
+
.reduce((total, databasePath) => total + countRowsForIds(databasePath, "queued_items", "thread_id", deletionIds), 0);
|
|
1985
|
+
const queueRevisionRowCount = (store.queueDatabasePaths ?? [store.queueDatabasePath].filter(Boolean))
|
|
1986
|
+
.reduce((total, databasePath) => total + (inspectSqliteTable(databasePath, "queued_thread_revisions").exists
|
|
1987
|
+
? countRowsForIds(databasePath, "queued_thread_revisions", "thread_id", deletionIds)
|
|
1988
|
+
: 0), 0);
|
|
1989
|
+
const threadHistoryRowCount = (store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath].filter(Boolean))
|
|
1990
|
+
.reduce((total, databasePath) => total + [
|
|
1991
|
+
"thread_items",
|
|
1992
|
+
"thread_turns",
|
|
1993
|
+
"thread_history_projection_state",
|
|
1994
|
+
].reduce((databaseTotal, tableName) =>
|
|
1995
|
+
databaseTotal + countRowsForIds(databasePath, tableName, "thread_id", deletionIds), 0), 0);
|
|
1957
1996
|
const stateTargets = (store.stateDatabases ?? [{ path: store.stateDatabasePath }]).map((database) => ({
|
|
1958
1997
|
database,
|
|
1959
1998
|
ids: findRowsForIds(database.path, "threads", "id", deletionIds).map((row) => String(row.id)),
|
|
@@ -1972,6 +2011,8 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
1972
2011
|
ids: deletionIds,
|
|
1973
2012
|
logRowCount,
|
|
1974
2013
|
memoryRowCount,
|
|
2014
|
+
queueRowCount,
|
|
2015
|
+
queueRevisionRowCount,
|
|
1975
2016
|
missingTranscriptPaths,
|
|
1976
2017
|
newestLinkedActivityAtMs,
|
|
1977
2018
|
records: selectedRecords,
|
|
@@ -1980,6 +2021,7 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
1980
2021
|
stateTargets,
|
|
1981
2022
|
transcriptBytes,
|
|
1982
2023
|
transcriptFileCount,
|
|
2024
|
+
threadHistoryRowCount,
|
|
1983
2025
|
transcriptPaths,
|
|
1984
2026
|
};
|
|
1985
2027
|
}
|
|
@@ -1991,6 +2033,8 @@ function getBackupSourcePaths({ plan, scope, store }) {
|
|
|
1991
2033
|
const databasePaths = [
|
|
1992
2034
|
...(store.stateDatabasePaths ?? [store.stateDatabasePath]),
|
|
1993
2035
|
...(store.logsDatabasePaths ?? [store.logsDatabasePath].filter(Boolean)),
|
|
2036
|
+
...(store.queueDatabasePaths ?? [store.queueDatabasePath].filter(Boolean)),
|
|
2037
|
+
...(store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath].filter(Boolean)),
|
|
1994
2038
|
...(scope === "deep" ? (store.memoryDatabasePaths ?? [store.memoryDatabasePath].filter(Boolean)) : []),
|
|
1995
2039
|
...(scope === "deep" ? (store.goalsDatabasePaths ?? [store.goalsDatabasePath].filter(Boolean)) : []),
|
|
1996
2040
|
].filter(Boolean);
|
|
@@ -2033,6 +2077,9 @@ export async function fingerprintSessionDeletion({ plan, scope, store }) {
|
|
|
2033
2077
|
plan.dynamicToolRowCount,
|
|
2034
2078
|
plan.historyMatchCount,
|
|
2035
2079
|
plan.logRowCount,
|
|
2080
|
+
plan.queueRowCount,
|
|
2081
|
+
plan.queueRevisionRowCount,
|
|
2082
|
+
plan.threadHistoryRowCount,
|
|
2036
2083
|
plan.sessionIndexMatchCount,
|
|
2037
2084
|
plan.spawnEdgeCount,
|
|
2038
2085
|
...(scope === "deep" ? [plan.goalRowCount, plan.memoryRowCount] : []),
|
|
@@ -2040,6 +2087,8 @@ export async function fingerprintSessionDeletion({ plan, scope, store }) {
|
|
|
2040
2087
|
hash.update(`counts\0${counts.join("\0")}\0`);
|
|
2041
2088
|
hash.update(`stores\0${[
|
|
2042
2089
|
store.hasLogsDatabase,
|
|
2090
|
+
store.hasQueueDatabase,
|
|
2091
|
+
store.hasThreadHistoryDatabase,
|
|
2043
2092
|
...(scope === "deep" ? [
|
|
2044
2093
|
store.hasDesktopState,
|
|
2045
2094
|
store.hasDesktopStateBackup,
|
|
@@ -2079,6 +2128,17 @@ export async function fingerprintSessionDeletion({ plan, scope, store }) {
|
|
|
2079
2128
|
for (const databasePath of store.logsDatabasePaths ?? [store.logsDatabasePath].filter(Boolean)) {
|
|
2080
2129
|
fingerprintRowsForIds(hash, databasePath, "logs", "thread_id", plan.ids);
|
|
2081
2130
|
}
|
|
2131
|
+
for (const databasePath of store.queueDatabasePaths ?? [store.queueDatabasePath].filter(Boolean)) {
|
|
2132
|
+
fingerprintRowsForIds(hash, databasePath, "queued_items", "thread_id", plan.ids);
|
|
2133
|
+
if (inspectSqliteTable(databasePath, "queued_thread_revisions").exists) {
|
|
2134
|
+
fingerprintRowsForIds(hash, databasePath, "queued_thread_revisions", "thread_id", plan.ids);
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
for (const databasePath of store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath].filter(Boolean)) {
|
|
2138
|
+
fingerprintRowsForIds(hash, databasePath, "thread_items", "thread_id", plan.ids);
|
|
2139
|
+
fingerprintRowsForIds(hash, databasePath, "thread_turns", "thread_id", plan.ids);
|
|
2140
|
+
fingerprintRowsForIds(hash, databasePath, "thread_history_projection_state", "thread_id", plan.ids);
|
|
2141
|
+
}
|
|
2082
2142
|
if (scope === "deep") {
|
|
2083
2143
|
for (const databasePath of store.memoryDatabasePaths ?? [store.memoryDatabasePath].filter(Boolean)) {
|
|
2084
2144
|
fingerprintRowsForIds(hash, databasePath, "stage1_outputs", "thread_id", plan.ids);
|
|
@@ -2292,6 +2352,8 @@ async function createOperationBackup({ onProgress, plan, scope, store }) {
|
|
|
2292
2352
|
const snapshotCandidates = [
|
|
2293
2353
|
...(store.stateDatabasePaths ?? [store.stateDatabasePath]).map((databasePath) => [databasePath, path.basename(databasePath), true]),
|
|
2294
2354
|
...(store.logsDatabasePaths ?? [store.logsDatabasePath].filter(Boolean)).map((databasePath) => [databasePath, path.basename(databasePath), true]),
|
|
2355
|
+
...(store.queueDatabasePaths ?? [store.queueDatabasePath].filter(Boolean)).map((databasePath) => [databasePath, path.basename(databasePath), true]),
|
|
2356
|
+
...(store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath].filter(Boolean)).map((databasePath) => [databasePath, path.basename(databasePath), true]),
|
|
2295
2357
|
...(scope === "deep" ? (store.memoryDatabasePaths ?? [store.memoryDatabasePath].filter(Boolean)) : []).map((databasePath) => [databasePath, path.basename(databasePath), true]),
|
|
2296
2358
|
...(scope === "deep" ? (store.goalsDatabasePaths ?? [store.goalsDatabasePath].filter(Boolean)) : []).map((databasePath) => [databasePath, path.basename(databasePath), true]),
|
|
2297
2359
|
];
|
|
@@ -2467,6 +2529,25 @@ export async function executeSessionDeletion({
|
|
|
2467
2529
|
executeTransaction(databasePath, deleteStatements("logs", "thread_id", plan.ids));
|
|
2468
2530
|
}
|
|
2469
2531
|
}
|
|
2532
|
+
if (store.hasQueueDatabase) {
|
|
2533
|
+
for (const databasePath of store.queueDatabasePaths ?? [store.queueDatabasePath]) {
|
|
2534
|
+
executeTransaction(databasePath, [
|
|
2535
|
+
...deleteStatements("queued_items", "thread_id", plan.ids),
|
|
2536
|
+
...(inspectSqliteTable(databasePath, "queued_thread_revisions").exists
|
|
2537
|
+
? deleteStatements("queued_thread_revisions", "thread_id", plan.ids)
|
|
2538
|
+
: []),
|
|
2539
|
+
]);
|
|
2540
|
+
}
|
|
2541
|
+
}
|
|
2542
|
+
if (store.hasThreadHistoryDatabase) {
|
|
2543
|
+
for (const databasePath of store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath]) {
|
|
2544
|
+
executeTransaction(databasePath, [
|
|
2545
|
+
...deleteStatements("thread_items", "thread_id", plan.ids),
|
|
2546
|
+
...deleteStatements("thread_turns", "thread_id", plan.ids),
|
|
2547
|
+
...deleteStatements("thread_history_projection_state", "thread_id", plan.ids),
|
|
2548
|
+
]);
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2470
2551
|
for (const { database, ids } of plan.stateTargets ?? (store.stateDatabases ?? [{ path: store.stateDatabasePath }]).map((database) => ({ database, ids: plan.ids }))) {
|
|
2471
2552
|
const stateStatements = [];
|
|
2472
2553
|
const schema = stateSchema(database);
|
|
@@ -2828,6 +2909,21 @@ export async function verifySessionDeletion({ plan, scope = "deep", store }) {
|
|
|
2828
2909
|
? (store.logsDatabasePaths ?? [store.logsDatabasePath]).flatMap((databasePath) =>
|
|
2829
2910
|
findRowsForIds(databasePath, "logs", "thread_id", plan.ids, { limitOne: true }))
|
|
2830
2911
|
: [];
|
|
2912
|
+
const remainingQueueRecords = store.hasQueueDatabase
|
|
2913
|
+
? (store.queueDatabasePaths ?? [store.queueDatabasePath]).flatMap((databasePath) => [
|
|
2914
|
+
...findRowsForIds(databasePath, "queued_items", "thread_id", plan.ids, { limitOne: true }),
|
|
2915
|
+
...(inspectSqliteTable(databasePath, "queued_thread_revisions").exists
|
|
2916
|
+
? findRowsForIds(databasePath, "queued_thread_revisions", "thread_id", plan.ids, { limitOne: true })
|
|
2917
|
+
: []),
|
|
2918
|
+
])
|
|
2919
|
+
: [];
|
|
2920
|
+
const remainingThreadHistoryRecords = store.hasThreadHistoryDatabase
|
|
2921
|
+
? (store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath]).flatMap((databasePath) => [
|
|
2922
|
+
...findRowsForIds(databasePath, "thread_items", "thread_id", plan.ids, { limitOne: true }),
|
|
2923
|
+
...findRowsForIds(databasePath, "thread_turns", "thread_id", plan.ids, { limitOne: true }),
|
|
2924
|
+
...findRowsForIds(databasePath, "thread_history_projection_state", "thread_id", plan.ids, { limitOne: true }),
|
|
2925
|
+
])
|
|
2926
|
+
: [];
|
|
2831
2927
|
const [sessionIndexMatches, historyMatches] = await Promise.all([
|
|
2832
2928
|
inspectJsonlMatches(
|
|
2833
2929
|
store.sessionIndexPath,
|
|
@@ -2872,6 +2968,8 @@ export async function verifySessionDeletion({ plan, scope = "deep", store }) {
|
|
|
2872
2968
|
remainingMemoryRecords.length === 0 &&
|
|
2873
2969
|
remainingGoalRecords.length === 0 &&
|
|
2874
2970
|
remainingLogRecords.length === 0 &&
|
|
2971
|
+
remainingQueueRecords.length === 0 &&
|
|
2972
|
+
remainingThreadHistoryRecords.length === 0 &&
|
|
2875
2973
|
sessionIndexMatches.count === 0 &&
|
|
2876
2974
|
historyMatches.count === 0 &&
|
|
2877
2975
|
remainingTranscriptPaths.length === 0 &&
|
|
@@ -2883,6 +2981,8 @@ export async function verifySessionDeletion({ plan, scope = "deep", store }) {
|
|
|
2883
2981
|
remainingHistoryEntryCount: historyMatches.count,
|
|
2884
2982
|
remainingLogRecords,
|
|
2885
2983
|
remainingMemoryRecords,
|
|
2984
|
+
remainingQueueRecords,
|
|
2985
|
+
remainingThreadHistoryRecords,
|
|
2886
2986
|
remainingSessionIndexEntries,
|
|
2887
2987
|
remainingSessionIndexEntryCount: sessionIndexMatches.count,
|
|
2888
2988
|
remainingThreads,
|