session-steward 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.10.2] - 2026-09-07
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- Codex cleanup now removes every rollout for a session, clears its memory jobs, and refreshes consolidated memory when needed.
|
|
8
|
+
|
|
3
9
|
## [0.10.1] - 2026-09-03
|
|
4
10
|
|
|
5
11
|
### Fixed
|
|
@@ -149,6 +155,7 @@
|
|
|
149
155
|
- Support for custom Codex home folders and a saved folder preference.
|
|
150
156
|
- Streaming and bounded-memory discovery for large session collections and transcripts.
|
|
151
157
|
|
|
158
|
+
[0.10.2]: https://github.com/mallikcheripally/session-steward/compare/v0.10.1...v0.10.2
|
|
152
159
|
[0.10.1]: https://github.com/mallikcheripally/session-steward/compare/v0.10.0...v0.10.1
|
|
153
160
|
[0.10.0]: https://github.com/mallikcheripally/session-steward/compare/v0.9.0...v0.10.0
|
|
154
161
|
[0.9.0]: https://github.com/mallikcheripally/session-steward/compare/v0.8.0...v0.9.0
|
|
@@ -30,7 +30,28 @@ const SCHEMA_REQUIREMENTS = Object.freeze({
|
|
|
30
30
|
fallback: "memories_1.sqlite",
|
|
31
31
|
pattern: /^memories_(\d+)\.sqlite$/u,
|
|
32
32
|
required: false,
|
|
33
|
-
tables: [
|
|
33
|
+
tables: [
|
|
34
|
+
{ name: "stage1_outputs", requiredColumns: ["thread_id"] },
|
|
35
|
+
{
|
|
36
|
+
name: "jobs",
|
|
37
|
+
optional: true,
|
|
38
|
+
requiredColumns: [
|
|
39
|
+
"kind",
|
|
40
|
+
"job_key",
|
|
41
|
+
"status",
|
|
42
|
+
"worker_id",
|
|
43
|
+
"ownership_token",
|
|
44
|
+
"started_at",
|
|
45
|
+
"finished_at",
|
|
46
|
+
"lease_until",
|
|
47
|
+
"retry_at",
|
|
48
|
+
"retry_remaining",
|
|
49
|
+
"last_error",
|
|
50
|
+
"input_watermark",
|
|
51
|
+
"last_success_watermark",
|
|
52
|
+
],
|
|
53
|
+
},
|
|
54
|
+
],
|
|
34
55
|
},
|
|
35
56
|
goals: {
|
|
36
57
|
fallback: "goals_1.sqlite",
|
|
@@ -4,6 +4,7 @@ import { constants as fsConstants, createReadStream } from "node:fs";
|
|
|
4
4
|
import { promises as fs } from "node:fs";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import readline from "node:readline";
|
|
7
|
+
import { createZstdDecompress } from "node:zlib";
|
|
7
8
|
|
|
8
9
|
import { measurePath } from "../../storage/files.mjs";
|
|
9
10
|
import {
|
|
@@ -142,7 +143,7 @@ async function* findTranscriptFiles(rootDirectory) {
|
|
|
142
143
|
continue;
|
|
143
144
|
}
|
|
144
145
|
|
|
145
|
-
if (entry.isFile() && /^rollout-.*\.jsonl
|
|
146
|
+
if (entry.isFile() && /^rollout-.*\.jsonl(?:\.zst)?$/u.test(entry.name)) {
|
|
146
147
|
yield resolvedPath;
|
|
147
148
|
}
|
|
148
149
|
}
|
|
@@ -163,13 +164,40 @@ function isContainedPath(rootDirectory, candidatePath) {
|
|
|
163
164
|
return relativePath === "" || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
|
|
164
165
|
}
|
|
165
166
|
|
|
167
|
+
function createTranscriptTextStream(filePath) {
|
|
168
|
+
const source = createReadStream(filePath);
|
|
169
|
+
|
|
170
|
+
if (!filePath.endsWith(".zst")) {
|
|
171
|
+
source.setEncoding("utf8");
|
|
172
|
+
return { input: source, source };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const input = createZstdDecompress();
|
|
176
|
+
input.setEncoding("utf8");
|
|
177
|
+
source.on("error", (error) => input.destroy(error));
|
|
178
|
+
source.pipe(input);
|
|
179
|
+
return { input, source };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function rolloutIdFromPath(filePath) {
|
|
183
|
+
return path.basename(filePath).match(
|
|
184
|
+
/([0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})\.jsonl(?:\.zst)?$/iu,
|
|
185
|
+
)?.[1]?.toLowerCase() ?? null;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function rolloutSiblingPath(filePath) {
|
|
189
|
+
return filePath.endsWith(".jsonl.zst")
|
|
190
|
+
? filePath.slice(0, -4)
|
|
191
|
+
: filePath.endsWith(".jsonl")
|
|
192
|
+
? `${filePath}.zst`
|
|
193
|
+
: null;
|
|
194
|
+
}
|
|
195
|
+
|
|
166
196
|
async function readFirstLine(filePath) {
|
|
167
|
-
const
|
|
168
|
-
encoding: "utf8",
|
|
169
|
-
});
|
|
197
|
+
const { input, source } = createTranscriptTextStream(filePath);
|
|
170
198
|
const interfaceHandle = readline.createInterface({
|
|
171
199
|
crlfDelay: Infinity,
|
|
172
|
-
input
|
|
200
|
+
input,
|
|
173
201
|
});
|
|
174
202
|
|
|
175
203
|
try {
|
|
@@ -178,7 +206,8 @@ async function readFirstLine(filePath) {
|
|
|
178
206
|
}
|
|
179
207
|
} finally {
|
|
180
208
|
interfaceHandle.close();
|
|
181
|
-
|
|
209
|
+
input.destroy();
|
|
210
|
+
source.destroy();
|
|
182
211
|
}
|
|
183
212
|
|
|
184
213
|
return "";
|
|
@@ -207,19 +236,21 @@ async function parseTranscriptHeader(filePath) {
|
|
|
207
236
|
cwd: payload.cwd ?? "",
|
|
208
237
|
filePath,
|
|
209
238
|
forkedFromId: payload.forked_from_id ?? null,
|
|
239
|
+
historyBaseId: payload.history_base?.thread_id
|
|
240
|
+
? String(payload.history_base.thread_id)
|
|
241
|
+
: null,
|
|
210
242
|
id: payload.id,
|
|
211
243
|
parentThreadId: subagentParentId,
|
|
244
|
+
rolloutId: rolloutIdFromPath(filePath),
|
|
212
245
|
timestampMs: toTimestampMs(payload.timestamp),
|
|
213
246
|
};
|
|
214
247
|
}
|
|
215
248
|
|
|
216
249
|
async function parseTranscriptFallback(filePath) {
|
|
217
|
-
const
|
|
218
|
-
encoding: "utf8",
|
|
219
|
-
});
|
|
250
|
+
const { input, source } = createTranscriptTextStream(filePath);
|
|
220
251
|
const interfaceHandle = readline.createInterface({
|
|
221
252
|
crlfDelay: Infinity,
|
|
222
|
-
input
|
|
253
|
+
input,
|
|
223
254
|
});
|
|
224
255
|
let firstUserMessage = "";
|
|
225
256
|
let latestThreadName = "";
|
|
@@ -261,7 +292,8 @@ async function parseTranscriptFallback(filePath) {
|
|
|
261
292
|
}
|
|
262
293
|
} finally {
|
|
263
294
|
interfaceHandle.close();
|
|
264
|
-
|
|
295
|
+
input.destroy();
|
|
296
|
+
source.destroy();
|
|
265
297
|
}
|
|
266
298
|
|
|
267
299
|
return {
|
|
@@ -603,7 +635,8 @@ export async function loadSessionStore({ codexHome }) {
|
|
|
603
635
|
}
|
|
604
636
|
const threadRows = [...threadRowsById.values()];
|
|
605
637
|
const spawnEdges = [...spawnEdgesByKey.values()];
|
|
606
|
-
const
|
|
638
|
+
const transcriptIndex = await indexTranscripts(paths.transcriptDirectories);
|
|
639
|
+
const transcriptHeaders = transcriptIndex.headersByThreadId;
|
|
607
640
|
|
|
608
641
|
const discoveryIds = new Set(threadRows.map((threadRow) => String(threadRow.id)));
|
|
609
642
|
|
|
@@ -801,6 +834,9 @@ export async function loadSessionStore({ codexHome }) {
|
|
|
801
834
|
threadHistoryDatabasePaths: paths.threadHistoryDatabasePaths,
|
|
802
835
|
goalsDatabasePaths: paths.goalsDatabasePaths,
|
|
803
836
|
transcriptHeaders,
|
|
837
|
+
transcriptPathsByThreadId: transcriptIndex.pathsByThreadId,
|
|
838
|
+
rolloutIdsByThreadId: transcriptIndex.rolloutIdsByThreadId,
|
|
839
|
+
rolloutReferencesById: transcriptIndex.rolloutReferencesById,
|
|
804
840
|
historyPath: paths.historyPath,
|
|
805
841
|
};
|
|
806
842
|
}
|
|
@@ -1257,17 +1293,40 @@ async function getStoreAvailability(paths) {
|
|
|
1257
1293
|
};
|
|
1258
1294
|
}
|
|
1259
1295
|
|
|
1260
|
-
async function
|
|
1261
|
-
const
|
|
1296
|
+
async function indexTranscripts(transcriptDirectories) {
|
|
1297
|
+
const headersByThreadId = new Map();
|
|
1298
|
+
const pathsByThreadId = new Map();
|
|
1299
|
+
const rolloutIdsByThreadId = new Map();
|
|
1300
|
+
const rolloutReferencesById = new Map();
|
|
1262
1301
|
|
|
1263
1302
|
for (const transcriptDirectory of transcriptDirectories) {
|
|
1264
1303
|
for await (const transcriptFile of findTranscriptFiles(transcriptDirectory)) {
|
|
1265
1304
|
try {
|
|
1266
1305
|
const header = await parseTranscriptHeader(transcriptFile);
|
|
1267
|
-
|
|
1306
|
+
if (!header?.id) continue;
|
|
1307
|
+
const threadId = String(header.id);
|
|
1308
|
+
const current = headersByThreadId.get(threadId);
|
|
1309
|
+
|
|
1310
|
+
if (!current || header.filePath > current.filePath) {
|
|
1311
|
+
headersByThreadId.set(threadId, { ...header, id: threadId });
|
|
1312
|
+
}
|
|
1268
1313
|
|
|
1269
|
-
|
|
1270
|
-
|
|
1314
|
+
const transcriptPaths = pathsByThreadId.get(threadId) ?? new Set();
|
|
1315
|
+
transcriptPaths.add(header.filePath);
|
|
1316
|
+
pathsByThreadId.set(threadId, transcriptPaths);
|
|
1317
|
+
|
|
1318
|
+
if (header.rolloutId) {
|
|
1319
|
+
const rolloutIds = rolloutIdsByThreadId.get(threadId) ?? new Set();
|
|
1320
|
+
rolloutIds.add(header.rolloutId);
|
|
1321
|
+
rolloutIdsByThreadId.set(threadId, rolloutIds);
|
|
1322
|
+
|
|
1323
|
+
const existingReference = rolloutReferencesById.get(header.rolloutId);
|
|
1324
|
+
if (!existingReference || (!existingReference.historyBaseId && header.historyBaseId)) {
|
|
1325
|
+
rolloutReferencesById.set(header.rolloutId, {
|
|
1326
|
+
historyBaseId: header.historyBaseId,
|
|
1327
|
+
threadId,
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1271
1330
|
}
|
|
1272
1331
|
} catch {
|
|
1273
1332
|
continue;
|
|
@@ -1275,7 +1334,12 @@ async function indexTranscriptHeaders(transcriptDirectories) {
|
|
|
1275
1334
|
}
|
|
1276
1335
|
}
|
|
1277
1336
|
|
|
1278
|
-
return
|
|
1337
|
+
return {
|
|
1338
|
+
headersByThreadId,
|
|
1339
|
+
pathsByThreadId,
|
|
1340
|
+
rolloutIdsByThreadId,
|
|
1341
|
+
rolloutReferencesById,
|
|
1342
|
+
};
|
|
1279
1343
|
}
|
|
1280
1344
|
|
|
1281
1345
|
function getTranscriptChildrenByParentId(transcriptHeaders) {
|
|
@@ -1322,10 +1386,11 @@ export async function loadDeletionStore({ codexHome, recordIds }) {
|
|
|
1322
1386
|
throw new Error("Select at least one session.");
|
|
1323
1387
|
}
|
|
1324
1388
|
|
|
1325
|
-
const [availability,
|
|
1389
|
+
const [availability, transcriptIndex] = await Promise.all([
|
|
1326
1390
|
getStoreAvailability(paths),
|
|
1327
|
-
|
|
1391
|
+
indexTranscripts(paths.transcriptDirectories),
|
|
1328
1392
|
]);
|
|
1393
|
+
const transcriptHeaders = transcriptIndex.headersByThreadId;
|
|
1329
1394
|
const transcriptChildrenByParentId = getTranscriptChildrenByParentId(transcriptHeaders);
|
|
1330
1395
|
const pendingIds = [...selectedIds];
|
|
1331
1396
|
|
|
@@ -1527,6 +1592,13 @@ export async function loadDeletionStore({ codexHome, recordIds }) {
|
|
|
1527
1592
|
transcriptHeaders: new Map(
|
|
1528
1593
|
[...transcriptHeaders].filter(([id]) => relevantIds.has(id)),
|
|
1529
1594
|
),
|
|
1595
|
+
transcriptPathsByThreadId: new Map(
|
|
1596
|
+
[...transcriptIndex.pathsByThreadId].filter(([id]) => relevantIds.has(id)),
|
|
1597
|
+
),
|
|
1598
|
+
rolloutIdsByThreadId: new Map(
|
|
1599
|
+
[...transcriptIndex.rolloutIdsByThreadId].filter(([id]) => relevantIds.has(id)),
|
|
1600
|
+
),
|
|
1601
|
+
rolloutReferencesById: transcriptIndex.rolloutReferencesById,
|
|
1530
1602
|
};
|
|
1531
1603
|
}
|
|
1532
1604
|
|
|
@@ -1915,6 +1987,11 @@ const THREAD_HISTORY_TABLES_WITH_REALTIME = Object.freeze([
|
|
|
1915
1987
|
"thread_turns",
|
|
1916
1988
|
"thread_history_projection_state",
|
|
1917
1989
|
]);
|
|
1990
|
+
const MEMORY_STAGE1_JOB_KIND = "memory_stage1";
|
|
1991
|
+
const MEMORY_GLOBAL_JOB_KIND = "memory_consolidate_global";
|
|
1992
|
+
const MEMORY_GLOBAL_JOB_KEY = "global";
|
|
1993
|
+
const MEMORY_JOB_RETRY_COUNT = 3;
|
|
1994
|
+
const MEMORY_DELETE_IDS_TABLE = "session_steward_memory_delete_ids";
|
|
1918
1995
|
|
|
1919
1996
|
function threadHistoryTables(databasePath) {
|
|
1920
1997
|
return inspectSqliteTable(databasePath, "thread_realtime_items").exists
|
|
@@ -1922,9 +1999,112 @@ function threadHistoryTables(databasePath) {
|
|
|
1922
1999
|
: REQUIRED_THREAD_HISTORY_TABLES;
|
|
1923
2000
|
}
|
|
1924
2001
|
|
|
2002
|
+
function memorySchema(databasePath) {
|
|
2003
|
+
const jobs = inspectSqliteTable(databasePath, "jobs");
|
|
2004
|
+
const stage1Outputs = inspectSqliteTable(databasePath, "stage1_outputs");
|
|
2005
|
+
|
|
2006
|
+
return {
|
|
2007
|
+
hasJobs: jobs.exists,
|
|
2008
|
+
hasSelectedForPhase2: stage1Outputs.columns?.includes("selected_for_phase2") ?? false,
|
|
2009
|
+
};
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
function countMemoryJobsForIds(databasePath, ids) {
|
|
2013
|
+
if (!memorySchema(databasePath).hasJobs) return 0;
|
|
2014
|
+
return [...batches(ids)].reduce((total, idBatch) => total + Number(queryRows(
|
|
2015
|
+
databasePath,
|
|
2016
|
+
`select count(*) as count from jobs where kind = ? and job_key in (${placeholders(idBatch)})`,
|
|
2017
|
+
[MEMORY_STAGE1_JOB_KIND, ...idBatch],
|
|
2018
|
+
)[0]?.count ?? 0), 0);
|
|
2019
|
+
}
|
|
2020
|
+
|
|
2021
|
+
function findMemoryJobsForIds(databasePath, ids) {
|
|
2022
|
+
if (!memorySchema(databasePath).hasJobs) return [];
|
|
2023
|
+
return [...batches(ids)].flatMap((idBatch) => queryRows(
|
|
2024
|
+
databasePath,
|
|
2025
|
+
`select * from jobs where kind = ? and job_key in (${placeholders(idBatch)})`,
|
|
2026
|
+
[MEMORY_STAGE1_JOB_KIND, ...idBatch],
|
|
2027
|
+
));
|
|
2028
|
+
}
|
|
2029
|
+
|
|
2030
|
+
function memoryConsolidationWatermark(databasePath) {
|
|
2031
|
+
if (!memorySchema(databasePath).hasJobs) return null;
|
|
2032
|
+
const row = queryRows(
|
|
2033
|
+
databasePath,
|
|
2034
|
+
"select input_watermark from jobs where kind = ? and job_key = ? limit 1",
|
|
2035
|
+
[MEMORY_GLOBAL_JOB_KIND, MEMORY_GLOBAL_JOB_KEY],
|
|
2036
|
+
)[0];
|
|
2037
|
+
return row ? Number(row.input_watermark ?? 0) : null;
|
|
2038
|
+
}
|
|
2039
|
+
|
|
2040
|
+
function needsMemoryConsolidation(databasePath, ids) {
|
|
2041
|
+
const schema = memorySchema(databasePath);
|
|
2042
|
+
if (!schema.hasJobs || !schema.hasSelectedForPhase2) return false;
|
|
2043
|
+
|
|
2044
|
+
return [...batches(ids)].some((idBatch) => Number(queryRows(
|
|
2045
|
+
databasePath,
|
|
2046
|
+
`select count(*) as count from stage1_outputs where selected_for_phase2 != 0 and thread_id in (${placeholders(idBatch)})`,
|
|
2047
|
+
idBatch,
|
|
2048
|
+
)[0]?.count ?? 0) > 0);
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
function memoryDeletionStatements(databasePath, ids, inputWatermark) {
|
|
2052
|
+
const schema = memorySchema(databasePath);
|
|
2053
|
+
const statements = [
|
|
2054
|
+
{ sql: `create temp table ${MEMORY_DELETE_IDS_TABLE} (thread_id text primary key) without rowid` },
|
|
2055
|
+
...[...batches(ids)].map((idBatch) => ({
|
|
2056
|
+
parameters: idBatch,
|
|
2057
|
+
sql: `insert into ${MEMORY_DELETE_IDS_TABLE} (thread_id) values ${idBatch.map(() => "(?)").join(", ")}`,
|
|
2058
|
+
})),
|
|
2059
|
+
];
|
|
2060
|
+
|
|
2061
|
+
if (schema.hasJobs && schema.hasSelectedForPhase2) {
|
|
2062
|
+
statements.push({
|
|
2063
|
+
parameters: [
|
|
2064
|
+
MEMORY_GLOBAL_JOB_KIND,
|
|
2065
|
+
MEMORY_GLOBAL_JOB_KEY,
|
|
2066
|
+
MEMORY_JOB_RETRY_COUNT,
|
|
2067
|
+
inputWatermark,
|
|
2068
|
+
],
|
|
2069
|
+
sql: `insert into jobs (
|
|
2070
|
+
kind, job_key, status, worker_id, ownership_token, started_at, finished_at,
|
|
2071
|
+
lease_until, retry_at, retry_remaining, last_error, input_watermark,
|
|
2072
|
+
last_success_watermark
|
|
2073
|
+
)
|
|
2074
|
+
select ?, ?, 'pending', null, null, null, null, null, null, ?, null, ?, 0
|
|
2075
|
+
where exists (
|
|
2076
|
+
select 1 from stage1_outputs
|
|
2077
|
+
where selected_for_phase2 != 0
|
|
2078
|
+
and thread_id in (select thread_id from ${MEMORY_DELETE_IDS_TABLE})
|
|
2079
|
+
)
|
|
2080
|
+
on conflict(kind, job_key) do update set
|
|
2081
|
+
status = case when jobs.status = 'running' then 'running' else 'pending' end,
|
|
2082
|
+
retry_at = case when jobs.status = 'running' then jobs.retry_at else null end,
|
|
2083
|
+
retry_remaining = max(jobs.retry_remaining, excluded.retry_remaining),
|
|
2084
|
+
input_watermark = case
|
|
2085
|
+
when excluded.input_watermark > coalesce(jobs.input_watermark, 0)
|
|
2086
|
+
then excluded.input_watermark
|
|
2087
|
+
else coalesce(jobs.input_watermark, 0) + 1
|
|
2088
|
+
end`,
|
|
2089
|
+
});
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
statements.push({
|
|
2093
|
+
sql: `delete from stage1_outputs where thread_id in (select thread_id from ${MEMORY_DELETE_IDS_TABLE})`,
|
|
2094
|
+
});
|
|
2095
|
+
if (schema.hasJobs) {
|
|
2096
|
+
statements.push({
|
|
2097
|
+
parameters: [MEMORY_STAGE1_JOB_KIND],
|
|
2098
|
+
sql: `delete from jobs where kind = ? and job_key in (select thread_id from ${MEMORY_DELETE_IDS_TABLE})`,
|
|
2099
|
+
});
|
|
2100
|
+
}
|
|
2101
|
+
statements.push({ sql: `drop table ${MEMORY_DELETE_IDS_TABLE}` });
|
|
2102
|
+
return statements;
|
|
2103
|
+
}
|
|
2104
|
+
|
|
1925
2105
|
export async function planSessionDeletion({ recordIds, store }) {
|
|
1926
2106
|
const idsToDelete = new Set();
|
|
1927
|
-
const pendingIds =
|
|
2107
|
+
const pendingIds = recordIds.map(String);
|
|
1928
2108
|
|
|
1929
2109
|
while (pendingIds.length > 0) {
|
|
1930
2110
|
const currentId = pendingIds.shift();
|
|
@@ -1945,9 +2125,55 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
1945
2125
|
.map((id) => store.recordsById.get(id))
|
|
1946
2126
|
.filter(Boolean)
|
|
1947
2127
|
.sort((left, right) => left.displayName.localeCompare(right.displayName));
|
|
1948
|
-
const
|
|
1949
|
-
|
|
1950
|
-
|
|
2128
|
+
const deletionRolloutIds = new Set();
|
|
2129
|
+
|
|
2130
|
+
for (const id of deletionIds) {
|
|
2131
|
+
for (const rolloutId of store.rolloutIdsByThreadId?.get(id) ?? []) {
|
|
2132
|
+
deletionRolloutIds.add(rolloutId);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
const threadHistoryIds = [...new Set([...deletionIds, ...deletionRolloutIds])];
|
|
2136
|
+
|
|
2137
|
+
const externalHistoryReferences = [];
|
|
2138
|
+
|
|
2139
|
+
for (const [rolloutId, reference] of store.rolloutReferencesById ?? []) {
|
|
2140
|
+
if (
|
|
2141
|
+
reference.historyBaseId &&
|
|
2142
|
+
deletionRolloutIds.has(reference.historyBaseId) &&
|
|
2143
|
+
!deletionRolloutIds.has(rolloutId)
|
|
2144
|
+
) {
|
|
2145
|
+
externalHistoryReferences.push({ rolloutId, ...reference });
|
|
2146
|
+
}
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
if (externalHistoryReferences.length > 0) {
|
|
2150
|
+
const error = new Error(
|
|
2151
|
+
"Another Codex session still depends on the selected session history. Select the related session too, or keep both.",
|
|
2152
|
+
);
|
|
2153
|
+
error.code = "CODEX_EXTERNAL_ROLLOUT_REFERENCE";
|
|
2154
|
+
error.references = externalHistoryReferences;
|
|
2155
|
+
throw error;
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
const transcriptPathSet = new Set(deletionIds.flatMap((id) => [
|
|
2159
|
+
...(store.transcriptPathsByThreadId?.get(id) ?? []),
|
|
2160
|
+
store.recordsById.get(id)?.rolloutPath,
|
|
2161
|
+
]).filter(Boolean));
|
|
2162
|
+
const transcriptDirectories = getCodexPaths(store.codexHome).transcriptDirectories;
|
|
2163
|
+
|
|
2164
|
+
for (const transcriptPath of [...transcriptPathSet]) {
|
|
2165
|
+
const siblingPath = rolloutSiblingPath(transcriptPath);
|
|
2166
|
+
if (
|
|
2167
|
+
siblingPath &&
|
|
2168
|
+
rolloutIdFromPath(transcriptPath) &&
|
|
2169
|
+
transcriptDirectories.some((directory) => isContainedPath(directory, transcriptPath)) &&
|
|
2170
|
+
await pathExists(siblingPath)
|
|
2171
|
+
) {
|
|
2172
|
+
transcriptPathSet.add(siblingPath);
|
|
2173
|
+
}
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2176
|
+
const transcriptPaths = [...transcriptPathSet].sort();
|
|
1951
2177
|
const missingTranscriptPaths = selectedRecords
|
|
1952
2178
|
.filter((record) => record.rolloutMissing)
|
|
1953
2179
|
.map((record) => record.rolloutPath)
|
|
@@ -2008,8 +2234,27 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
2008
2234
|
: 0), 0);
|
|
2009
2235
|
const logRowCount = (store.logsDatabasePaths ?? [store.logsDatabasePath].filter(Boolean))
|
|
2010
2236
|
.reduce((total, databasePath) => total + countRowsForIds(databasePath, "logs", "thread_id", deletionIds), 0);
|
|
2011
|
-
const
|
|
2012
|
-
|
|
2237
|
+
const memoryDatabasePaths = store.memoryDatabasePaths ?? [store.memoryDatabasePath].filter(Boolean);
|
|
2238
|
+
const memoryOutputRowCount = memoryDatabasePaths.reduce(
|
|
2239
|
+
(total, databasePath) => total + countRowsForIds(
|
|
2240
|
+
databasePath,
|
|
2241
|
+
"stage1_outputs",
|
|
2242
|
+
"thread_id",
|
|
2243
|
+
deletionIds,
|
|
2244
|
+
),
|
|
2245
|
+
0,
|
|
2246
|
+
);
|
|
2247
|
+
const memoryJobRowCount = memoryDatabasePaths.reduce(
|
|
2248
|
+
(total, databasePath) => total + countMemoryJobsForIds(databasePath, deletionIds),
|
|
2249
|
+
0,
|
|
2250
|
+
);
|
|
2251
|
+
const memoryConsolidations = memoryDatabasePaths
|
|
2252
|
+
.filter((databasePath) => needsMemoryConsolidation(databasePath, deletionIds))
|
|
2253
|
+
.map((databasePath) => ({
|
|
2254
|
+
databasePath,
|
|
2255
|
+
inputWatermark: memoryConsolidationWatermark(databasePath),
|
|
2256
|
+
}));
|
|
2257
|
+
const memoryRowCount = memoryOutputRowCount + memoryJobRowCount;
|
|
2013
2258
|
const goalRowCount = (store.goalsDatabasePaths ?? [store.goalsDatabasePath].filter(Boolean))
|
|
2014
2259
|
.reduce((total, databasePath) => total + countRowsForIds(databasePath, "thread_goals", "thread_id", deletionIds), 0);
|
|
2015
2260
|
const queueRowCount = (store.queueDatabasePaths ?? [store.queueDatabasePath].filter(Boolean))
|
|
@@ -2020,7 +2265,7 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
2020
2265
|
: 0), 0);
|
|
2021
2266
|
const threadHistoryRowCount = (store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath].filter(Boolean))
|
|
2022
2267
|
.reduce((total, databasePath) => total + threadHistoryTables(databasePath).reduce((databaseTotal, tableName) =>
|
|
2023
|
-
databaseTotal + countRowsForIds(databasePath, tableName, "thread_id",
|
|
2268
|
+
databaseTotal + countRowsForIds(databasePath, tableName, "thread_id", threadHistoryIds), 0), 0);
|
|
2024
2269
|
const stateTargets = (store.stateDatabases ?? [{ path: store.stateDatabasePath }]).map((database) => ({
|
|
2025
2270
|
database,
|
|
2026
2271
|
ids: findRowsForIds(database.path, "threads", "id", deletionIds).map((row) => String(row.id)),
|
|
@@ -2038,6 +2283,9 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
2038
2283
|
historyMatchCount: historyMatches.count,
|
|
2039
2284
|
ids: deletionIds,
|
|
2040
2285
|
logRowCount,
|
|
2286
|
+
memoryConsolidations,
|
|
2287
|
+
memoryJobRowCount,
|
|
2288
|
+
memoryOutputRowCount,
|
|
2041
2289
|
memoryRowCount,
|
|
2042
2290
|
queueRowCount,
|
|
2043
2291
|
queueRevisionRowCount,
|
|
@@ -2049,6 +2297,7 @@ export async function planSessionDeletion({ recordIds, store }) {
|
|
|
2049
2297
|
stateTargets,
|
|
2050
2298
|
transcriptBytes,
|
|
2051
2299
|
transcriptFileCount,
|
|
2300
|
+
threadHistoryIds,
|
|
2052
2301
|
threadHistoryRowCount,
|
|
2053
2302
|
transcriptPaths,
|
|
2054
2303
|
};
|
|
@@ -2164,12 +2413,31 @@ export async function fingerprintSessionDeletion({ plan, scope, store }) {
|
|
|
2164
2413
|
}
|
|
2165
2414
|
for (const databasePath of store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath].filter(Boolean)) {
|
|
2166
2415
|
for (const tableName of threadHistoryTables(databasePath)) {
|
|
2167
|
-
fingerprintRowsForIds(
|
|
2416
|
+
fingerprintRowsForIds(
|
|
2417
|
+
hash,
|
|
2418
|
+
databasePath,
|
|
2419
|
+
tableName,
|
|
2420
|
+
"thread_id",
|
|
2421
|
+
plan.threadHistoryIds ?? plan.ids,
|
|
2422
|
+
);
|
|
2168
2423
|
}
|
|
2169
2424
|
}
|
|
2170
2425
|
if (scope === "deep") {
|
|
2171
2426
|
for (const databasePath of store.memoryDatabasePaths ?? [store.memoryDatabasePath].filter(Boolean)) {
|
|
2172
2427
|
fingerprintRowsForIds(hash, databasePath, "stage1_outputs", "thread_id", plan.ids);
|
|
2428
|
+
if (memorySchema(databasePath).hasJobs) {
|
|
2429
|
+
const jobKeys = (plan.memoryConsolidations ?? [])
|
|
2430
|
+
.some((consolidation) => consolidation.databasePath === databasePath)
|
|
2431
|
+
? [...plan.ids, MEMORY_GLOBAL_JOB_KEY]
|
|
2432
|
+
: plan.ids;
|
|
2433
|
+
fingerprintRowsForIds(
|
|
2434
|
+
hash,
|
|
2435
|
+
databasePath,
|
|
2436
|
+
"jobs",
|
|
2437
|
+
"job_key",
|
|
2438
|
+
jobKeys,
|
|
2439
|
+
);
|
|
2440
|
+
}
|
|
2173
2441
|
}
|
|
2174
2442
|
for (const databasePath of store.goalsDatabasePaths ?? [store.goalsDatabasePath].filter(Boolean)) {
|
|
2175
2443
|
fingerprintRowsForIds(hash, databasePath, "thread_goals", "thread_id", plan.ids);
|
|
@@ -2675,7 +2943,10 @@ export async function executeSessionDeletion({
|
|
|
2675
2943
|
try {
|
|
2676
2944
|
if (scope === "deep" && store.hasMemoryDatabase) {
|
|
2677
2945
|
for (const databasePath of store.memoryDatabasePaths ?? [store.memoryDatabasePath]) {
|
|
2678
|
-
executeTransaction(
|
|
2946
|
+
executeTransaction(
|
|
2947
|
+
databasePath,
|
|
2948
|
+
memoryDeletionStatements(databasePath, plan.ids, Math.floor(Date.now() / 1_000)),
|
|
2949
|
+
);
|
|
2679
2950
|
}
|
|
2680
2951
|
}
|
|
2681
2952
|
if (scope === "deep" && store.hasGoalsDatabase) {
|
|
@@ -2705,7 +2976,7 @@ export async function executeSessionDeletion({
|
|
|
2705
2976
|
if (store.hasThreadHistoryDatabase) {
|
|
2706
2977
|
for (const databasePath of store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath]) {
|
|
2707
2978
|
executeTransaction(databasePath, threadHistoryTables(databasePath).flatMap((tableName) =>
|
|
2708
|
-
[...deleteStatements(tableName, "thread_id", plan.ids)]));
|
|
2979
|
+
[...deleteStatements(tableName, "thread_id", plan.threadHistoryIds ?? plan.ids)]));
|
|
2709
2980
|
}
|
|
2710
2981
|
}
|
|
2711
2982
|
for (const { database, ids } of plan.stateTargets ?? (store.stateDatabases ?? [{ path: store.stateDatabasePath }]).map((database) => ({ database, ids: plan.ids }))) {
|
|
@@ -3056,7 +3327,16 @@ export async function verifySessionDeletion({ plan, scope = "deep", store }) {
|
|
|
3056
3327
|
: []);
|
|
3057
3328
|
const remainingMemoryRecords = scope === "deep" && store.hasMemoryDatabase
|
|
3058
3329
|
? (store.memoryDatabasePaths ?? [store.memoryDatabasePath]).flatMap((databasePath) =>
|
|
3059
|
-
|
|
3330
|
+
[
|
|
3331
|
+
...findRowsForIds(databasePath, "stage1_outputs", "thread_id", plan.ids),
|
|
3332
|
+
...findMemoryJobsForIds(databasePath, plan.ids),
|
|
3333
|
+
])
|
|
3334
|
+
: [];
|
|
3335
|
+
const missingMemoryConsolidations = scope === "deep"
|
|
3336
|
+
? (plan.memoryConsolidations ?? []).filter(({ databasePath, inputWatermark }) => {
|
|
3337
|
+
const currentWatermark = memoryConsolidationWatermark(databasePath);
|
|
3338
|
+
return currentWatermark === null || currentWatermark <= (inputWatermark ?? 0);
|
|
3339
|
+
})
|
|
3060
3340
|
: [];
|
|
3061
3341
|
const remainingGoalRecords = scope === "deep" && store.hasGoalsDatabase
|
|
3062
3342
|
? (store.goalsDatabasePaths ?? [store.goalsDatabasePath]).flatMap((databasePath) =>
|
|
@@ -3080,7 +3360,13 @@ export async function verifySessionDeletion({ plan, scope = "deep", store }) {
|
|
|
3080
3360
|
const remainingThreadHistoryRecords = store.hasThreadHistoryDatabase
|
|
3081
3361
|
? (store.threadHistoryDatabasePaths ?? [store.threadHistoryDatabasePath]).flatMap((databasePath) =>
|
|
3082
3362
|
threadHistoryTables(databasePath).flatMap((tableName) =>
|
|
3083
|
-
findRowsForIds(
|
|
3363
|
+
findRowsForIds(
|
|
3364
|
+
databasePath,
|
|
3365
|
+
tableName,
|
|
3366
|
+
"thread_id",
|
|
3367
|
+
plan.threadHistoryIds ?? plan.ids,
|
|
3368
|
+
{ limitOne: true },
|
|
3369
|
+
)))
|
|
3084
3370
|
: [];
|
|
3085
3371
|
const [sessionIndexMatches, historyMatches] = await Promise.all([
|
|
3086
3372
|
inspectJsonlMatches(
|
|
@@ -3124,6 +3410,7 @@ export async function verifySessionDeletion({ plan, scope = "deep", store }) {
|
|
|
3124
3410
|
remainingThreads.length === 0 &&
|
|
3125
3411
|
remainingDynamicTools.length === 0 &&
|
|
3126
3412
|
remainingMemoryRecords.length === 0 &&
|
|
3413
|
+
missingMemoryConsolidations.length === 0 &&
|
|
3127
3414
|
remainingGoalRecords.length === 0 &&
|
|
3128
3415
|
remainingLogRecords.length === 0 &&
|
|
3129
3416
|
remainingQueueRecords.length === 0 &&
|
|
@@ -3138,6 +3425,7 @@ export async function verifySessionDeletion({ plan, scope = "deep", store }) {
|
|
|
3138
3425
|
remainingHistoryEntries,
|
|
3139
3426
|
remainingHistoryEntryCount: historyMatches.count,
|
|
3140
3427
|
remainingLogRecords,
|
|
3428
|
+
missingMemoryConsolidations,
|
|
3141
3429
|
remainingMemoryRecords,
|
|
3142
3430
|
remainingQueueRecords,
|
|
3143
3431
|
remainingThreadHistoryRecords,
|
package/package.json
CHANGED