tokenusage-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +22 -0
- package/README.md +94 -0
- package/THIRD_PARTY_NOTICES.md +140 -0
- package/bin/tokenusage.js +69 -0
- package/package.json +59 -0
- package/src/cli.js +134 -0
- package/src/commands/device-login.js +205 -0
- package/src/commands/diagnostics.js +38 -0
- package/src/commands/doctor.js +110 -0
- package/src/commands/init.js +2436 -0
- package/src/commands/sessions.js +29 -0
- package/src/commands/status.js +1747 -0
- package/src/commands/sync.js +7934 -0
- package/src/commands/uninstall.js +312 -0
- package/src/lib/account-usage-dedup.js +267 -0
- package/src/lib/antigravity-paths.js +48 -0
- package/src/lib/ark-agent-plan-limits.js +438 -0
- package/src/lib/ark-coding-plan-limits.js +537 -0
- package/src/lib/async-queue-reader.js +138 -0
- package/src/lib/atomic-json.js +27 -0
- package/src/lib/bearer-token.js +11 -0
- package/src/lib/categorizer-utils.js +232 -0
- package/src/lib/claude-categorizer.js +1343 -0
- package/src/lib/claude-config.js +279 -0
- package/src/lib/claude-user-identity.js +56 -0
- package/src/lib/cli-ui.js +179 -0
- package/src/lib/cloud-account.js +541 -0
- package/src/lib/cloud-device-token.js +141 -0
- package/src/lib/cloud-sync-prefs.js +14 -0
- package/src/lib/cloud-usage-reset.js +115 -0
- package/src/lib/codex-config.js +595 -0
- package/src/lib/codex-context-breakdown.js +806 -0
- package/src/lib/codex-model-attribution.js +128 -0
- package/src/lib/codex-rollout-parser.js +1471 -0
- package/src/lib/codex-service-tier.js +49 -0
- package/src/lib/codex-token-refresh.js +140 -0
- package/src/lib/codex-token-usage.js +195 -0
- package/src/lib/codex-usage-record.js +195 -0
- package/src/lib/command-runner.js +313 -0
- package/src/lib/commandcode-limits.js +317 -0
- package/src/lib/context-health.js +124 -0
- package/src/lib/cursor-config.js +605 -0
- package/src/lib/cursor-store.js +1918 -0
- package/src/lib/debug-flags.js +7 -0
- package/src/lib/devin-limits.js +249 -0
- package/src/lib/devin-usage.js +212 -0
- package/src/lib/diagnostics.js +383 -0
- package/src/lib/doctor.js +385 -0
- package/src/lib/dumate.js +95 -0
- package/src/lib/fs.js +455 -0
- package/src/lib/function-url.js +14 -0
- package/src/lib/gemini-config.js +291 -0
- package/src/lib/git-outcomes.js +243 -0
- package/src/lib/grok-context-breakdown.js +890 -0
- package/src/lib/grok-hook.js +360 -0
- package/src/lib/grok-limits.js +865 -0
- package/src/lib/grok-usage.js +98 -0
- package/src/lib/heatmap-compact.js +114 -0
- package/src/lib/ingest-sanitize.js +41 -0
- package/src/lib/init-flow.js +42 -0
- package/src/lib/install-resolver.js +170 -0
- package/src/lib/jsonl-lines.js +134 -0
- package/src/lib/kimi-model-repair.js +173 -0
- package/src/lib/kimi-profile.js +61 -0
- package/src/lib/kimi-session.js +216 -0
- package/src/lib/local-api.js +3588 -0
- package/src/lib/machine-id.js +238 -0
- package/src/lib/multi-install-parser.js +113 -0
- package/src/lib/notify-command.js +38 -0
- package/src/lib/omp-hook.js +589 -0
- package/src/lib/openclaw-cli.js +104 -0
- package/src/lib/openclaw-hook.js +375 -0
- package/src/lib/openclaw-session-plugin.js +560 -0
- package/src/lib/opencode-config.js +121 -0
- package/src/lib/opencode-go-limits.js +807 -0
- package/src/lib/outcomes-engine.js +253 -0
- package/src/lib/passive-mode.js +219 -0
- package/src/lib/pet-packages.js +722 -0
- package/src/lib/pricing/curated-overrides.json +822 -0
- package/src/lib/pricing/index.js +379 -0
- package/src/lib/pricing/litellm-fetcher.js +172 -0
- package/src/lib/pricing/matcher.js +456 -0
- package/src/lib/pricing/seed-snapshot.json +1 -0
- package/src/lib/progress.js +76 -0
- package/src/lib/project-usage-purge.js +117 -0
- package/src/lib/prompt.js +24 -0
- package/src/lib/provider-status.js +104 -0
- package/src/lib/proxy-env.js +489 -0
- package/src/lib/proxy-settings.js +163 -0
- package/src/lib/qoder-cn-cli-limits.js +119 -0
- package/src/lib/qoder-limits.js +880 -0
- package/src/lib/quiet-warnings.js +26 -0
- package/src/lib/rollout.js +25966 -0
- package/src/lib/runtime-config.js +259 -0
- package/src/lib/scan-roots.js +277 -0
- package/src/lib/sentry.js +231 -0
- package/src/lib/session-analytics.js +2972 -0
- package/src/lib/session-performance.js +237 -0
- package/src/lib/session-upload.js +379 -0
- package/src/lib/skill-usage.js +267 -0
- package/src/lib/skills-manager.js +1591 -0
- package/src/lib/source-metadata.js +48 -0
- package/src/lib/sqlite-reader.js +215 -0
- package/src/lib/star-cta.js +73 -0
- package/src/lib/subscription-manager.js +293 -0
- package/src/lib/subscriptions.js +508 -0
- package/src/lib/telemetry.js +193 -0
- package/src/lib/tracker-paths.js +16 -0
- package/src/lib/trae-cn-config.js +548 -0
- package/src/lib/trae-db.js +127 -0
- package/src/lib/trae-snapshot.js +230 -0
- package/src/lib/trae-sqlite.js +143 -0
- package/src/lib/trae-usage.js +121 -0
- package/src/lib/upload-queue.js +134 -0
- package/src/lib/upload-throttle.js +171 -0
- package/src/lib/usage-limits.js +4552 -0
- package/src/lib/wrapped-aggregator.js +236 -0
- package/src/lib/wsl-probe.js +281 -0
- package/src/lib/zcode-limits.js +1172 -0
- package/src/lib/zcode-reset-credits.js +66 -0
- package/test/cli-smoke.test.js +53 -0
- package/test/ingest-sanitize.test.js +162 -0
- package/test/quiet-warnings.test.js +53 -0
- package/test/session-upload.test.js +427 -0
|
@@ -0,0 +1,1918 @@
|
|
|
1
|
+
const crypto = require("node:crypto");
|
|
2
|
+
const fssync = require("node:fs");
|
|
3
|
+
const fs = require("node:fs/promises");
|
|
4
|
+
const path = require("node:path");
|
|
5
|
+
|
|
6
|
+
const {
|
|
7
|
+
ensureDir,
|
|
8
|
+
readJson,
|
|
9
|
+
readJsonStrict,
|
|
10
|
+
writeJson,
|
|
11
|
+
} = require("./fs");
|
|
12
|
+
|
|
13
|
+
const STORE_VERSION = 2;
|
|
14
|
+
const STORE_DIRNAME = "cursor-store-v2";
|
|
15
|
+
const MANIFEST_FILENAME = "manifest.json";
|
|
16
|
+
const GENERATIONS_DIRNAME = "generations";
|
|
17
|
+
const DEFERRED_CODEX_AUDIT_FILENAME = "deferred-codex-audit.json";
|
|
18
|
+
const SHARDED_CORE_FILENAME = "core-v3.json";
|
|
19
|
+
const OPENCODE_MESSAGES_DIRNAME = "opencode-messages";
|
|
20
|
+
const OPENCODE_FINGERPRINTS_DIRNAME = "opencode-fingerprints";
|
|
21
|
+
const DEFAULT_ACTIVATION_BYTES = 16 * 1024 * 1024;
|
|
22
|
+
const PROJECT_ABSENT_CONTEXT_RESCAN_MS = 24 * 60 * 60 * 1000;
|
|
23
|
+
const CURSOR_STORE_RETRY_CODE = "TOKENUSAGE_CURSOR_STORE_RETRY";
|
|
24
|
+
|
|
25
|
+
async function openCursorStore({
|
|
26
|
+
trackerDir,
|
|
27
|
+
cursorsPath = path.join(trackerDir, "cursors.json"),
|
|
28
|
+
activationBytes = DEFAULT_ACTIVATION_BYTES,
|
|
29
|
+
forceV2 = false,
|
|
30
|
+
codexRoots = [],
|
|
31
|
+
failureInjector = null,
|
|
32
|
+
} = {}) {
|
|
33
|
+
if (typeof trackerDir !== "string" || trackerDir.length === 0) {
|
|
34
|
+
throw new Error("trackerDir is required");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const storeRoot = path.join(trackerDir, STORE_DIRNAME);
|
|
38
|
+
const manifestPath = path.join(storeRoot, MANIFEST_FILENAME);
|
|
39
|
+
const normalizedCodexRoots = normalizeCodexRoots(codexRoots);
|
|
40
|
+
const legacyFingerprint = await fingerprintFile(cursorsPath);
|
|
41
|
+
let manifest = await readJson(manifestPath);
|
|
42
|
+
let migratedDuringOpen = false;
|
|
43
|
+
|
|
44
|
+
if (isManifest(manifest)) {
|
|
45
|
+
const legacyChanged = legacyFingerprint
|
|
46
|
+
? !sameFingerprint(legacyFingerprint, manifest.legacyFingerprint)
|
|
47
|
+
: false;
|
|
48
|
+
if (legacyChanged) {
|
|
49
|
+
manifest = await migrateLegacyCursorState({
|
|
50
|
+
cursorsPath,
|
|
51
|
+
storeRoot,
|
|
52
|
+
legacyFingerprint,
|
|
53
|
+
previousManifest: manifest,
|
|
54
|
+
codexRoots: normalizedCodexRoots,
|
|
55
|
+
failureInjector,
|
|
56
|
+
});
|
|
57
|
+
migratedDuringOpen = true;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const opened = await openManifestGeneration({
|
|
61
|
+
cursorsPath,
|
|
62
|
+
storeRoot,
|
|
63
|
+
manifest,
|
|
64
|
+
codexRoots: normalizedCodexRoots,
|
|
65
|
+
failureInjector,
|
|
66
|
+
});
|
|
67
|
+
if (opened) {
|
|
68
|
+
if (migratedDuringOpen) opened.requiresCommit = true;
|
|
69
|
+
return opened;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
throw cursorStoreCorruption(
|
|
73
|
+
`No valid cursor store generation in ${storeRoot}`,
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const legacySize = Number(legacyFingerprint?.size || 0);
|
|
78
|
+
const shouldActivate = forceV2 || (
|
|
79
|
+
Number.isFinite(activationBytes) &&
|
|
80
|
+
activationBytes >= 0 &&
|
|
81
|
+
legacySize >= activationBytes
|
|
82
|
+
);
|
|
83
|
+
if (!shouldActivate) {
|
|
84
|
+
const cursors = (await readJson(cursorsPath)) || defaultCursorState();
|
|
85
|
+
return new LegacyCursorStore({ cursorsPath, cursors });
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
manifest = await migrateLegacyCursorState({
|
|
89
|
+
cursorsPath,
|
|
90
|
+
storeRoot,
|
|
91
|
+
legacyFingerprint,
|
|
92
|
+
previousManifest: null,
|
|
93
|
+
codexRoots: normalizedCodexRoots,
|
|
94
|
+
failureInjector,
|
|
95
|
+
});
|
|
96
|
+
migratedDuringOpen = true;
|
|
97
|
+
const opened = await openManifestGeneration({
|
|
98
|
+
cursorsPath,
|
|
99
|
+
storeRoot,
|
|
100
|
+
manifest,
|
|
101
|
+
codexRoots: normalizedCodexRoots,
|
|
102
|
+
failureInjector,
|
|
103
|
+
});
|
|
104
|
+
if (!opened) throw new Error("Unable to open migrated cursor store");
|
|
105
|
+
if (migratedDuringOpen) opened.requiresCommit = true;
|
|
106
|
+
return opened;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function readCursorStateSummary({ trackerDir, cursorsPath } = {}) {
|
|
110
|
+
const resolvedCursorsPath = cursorsPath || path.join(trackerDir, "cursors.json");
|
|
111
|
+
const storeRoot = path.join(path.dirname(resolvedCursorsPath), STORE_DIRNAME);
|
|
112
|
+
const manifest = await readJson(path.join(storeRoot, MANIFEST_FILENAME));
|
|
113
|
+
if (isManifest(manifest)) {
|
|
114
|
+
const legacyFingerprint = await fingerprintFile(resolvedCursorsPath);
|
|
115
|
+
if (
|
|
116
|
+
legacyFingerprint &&
|
|
117
|
+
!sameFingerprint(legacyFingerprint, manifest.legacyFingerprint)
|
|
118
|
+
) {
|
|
119
|
+
return summarizeLegacyCursorState(
|
|
120
|
+
await readJson(resolvedCursorsPath),
|
|
121
|
+
{ legacyDrift: true },
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
const opened = await readGeneration({ storeRoot, generationId: manifest.current }) ||
|
|
125
|
+
await readGeneration({
|
|
126
|
+
storeRoot,
|
|
127
|
+
generationId: manifest.rollback || manifest.previous,
|
|
128
|
+
});
|
|
129
|
+
if (opened) {
|
|
130
|
+
return {
|
|
131
|
+
cursors: opened.core,
|
|
132
|
+
mode: "v2",
|
|
133
|
+
legacyDrift: false,
|
|
134
|
+
fileCount: Number(opened.metadata?.counts?.totalFiles || 0),
|
|
135
|
+
codexFileCount: Number(opened.metadata?.counts?.codexFiles || 0),
|
|
136
|
+
codexEventCount: Number(opened.metadata?.counts?.codexEvents || 0),
|
|
137
|
+
opencodeMessageCount: Number(opened.metadata?.counts?.opencodeMessages || 0),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return summarizeLegacyCursorState(await readJson(resolvedCursorsPath));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function summarizeLegacyCursorState(cursors, { legacyDrift = false } = {}) {
|
|
145
|
+
return {
|
|
146
|
+
cursors,
|
|
147
|
+
mode: "legacy",
|
|
148
|
+
legacyDrift,
|
|
149
|
+
fileCount:
|
|
150
|
+
cursors?.files && typeof cursors.files === "object"
|
|
151
|
+
? Object.keys(cursors.files).length
|
|
152
|
+
: null,
|
|
153
|
+
codexFileCount: null,
|
|
154
|
+
codexEventCount: Array.isArray(cursors?.codexHashes)
|
|
155
|
+
? cursors.codexHashes.length
|
|
156
|
+
: null,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
class LegacyCursorStore {
|
|
161
|
+
constructor({ cursorsPath, cursors }) {
|
|
162
|
+
this.mode = "legacy";
|
|
163
|
+
this.cursorsPath = cursorsPath;
|
|
164
|
+
this.cursors = cursors;
|
|
165
|
+
this.codexEventStore = null;
|
|
166
|
+
this.fileShardLoadCount = 0;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
get fileCount() {
|
|
170
|
+
return Object.keys(this.cursors?.files || {}).length;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
get currentCorePath() {
|
|
174
|
+
return this.cursorsPath;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async loadCodexFilesForPaths() {}
|
|
178
|
+
|
|
179
|
+
async materializeAllCodexState() {}
|
|
180
|
+
|
|
181
|
+
async loadOpencodeMessagesForBatch() {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async materializeAllOpencodeState() {}
|
|
186
|
+
|
|
187
|
+
async canSkipCodexDay() {
|
|
188
|
+
return false;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async readDeferredCodexAuditSyncs() {
|
|
192
|
+
return 0;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async writeDeferredCodexAuditSyncs() {}
|
|
196
|
+
|
|
197
|
+
async clearDeferredCodexAuditSyncs() {}
|
|
198
|
+
|
|
199
|
+
async commit(cursors = this.cursors) {
|
|
200
|
+
await writeJson(this.cursorsPath, cursors);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
class V2CursorStore {
|
|
205
|
+
constructor({
|
|
206
|
+
cursorsPath,
|
|
207
|
+
storeRoot,
|
|
208
|
+
manifest,
|
|
209
|
+
generation,
|
|
210
|
+
fallbackGenerationIds,
|
|
211
|
+
codexRoots,
|
|
212
|
+
failureInjector,
|
|
213
|
+
}) {
|
|
214
|
+
this.mode = "v2";
|
|
215
|
+
this.cursorsPath = cursorsPath;
|
|
216
|
+
this.storeRoot = storeRoot;
|
|
217
|
+
this.manifest = manifest;
|
|
218
|
+
this.generation = generation.metadata;
|
|
219
|
+
this.generationDir = generation.directory;
|
|
220
|
+
this.fallbackGenerationIds = Array.isArray(fallbackGenerationIds)
|
|
221
|
+
? [...fallbackGenerationIds]
|
|
222
|
+
: [];
|
|
223
|
+
this.codexRoots = codexRoots;
|
|
224
|
+
this.cursors = generation.core;
|
|
225
|
+
this.failureInjector = failureInjector;
|
|
226
|
+
this.deferredCodexAuditPath = path.join(
|
|
227
|
+
storeRoot,
|
|
228
|
+
DEFERRED_CODEX_AUDIT_FILENAME,
|
|
229
|
+
);
|
|
230
|
+
this.loadedFileShards = new Map();
|
|
231
|
+
this.loadedEventSets = new Map();
|
|
232
|
+
this.pendingEventKeys = new Map();
|
|
233
|
+
this.skipValidationCache = new Map();
|
|
234
|
+
this.loadedOpencodeMessageShards = new Map();
|
|
235
|
+
this.loadedOpencodeFingerprintShards = new Map();
|
|
236
|
+
this.opencodeFingerprintIndexes = new Map();
|
|
237
|
+
this.materializedOpencodeNamespaces = new Set();
|
|
238
|
+
this.fileShardLoadCount = 0;
|
|
239
|
+
this.opencodeMessageShardLoadCount = 0;
|
|
240
|
+
this.opencodeFingerprintShardLoadCount = 0;
|
|
241
|
+
this.materializedCodexHashes = false;
|
|
242
|
+
this.requiresCommit = generation.metadata.id !== manifest.current;
|
|
243
|
+
|
|
244
|
+
this.normalizeGenerationMetadata();
|
|
245
|
+
this.stageInlineOpencodeMessages();
|
|
246
|
+
|
|
247
|
+
if (!this.cursors.files || typeof this.cursors.files !== "object") {
|
|
248
|
+
this.cursors.files = {};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const self = this;
|
|
252
|
+
this.codexEventStore = {
|
|
253
|
+
get size() {
|
|
254
|
+
let pending = 0;
|
|
255
|
+
for (const values of self.pendingEventKeys.values()) pending += values.size;
|
|
256
|
+
return Number(self.generation?.counts?.codexEvents || 0) + pending;
|
|
257
|
+
},
|
|
258
|
+
has(key) {
|
|
259
|
+
return self.hasCodexEvent(key);
|
|
260
|
+
},
|
|
261
|
+
add(key) {
|
|
262
|
+
self.addCodexEvent(key);
|
|
263
|
+
},
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
get fileCount() {
|
|
268
|
+
return Number(this.generation?.counts?.totalFiles || 0);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
get currentCorePath() {
|
|
272
|
+
return path.join(this.generationDir, this.generation?.coreFile || "core.json");
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
normalizeGenerationMetadata() {
|
|
276
|
+
if (!this.generation.opencodeMessages) this.generation.opencodeMessages = {};
|
|
277
|
+
if (!this.generation.opencodeFingerprints) this.generation.opencodeFingerprints = {};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
stageInlineOpencodeMessages() {
|
|
281
|
+
const states = opencodeNamespaceStates(this.cursors.opencode);
|
|
282
|
+
const messageCount = Array.from(states.values()).reduce(
|
|
283
|
+
(sum, state) => sum + Object.keys(state.messages || {}).length,
|
|
284
|
+
0,
|
|
285
|
+
);
|
|
286
|
+
if (messageCount === 0) return;
|
|
287
|
+
if (
|
|
288
|
+
Object.keys(this.generation.opencodeMessages).length > 0 ||
|
|
289
|
+
Object.keys(this.generation.opencodeFingerprints).length > 0
|
|
290
|
+
) {
|
|
291
|
+
throw cursorStoreCorruption("OpenCode messages exist in both core and shards");
|
|
292
|
+
}
|
|
293
|
+
for (const namespace of states.keys()) {
|
|
294
|
+
this.materializedOpencodeNamespaces.add(namespace);
|
|
295
|
+
}
|
|
296
|
+
this.requiresCommit = true;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
async loadCodexFilesForPaths(paths, cursors = this.cursors) {
|
|
300
|
+
const shardKeys = new Set();
|
|
301
|
+
for (const entry of Array.isArray(paths) ? paths : []) {
|
|
302
|
+
const filePath = typeof entry === "string" ? entry : entry?.path;
|
|
303
|
+
if (!isCodexSessionCursorPathForRoots(filePath, this.codexRoots)) continue;
|
|
304
|
+
shardKeys.add(codexFileShardKey(filePath));
|
|
305
|
+
}
|
|
306
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
307
|
+
try {
|
|
308
|
+
for (const shardKey of shardKeys) {
|
|
309
|
+
await this.loadFileShard(shardKey, cursors);
|
|
310
|
+
}
|
|
311
|
+
return { restarted: attempt > 0 };
|
|
312
|
+
} catch (error) {
|
|
313
|
+
if (!isCursorStoreRetry(error) || attempt > 0) throw error;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return { restarted: false };
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
async loadFileShard(shardKey, cursors = this.cursors) {
|
|
320
|
+
try {
|
|
321
|
+
return await this.loadFileShardUnchecked(shardKey, cursors);
|
|
322
|
+
} catch (error) {
|
|
323
|
+
if (
|
|
324
|
+
error?.code !== "TOKENUSAGE_CURSOR_STORE_CORRUPT" ||
|
|
325
|
+
!this.activateFallbackGeneration(cursors)
|
|
326
|
+
) {
|
|
327
|
+
throw error;
|
|
328
|
+
}
|
|
329
|
+
throw cursorStoreRetry(error);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
async loadFileShardUnchecked(shardKey, cursors = this.cursors) {
|
|
334
|
+
if (this.loadedFileShards.has(shardKey)) {
|
|
335
|
+
mergeFileShardIntoCursors(this.loadedFileShards.get(shardKey).data, cursors);
|
|
336
|
+
return this.loadedFileShards.get(shardKey).data;
|
|
337
|
+
}
|
|
338
|
+
const metadata = this.generation?.codexFiles?.[shardKey] || null;
|
|
339
|
+
const filePath = metadata?.file
|
|
340
|
+
? path.join(this.generationDir, metadata.file)
|
|
341
|
+
: null;
|
|
342
|
+
let data = {};
|
|
343
|
+
if (filePath) {
|
|
344
|
+
let raw;
|
|
345
|
+
try {
|
|
346
|
+
raw = await fs.readFile(filePath, "utf8");
|
|
347
|
+
} catch (error) {
|
|
348
|
+
throw cursorStoreCorruption(
|
|
349
|
+
`Unable to read Codex cursor shard ${metadata.file}`,
|
|
350
|
+
error,
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
assertShardIntegrity(raw, metadata, `Codex cursor shard ${metadata.file}`);
|
|
354
|
+
try {
|
|
355
|
+
data = JSON.parse(raw);
|
|
356
|
+
} catch (error) {
|
|
357
|
+
throw cursorStoreCorruption(
|
|
358
|
+
`Invalid Codex cursor shard ${metadata.file}`,
|
|
359
|
+
error,
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
const normalized = data && typeof data === "object" && !Array.isArray(data)
|
|
364
|
+
? data
|
|
365
|
+
: {};
|
|
366
|
+
if (
|
|
367
|
+
filePath &&
|
|
368
|
+
(normalized !== data || Object.keys(normalized).length !== metadata.count)
|
|
369
|
+
) {
|
|
370
|
+
throw cursorStoreCorruption(`Invalid Codex cursor shard ${metadata.file}`);
|
|
371
|
+
}
|
|
372
|
+
this.loadedFileShards.set(shardKey, {
|
|
373
|
+
data: normalized,
|
|
374
|
+
originalSerialized: JSON.stringify(normalized),
|
|
375
|
+
});
|
|
376
|
+
this.fileShardLoadCount += 1;
|
|
377
|
+
mergeFileShardIntoCursors(normalized, cursors);
|
|
378
|
+
return normalized;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
async materializeAllCodexState(cursors = this.cursors) {
|
|
382
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
383
|
+
try {
|
|
384
|
+
for (const shardKey of Object.keys(this.generation?.codexFiles || {})) {
|
|
385
|
+
await this.loadFileShard(shardKey, cursors);
|
|
386
|
+
}
|
|
387
|
+
const hashes = [];
|
|
388
|
+
for (const shardKey of Object.keys(this.generation?.codexEvents || {})) {
|
|
389
|
+
const values = this.loadEventSet(shardKey, cursors);
|
|
390
|
+
for (const key of values) hashes.push(key);
|
|
391
|
+
}
|
|
392
|
+
for (const values of this.pendingEventKeys.values()) {
|
|
393
|
+
for (const key of values) hashes.push(key);
|
|
394
|
+
}
|
|
395
|
+
cursors.codexHashes = hashes;
|
|
396
|
+
this.materializedCodexHashes = true;
|
|
397
|
+
return { restarted: attempt > 0 };
|
|
398
|
+
} catch (error) {
|
|
399
|
+
if (!isCursorStoreRetry(error) || attempt > 0) throw error;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return { restarted: false };
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
async loadOpencodeMessagesForBatch({
|
|
406
|
+
namespace = "flat",
|
|
407
|
+
messageKeys = [],
|
|
408
|
+
fingerprints = [],
|
|
409
|
+
} = {}) {
|
|
410
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
411
|
+
try {
|
|
412
|
+
const loaded = await this.loadOpencodeMessagesForBatchUnchecked({
|
|
413
|
+
namespace,
|
|
414
|
+
messageKeys,
|
|
415
|
+
fingerprints,
|
|
416
|
+
});
|
|
417
|
+
if (loaded) loaded.restarted = attempt > 0;
|
|
418
|
+
return loaded;
|
|
419
|
+
} catch (error) {
|
|
420
|
+
if (
|
|
421
|
+
!isCursorStoreRetry(error) ||
|
|
422
|
+
normalizeOpencodeNamespace(namespace) !== "flat" ||
|
|
423
|
+
attempt > 0
|
|
424
|
+
) {
|
|
425
|
+
throw error;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return null;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
async loadOpencodeMessagesForBatchUnchecked({
|
|
433
|
+
namespace,
|
|
434
|
+
messageKeys,
|
|
435
|
+
fingerprints,
|
|
436
|
+
}) {
|
|
437
|
+
const normalizedNamespace = normalizeOpencodeNamespace(namespace);
|
|
438
|
+
const activeRoot = this.cursors.opencode;
|
|
439
|
+
const activeRootIsFlat = Boolean(
|
|
440
|
+
activeRoot &&
|
|
441
|
+
typeof activeRoot === "object" &&
|
|
442
|
+
activeRoot.native === undefined &&
|
|
443
|
+
activeRoot.wsl === undefined,
|
|
444
|
+
);
|
|
445
|
+
const state = normalizedNamespace !== "flat" && activeRootIsFlat
|
|
446
|
+
? activeRoot
|
|
447
|
+
: opencodeStateForNamespace(this.cursors, normalizedNamespace, true);
|
|
448
|
+
if (!state || this.materializedOpencodeNamespaces.has(normalizedNamespace)) return null;
|
|
449
|
+
const messages = state.messages && typeof state.messages === "object"
|
|
450
|
+
? state.messages
|
|
451
|
+
: (state.messages = {});
|
|
452
|
+
const managed = hasOpencodeNamespaceShards({
|
|
453
|
+
namespace: normalizedNamespace,
|
|
454
|
+
metadata: this.generation,
|
|
455
|
+
loadedMessages: this.loadedOpencodeMessageShards,
|
|
456
|
+
loadedFingerprints: this.loadedOpencodeFingerprintShards,
|
|
457
|
+
});
|
|
458
|
+
if (!managed && Object.keys(messages).length > 0) return null;
|
|
459
|
+
if (!managed && this.generation.coreFile !== SHARDED_CORE_FILENAME) return null;
|
|
460
|
+
|
|
461
|
+
const wantedMessageKeys = Array.from(new Set(
|
|
462
|
+
Array.from(messageKeys || []).filter((key) => typeof key === "string" && key.length > 0),
|
|
463
|
+
));
|
|
464
|
+
for (const shardKey of new Set(wantedMessageKeys.map((key) => (
|
|
465
|
+
opencodeMessageShardKey(key, normalizedNamespace)
|
|
466
|
+
)))) {
|
|
467
|
+
const alreadyLoaded = this.loadedOpencodeMessageShards.has(shardKey);
|
|
468
|
+
const data = await this.loadOpencodeMessageShard(shardKey);
|
|
469
|
+
if (!alreadyLoaded) Object.assign(messages, data);
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
const wantedFingerprints = new Set(
|
|
473
|
+
Array.from(fingerprints || []).filter((value) => (
|
|
474
|
+
typeof value === "string" && value.length > 0
|
|
475
|
+
)),
|
|
476
|
+
);
|
|
477
|
+
for (const messageKey of wantedMessageKeys) {
|
|
478
|
+
const previousFingerprint = messages[messageKey]?.fingerprint;
|
|
479
|
+
if (typeof previousFingerprint === "string" && previousFingerprint.length > 0) {
|
|
480
|
+
wantedFingerprints.add(previousFingerprint);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
for (const shardKey of new Set(Array.from(wantedFingerprints, (fingerprint) => (
|
|
484
|
+
opencodeFingerprintShardKey(fingerprint, normalizedNamespace)
|
|
485
|
+
)))) {
|
|
486
|
+
await this.loadOpencodeFingerprintShard(shardKey, normalizedNamespace);
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
if (!this.opencodeFingerprintIndexes.has(normalizedNamespace)) {
|
|
490
|
+
this.opencodeFingerprintIndexes.set(normalizedNamespace, new Map());
|
|
491
|
+
}
|
|
492
|
+
return {
|
|
493
|
+
fingerprintIndex: this.opencodeFingerprintIndexes.get(normalizedNamespace),
|
|
494
|
+
restarted: false,
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
async loadOpencodeMessageShard(shardKey) {
|
|
499
|
+
try {
|
|
500
|
+
return await this.loadOpencodeMessageShardUnchecked(shardKey);
|
|
501
|
+
} catch (error) {
|
|
502
|
+
if (
|
|
503
|
+
error?.code !== "TOKENUSAGE_CURSOR_STORE_CORRUPT" ||
|
|
504
|
+
!this.activateFallbackGeneration(this.cursors)
|
|
505
|
+
) {
|
|
506
|
+
throw error;
|
|
507
|
+
}
|
|
508
|
+
throw cursorStoreRetry(error);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
async loadOpencodeMessageShardUnchecked(shardKey) {
|
|
513
|
+
if (this.loadedOpencodeMessageShards.has(shardKey)) {
|
|
514
|
+
return this.loadedOpencodeMessageShards.get(shardKey).data;
|
|
515
|
+
}
|
|
516
|
+
const metadata = this.generation.opencodeMessages[shardKey] || null;
|
|
517
|
+
const data = await readJsonObjectShard({
|
|
518
|
+
generationDir: this.generationDir,
|
|
519
|
+
metadata,
|
|
520
|
+
label: `OpenCode message shard ${metadata?.file || shardKey}`,
|
|
521
|
+
});
|
|
522
|
+
this.loadedOpencodeMessageShards.set(shardKey, { data });
|
|
523
|
+
this.opencodeMessageShardLoadCount += 1;
|
|
524
|
+
return data;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
async loadOpencodeFingerprintShard(shardKey, namespace) {
|
|
528
|
+
try {
|
|
529
|
+
return await this.loadOpencodeFingerprintShardUnchecked(shardKey, namespace);
|
|
530
|
+
} catch (error) {
|
|
531
|
+
if (
|
|
532
|
+
error?.code !== "TOKENUSAGE_CURSOR_STORE_CORRUPT" ||
|
|
533
|
+
!this.activateFallbackGeneration(this.cursors)
|
|
534
|
+
) {
|
|
535
|
+
throw error;
|
|
536
|
+
}
|
|
537
|
+
throw cursorStoreRetry(error);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
async loadOpencodeFingerprintShardUnchecked(shardKey, namespace) {
|
|
542
|
+
const alreadyLoaded = this.loadedOpencodeFingerprintShards.has(shardKey);
|
|
543
|
+
if (!alreadyLoaded) {
|
|
544
|
+
const metadata = this.generation.opencodeFingerprints[shardKey] || null;
|
|
545
|
+
const data = await readJsonObjectShard({
|
|
546
|
+
generationDir: this.generationDir,
|
|
547
|
+
metadata,
|
|
548
|
+
label: `OpenCode fingerprint shard ${metadata?.file || shardKey}`,
|
|
549
|
+
validate: (value) => Array.isArray(value) && value.every((item) => (
|
|
550
|
+
typeof item === "string" && item.length > 0
|
|
551
|
+
)),
|
|
552
|
+
});
|
|
553
|
+
this.loadedOpencodeFingerprintShards.set(shardKey, { data });
|
|
554
|
+
this.opencodeFingerprintShardLoadCount += 1;
|
|
555
|
+
}
|
|
556
|
+
if (!this.opencodeFingerprintIndexes.has(namespace)) {
|
|
557
|
+
this.opencodeFingerprintIndexes.set(namespace, new Map());
|
|
558
|
+
}
|
|
559
|
+
const index = this.opencodeFingerprintIndexes.get(namespace);
|
|
560
|
+
if (!alreadyLoaded) {
|
|
561
|
+
for (const [fingerprint, owners] of Object.entries(
|
|
562
|
+
this.loadedOpencodeFingerprintShards.get(shardKey).data,
|
|
563
|
+
)) {
|
|
564
|
+
index.set(fingerprint, new Set(owners));
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
return index;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
async materializeAllOpencodeState() {
|
|
571
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
572
|
+
try {
|
|
573
|
+
const namespaces = opencodeNamespacesForStore({
|
|
574
|
+
state: this.cursors.opencode,
|
|
575
|
+
metadata: this.generation,
|
|
576
|
+
loadedMessages: this.loadedOpencodeMessageShards,
|
|
577
|
+
loadedFingerprints: this.loadedOpencodeFingerprintShards,
|
|
578
|
+
});
|
|
579
|
+
for (const namespace of namespaces) {
|
|
580
|
+
const state = opencodeStateForNamespace(this.cursors, namespace, true);
|
|
581
|
+
if (!state.messages || typeof state.messages !== "object") state.messages = {};
|
|
582
|
+
for (const shardKey of opencodeShardKeysForNamespace(
|
|
583
|
+
namespace,
|
|
584
|
+
this.generation.opencodeMessages,
|
|
585
|
+
this.loadedOpencodeMessageShards,
|
|
586
|
+
)) {
|
|
587
|
+
Object.assign(
|
|
588
|
+
state.messages,
|
|
589
|
+
await this.loadOpencodeMessageShard(shardKey),
|
|
590
|
+
);
|
|
591
|
+
}
|
|
592
|
+
for (const shardKey of opencodeShardKeysForNamespace(
|
|
593
|
+
namespace,
|
|
594
|
+
this.generation.opencodeFingerprints,
|
|
595
|
+
this.loadedOpencodeFingerprintShards,
|
|
596
|
+
)) {
|
|
597
|
+
await this.loadOpencodeFingerprintShard(shardKey, namespace);
|
|
598
|
+
}
|
|
599
|
+
this.materializedOpencodeNamespaces.add(namespace);
|
|
600
|
+
}
|
|
601
|
+
return { restarted: attempt > 0 };
|
|
602
|
+
} catch (error) {
|
|
603
|
+
if (!isCursorStoreRetry(error) || attempt > 0) throw error;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
return { restarted: false };
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
async canSkipCodexDay({
|
|
610
|
+
filePath,
|
|
611
|
+
dayInventoryCache,
|
|
612
|
+
nowMs = Date.now(),
|
|
613
|
+
} = {}) {
|
|
614
|
+
const shardKey = codexFileShardKey(filePath);
|
|
615
|
+
if (shardKey === "misc") return false;
|
|
616
|
+
const dayDir = codexDayDirectory(filePath);
|
|
617
|
+
if (!dayDir) return false;
|
|
618
|
+
const shardMetadata = this.generation?.codexFiles?.[shardKey];
|
|
619
|
+
const directoryMetadata = shardMetadata?.directories?.[dayDir];
|
|
620
|
+
const inventory = dayInventoryCache?.days?.[dayDir];
|
|
621
|
+
if (
|
|
622
|
+
!directoryMetadata?.complete ||
|
|
623
|
+
!inventory ||
|
|
624
|
+
typeof inventory.statKey !== "string" ||
|
|
625
|
+
inventory.statKey !== directoryMetadata.statKey ||
|
|
626
|
+
!Array.isArray(inventory.files) ||
|
|
627
|
+
inventory.files.length !== directoryMetadata.fileCount
|
|
628
|
+
) {
|
|
629
|
+
return false;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
if (this.skipValidationCache.has(shardKey)) {
|
|
633
|
+
return this.skipValidationCache.get(shardKey);
|
|
634
|
+
}
|
|
635
|
+
const valid = await validateProjectSummary(shardMetadata.projectSummary, nowMs);
|
|
636
|
+
this.skipValidationCache.set(shardKey, valid);
|
|
637
|
+
return valid;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
async readDeferredCodexAuditSyncs() {
|
|
641
|
+
const state = await readJson(this.deferredCodexAuditPath);
|
|
642
|
+
const count = Number(state?.syncsSinceFullScan || 0);
|
|
643
|
+
return Number.isSafeInteger(count) && count > 0 ? count : 0;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
async writeDeferredCodexAuditSyncs(count) {
|
|
647
|
+
const normalized = Math.max(0, Math.floor(Number(count) || 0));
|
|
648
|
+
if (normalized === 0) {
|
|
649
|
+
await this.clearDeferredCodexAuditSyncs();
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
await writeJson(this.deferredCodexAuditPath, {
|
|
653
|
+
version: 1,
|
|
654
|
+
syncsSinceFullScan: normalized,
|
|
655
|
+
updatedAt: new Date().toISOString(),
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
async clearDeferredCodexAuditSyncs() {
|
|
660
|
+
await fs.unlink(this.deferredCodexAuditPath).catch(() => {});
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
hasCodexEvent(key) {
|
|
664
|
+
const shardKey = codexEventShardKey(key);
|
|
665
|
+
if (this.pendingEventKeys.get(shardKey)?.has(key)) return true;
|
|
666
|
+
return this.loadEventSet(shardKey).has(key);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
addCodexEvent(key) {
|
|
670
|
+
if (typeof key !== "string" || key.length === 0) return;
|
|
671
|
+
const shardKey = codexEventShardKey(key);
|
|
672
|
+
const loaded = this.loadedEventSets.get(shardKey);
|
|
673
|
+
if (loaded?.has(key)) return;
|
|
674
|
+
if (!this.pendingEventKeys.has(shardKey)) {
|
|
675
|
+
this.pendingEventKeys.set(shardKey, new Set());
|
|
676
|
+
}
|
|
677
|
+
this.pendingEventKeys.get(shardKey).add(key);
|
|
678
|
+
if (loaded) loaded.add(key);
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
loadEventSet(shardKey, cursors = this.cursors) {
|
|
682
|
+
try {
|
|
683
|
+
return this.loadEventSetUnchecked(shardKey);
|
|
684
|
+
} catch (error) {
|
|
685
|
+
if (
|
|
686
|
+
error?.code !== "TOKENUSAGE_CURSOR_STORE_CORRUPT" ||
|
|
687
|
+
!this.activateFallbackGeneration(cursors)
|
|
688
|
+
) {
|
|
689
|
+
throw error;
|
|
690
|
+
}
|
|
691
|
+
throw cursorStoreRetry(error);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
loadEventSetUnchecked(shardKey) {
|
|
696
|
+
if (this.loadedEventSets.has(shardKey)) {
|
|
697
|
+
return this.loadedEventSets.get(shardKey);
|
|
698
|
+
}
|
|
699
|
+
const metadata = this.generation?.codexEvents?.[shardKey] || null;
|
|
700
|
+
const filePath = metadata?.file
|
|
701
|
+
? path.join(this.generationDir, metadata.file)
|
|
702
|
+
: null;
|
|
703
|
+
let raw = "";
|
|
704
|
+
if (filePath) {
|
|
705
|
+
try {
|
|
706
|
+
raw = fssync.readFileSync(filePath, "utf8");
|
|
707
|
+
} catch (e) {
|
|
708
|
+
throw cursorStoreCorruption(
|
|
709
|
+
`Unable to read Codex event shard ${metadata.file}`,
|
|
710
|
+
e,
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
assertShardIntegrity(raw, metadata, `Codex event shard ${metadata.file}`);
|
|
714
|
+
}
|
|
715
|
+
const rows = raw.split("\n").filter(Boolean);
|
|
716
|
+
const values = new Set(rows);
|
|
717
|
+
if (
|
|
718
|
+
filePath &&
|
|
719
|
+
(rows.length !== metadata.count || values.size !== metadata.count)
|
|
720
|
+
) {
|
|
721
|
+
throw cursorStoreCorruption(`Invalid Codex event shard ${metadata.file}`);
|
|
722
|
+
}
|
|
723
|
+
this.loadedEventSets.set(shardKey, values);
|
|
724
|
+
return values;
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
activateFallbackGeneration(cursors = this.cursors) {
|
|
728
|
+
let fallback = null;
|
|
729
|
+
while (!fallback && this.fallbackGenerationIds.length > 0) {
|
|
730
|
+
fallback = readGenerationSync({
|
|
731
|
+
storeRoot: this.storeRoot,
|
|
732
|
+
generationId: this.fallbackGenerationIds.shift(),
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
if (!fallback) return false;
|
|
736
|
+
|
|
737
|
+
const fallbackCore = cloneJson(fallback.core);
|
|
738
|
+
replaceObjectContents(this.cursors, fallbackCore);
|
|
739
|
+
if (cursors !== this.cursors) {
|
|
740
|
+
replaceObjectContents(cursors, cloneJson(fallback.core));
|
|
741
|
+
}
|
|
742
|
+
this.generation = fallback.metadata;
|
|
743
|
+
this.generationDir = fallback.directory;
|
|
744
|
+
this.loadedFileShards.clear();
|
|
745
|
+
this.loadedEventSets.clear();
|
|
746
|
+
this.pendingEventKeys.clear();
|
|
747
|
+
this.skipValidationCache.clear();
|
|
748
|
+
this.loadedOpencodeMessageShards.clear();
|
|
749
|
+
this.loadedOpencodeFingerprintShards.clear();
|
|
750
|
+
this.opencodeFingerprintIndexes.clear();
|
|
751
|
+
this.materializedOpencodeNamespaces.clear();
|
|
752
|
+
this.materializedCodexHashes = false;
|
|
753
|
+
this.requiresCommit = true;
|
|
754
|
+
this.normalizeGenerationMetadata();
|
|
755
|
+
this.stageInlineOpencodeMessages();
|
|
756
|
+
return true;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
async commitOpencodeShards({ cursors, generationDir, metadata }) {
|
|
760
|
+
const states = opencodeNamespaceStates(cursors.opencode);
|
|
761
|
+
const currentNamespaces = new Set(states.keys());
|
|
762
|
+
|
|
763
|
+
for (const shardMap of [metadata.opencodeMessages, metadata.opencodeFingerprints]) {
|
|
764
|
+
for (const [shardKey, entry] of Object.entries(shardMap)) {
|
|
765
|
+
if (currentNamespaces.has(opencodeShardNamespace(shardKey))) continue;
|
|
766
|
+
if (entry?.file) await fs.unlink(path.join(generationDir, entry.file)).catch(() => {});
|
|
767
|
+
delete shardMap[shardKey];
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
for (const [namespace, state] of states) {
|
|
772
|
+
const messages = state.messages && typeof state.messages === "object"
|
|
773
|
+
? state.messages
|
|
774
|
+
: {};
|
|
775
|
+
const managed = hasOpencodeNamespaceShards({
|
|
776
|
+
namespace,
|
|
777
|
+
metadata: this.generation,
|
|
778
|
+
loadedMessages: this.loadedOpencodeMessageShards,
|
|
779
|
+
loadedFingerprints: this.loadedOpencodeFingerprintShards,
|
|
780
|
+
});
|
|
781
|
+
const materialized = this.materializedOpencodeNamespaces.has(namespace) ||
|
|
782
|
+
(!managed && Object.keys(messages).length > 0);
|
|
783
|
+
|
|
784
|
+
if (materialized) {
|
|
785
|
+
await removeOpencodeNamespaceShards({ generationDir, metadata, namespace });
|
|
786
|
+
const partitioned = partitionOpencodeMessages(messages, namespace);
|
|
787
|
+
for (const [shardKey, data] of partitioned.messageShards) {
|
|
788
|
+
await writeOpencodeShard({
|
|
789
|
+
generationDir,
|
|
790
|
+
metadataMap: metadata.opencodeMessages,
|
|
791
|
+
directory: OPENCODE_MESSAGES_DIRNAME,
|
|
792
|
+
shardKey,
|
|
793
|
+
data,
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
for (const [shardKey, data] of partitioned.fingerprintShards) {
|
|
797
|
+
await writeOpencodeShard({
|
|
798
|
+
generationDir,
|
|
799
|
+
metadataMap: metadata.opencodeFingerprints,
|
|
800
|
+
directory: OPENCODE_FINGERPRINTS_DIRNAME,
|
|
801
|
+
shardKey,
|
|
802
|
+
data,
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
const dirtyMessageShards = new Set([
|
|
809
|
+
...opencodeShardKeysForNamespace(
|
|
810
|
+
namespace,
|
|
811
|
+
{},
|
|
812
|
+
this.loadedOpencodeMessageShards,
|
|
813
|
+
),
|
|
814
|
+
...Object.keys(messages).map((key) => opencodeMessageShardKey(key, namespace)),
|
|
815
|
+
]);
|
|
816
|
+
for (const shardKey of dirtyMessageShards) {
|
|
817
|
+
const data = {};
|
|
818
|
+
for (const [messageKey, entry] of Object.entries(messages)) {
|
|
819
|
+
if (opencodeMessageShardKey(messageKey, namespace) === shardKey) {
|
|
820
|
+
data[messageKey] = entry;
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
await writeOpencodeShard({
|
|
824
|
+
generationDir,
|
|
825
|
+
metadataMap: metadata.opencodeMessages,
|
|
826
|
+
directory: OPENCODE_MESSAGES_DIRNAME,
|
|
827
|
+
shardKey,
|
|
828
|
+
data,
|
|
829
|
+
});
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
const fingerprintIndex = this.opencodeFingerprintIndexes.get(namespace) || new Map();
|
|
833
|
+
for (const shardKey of opencodeShardKeysForNamespace(
|
|
834
|
+
namespace,
|
|
835
|
+
{},
|
|
836
|
+
this.loadedOpencodeFingerprintShards,
|
|
837
|
+
)) {
|
|
838
|
+
const data = {};
|
|
839
|
+
for (const [fingerprint, owners] of fingerprintIndex) {
|
|
840
|
+
if (opencodeFingerprintShardKey(fingerprint, namespace) === shardKey) {
|
|
841
|
+
data[fingerprint] = Array.from(owners);
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
await writeOpencodeShard({
|
|
845
|
+
generationDir,
|
|
846
|
+
metadataMap: metadata.opencodeFingerprints,
|
|
847
|
+
directory: OPENCODE_FINGERPRINTS_DIRNAME,
|
|
848
|
+
shardKey,
|
|
849
|
+
data,
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
async commit(cursors = this.cursors) {
|
|
856
|
+
const generationId = newGenerationId();
|
|
857
|
+
const generationDir = generationDirectory(this.storeRoot, generationId);
|
|
858
|
+
let published = false;
|
|
859
|
+
try {
|
|
860
|
+
await ensureDir(path.join(generationDir, "codex-files"));
|
|
861
|
+
await ensureDir(path.join(generationDir, "codex-events"));
|
|
862
|
+
await ensureDir(path.join(generationDir, OPENCODE_MESSAGES_DIRNAME));
|
|
863
|
+
await ensureDir(path.join(generationDir, OPENCODE_FINGERPRINTS_DIRNAME));
|
|
864
|
+
|
|
865
|
+
const nextMetadata = {
|
|
866
|
+
version: STORE_VERSION,
|
|
867
|
+
id: generationId,
|
|
868
|
+
createdAt: new Date().toISOString(),
|
|
869
|
+
coreFile: SHARDED_CORE_FILENAME,
|
|
870
|
+
codexFiles: cloneJson(this.generation?.codexFiles || {}),
|
|
871
|
+
codexEvents: cloneJson(this.generation?.codexEvents || {}),
|
|
872
|
+
opencodeMessages: cloneJson(this.generation?.opencodeMessages || {}),
|
|
873
|
+
opencodeFingerprints: cloneJson(this.generation?.opencodeFingerprints || {}),
|
|
874
|
+
counts: {},
|
|
875
|
+
};
|
|
876
|
+
|
|
877
|
+
await cloneGenerationFiles({
|
|
878
|
+
fromDirectory: this.generationDir,
|
|
879
|
+
toDirectory: generationDir,
|
|
880
|
+
metadata: nextMetadata,
|
|
881
|
+
});
|
|
882
|
+
|
|
883
|
+
const codexShardKeysPresent = new Set();
|
|
884
|
+
for (const filePath of Object.keys(cursors.files || {})) {
|
|
885
|
+
if (isCodexSessionCursorPathForRoots(filePath, this.codexRoots)) {
|
|
886
|
+
codexShardKeysPresent.add(codexFileShardKey(filePath));
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
for (const shardKey of codexShardKeysPresent) {
|
|
890
|
+
if (!this.loadedFileShards.has(shardKey) && nextMetadata.codexFiles[shardKey]) {
|
|
891
|
+
await this.loadFileShardUnchecked(shardKey, cursors);
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
const { coreFiles, codexFilesByShard } = partitionCursorFiles(
|
|
896
|
+
cursors.files,
|
|
897
|
+
this.codexRoots,
|
|
898
|
+
);
|
|
899
|
+
const dirtyFileShards = new Set([
|
|
900
|
+
...this.loadedFileShards.keys(),
|
|
901
|
+
...codexFilesByShard.keys(),
|
|
902
|
+
]);
|
|
903
|
+
for (const shardKey of dirtyFileShards) {
|
|
904
|
+
const data = codexFilesByShard.get(shardKey) || {};
|
|
905
|
+
const relativeFile = path.join("codex-files", `${safeShardFilename(shardKey)}.json`);
|
|
906
|
+
const targetPath = path.join(generationDir, relativeFile);
|
|
907
|
+
await fs.unlink(targetPath).catch(() => {});
|
|
908
|
+
if (Object.keys(data).length === 0) {
|
|
909
|
+
delete nextMetadata.codexFiles[shardKey];
|
|
910
|
+
continue;
|
|
911
|
+
}
|
|
912
|
+
const serialized = `${JSON.stringify(data)}\n`;
|
|
913
|
+
await fs.writeFile(targetPath, serialized, "utf8");
|
|
914
|
+
nextMetadata.codexFiles[shardKey] = buildCodexFileShardMetadata({
|
|
915
|
+
shardKey,
|
|
916
|
+
data,
|
|
917
|
+
relativeFile,
|
|
918
|
+
serialized,
|
|
919
|
+
dayInventoryCache: cursors.codexDayInventoryCache,
|
|
920
|
+
});
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
if (this.materializedCodexHashes || Array.isArray(cursors.codexHashes)) {
|
|
924
|
+
await rebuildEventShards({
|
|
925
|
+
generationDir,
|
|
926
|
+
metadata: nextMetadata,
|
|
927
|
+
hashes: Array.isArray(cursors.codexHashes) ? cursors.codexHashes : [],
|
|
928
|
+
});
|
|
929
|
+
} else {
|
|
930
|
+
for (const [shardKey, pending] of this.pendingEventKeys.entries()) {
|
|
931
|
+
if (pending.size === 0) continue;
|
|
932
|
+
const existing = this.loadEventSetUnchecked(shardKey);
|
|
933
|
+
for (const key of pending) existing.add(key);
|
|
934
|
+
const relativeFile = path.join("codex-events", `${safeShardFilename(shardKey)}.txt`);
|
|
935
|
+
const targetPath = path.join(generationDir, relativeFile);
|
|
936
|
+
await fs.unlink(targetPath).catch(() => {});
|
|
937
|
+
const serialized = serializeEventSet(existing);
|
|
938
|
+
await fs.writeFile(targetPath, serialized, "utf8");
|
|
939
|
+
nextMetadata.codexEvents[shardKey] = buildEventShardMetadata({
|
|
940
|
+
relativeFile,
|
|
941
|
+
values: existing,
|
|
942
|
+
serialized,
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
await this.commitOpencodeShards({
|
|
948
|
+
cursors,
|
|
949
|
+
generationDir,
|
|
950
|
+
metadata: nextMetadata,
|
|
951
|
+
});
|
|
952
|
+
|
|
953
|
+
const core = {
|
|
954
|
+
...cursors,
|
|
955
|
+
files: coreFiles,
|
|
956
|
+
opencode: cloneOpencodeCoreState(cursors.opencode),
|
|
957
|
+
};
|
|
958
|
+
delete core.codexHashes;
|
|
959
|
+
await fs.writeFile(
|
|
960
|
+
path.join(generationDir, nextMetadata.coreFile),
|
|
961
|
+
`${JSON.stringify(core)}\n`,
|
|
962
|
+
"utf8",
|
|
963
|
+
);
|
|
964
|
+
|
|
965
|
+
nextMetadata.counts = calculateGenerationCounts(nextMetadata, coreFiles);
|
|
966
|
+
await fs.writeFile(
|
|
967
|
+
path.join(generationDir, "generation.json"),
|
|
968
|
+
`${JSON.stringify(nextMetadata, null, 2)}\n`,
|
|
969
|
+
"utf8",
|
|
970
|
+
);
|
|
971
|
+
|
|
972
|
+
await maybeInjectFailure(this.failureInjector, "beforeManifestSwap");
|
|
973
|
+
|
|
974
|
+
const compatibleGeneration = this.generation?.coreFile === "core.json"
|
|
975
|
+
? this.generation.id
|
|
976
|
+
: this.manifest?.previous || null;
|
|
977
|
+
const nextManifest = {
|
|
978
|
+
version: STORE_VERSION,
|
|
979
|
+
current: generationId,
|
|
980
|
+
rollback: this.generation?.id || null,
|
|
981
|
+
// Older v2 readers reject core-v3.json and fall back to this frozen,
|
|
982
|
+
// fully-inline checkpoint instead of opening an empty message index.
|
|
983
|
+
previous: compatibleGeneration,
|
|
984
|
+
legacyFingerprint: this.manifest?.legacyFingerprint || null,
|
|
985
|
+
updatedAt: new Date().toISOString(),
|
|
986
|
+
};
|
|
987
|
+
await writeJson(path.join(this.storeRoot, MANIFEST_FILENAME), nextManifest);
|
|
988
|
+
published = true;
|
|
989
|
+
|
|
990
|
+
this.manifest = nextManifest;
|
|
991
|
+
this.generation = nextMetadata;
|
|
992
|
+
this.generationDir = generationDir;
|
|
993
|
+
this.cursors = core;
|
|
994
|
+
this.loadedFileShards.clear();
|
|
995
|
+
this.loadedEventSets.clear();
|
|
996
|
+
this.pendingEventKeys.clear();
|
|
997
|
+
this.skipValidationCache.clear();
|
|
998
|
+
this.loadedOpencodeMessageShards.clear();
|
|
999
|
+
this.loadedOpencodeFingerprintShards.clear();
|
|
1000
|
+
this.opencodeFingerprintIndexes.clear();
|
|
1001
|
+
this.materializedOpencodeNamespaces.clear();
|
|
1002
|
+
this.materializedCodexHashes = false;
|
|
1003
|
+
this.fallbackGenerationIds = [nextManifest.rollback].filter(Boolean);
|
|
1004
|
+
this.requiresCommit = false;
|
|
1005
|
+
await cleanupGenerations(this.storeRoot, new Set([
|
|
1006
|
+
nextManifest.current,
|
|
1007
|
+
nextManifest.rollback,
|
|
1008
|
+
nextManifest.previous,
|
|
1009
|
+
].filter(Boolean)));
|
|
1010
|
+
} catch (error) {
|
|
1011
|
+
if (!published) {
|
|
1012
|
+
await fs.rm(generationDir, { recursive: true, force: true }).catch(() => {});
|
|
1013
|
+
}
|
|
1014
|
+
throw error;
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
async function migrateLegacyCursorState({
|
|
1020
|
+
cursorsPath,
|
|
1021
|
+
storeRoot,
|
|
1022
|
+
legacyFingerprint,
|
|
1023
|
+
previousManifest,
|
|
1024
|
+
codexRoots,
|
|
1025
|
+
failureInjector,
|
|
1026
|
+
}) {
|
|
1027
|
+
const cursors = await readLegacyCursorStateForMigration(cursorsPath);
|
|
1028
|
+
const generationId = newGenerationId();
|
|
1029
|
+
const generationDir = generationDirectory(storeRoot, generationId);
|
|
1030
|
+
await ensureDir(path.join(generationDir, "codex-files"));
|
|
1031
|
+
await ensureDir(path.join(generationDir, "codex-events"));
|
|
1032
|
+
|
|
1033
|
+
const { coreFiles, codexFilesByShard } = partitionCursorFiles(
|
|
1034
|
+
cursors.files || {},
|
|
1035
|
+
codexRoots,
|
|
1036
|
+
);
|
|
1037
|
+
const eventShards = partitionEventHashes(cursors.codexHashes || []);
|
|
1038
|
+
const core = { ...cursors, files: coreFiles };
|
|
1039
|
+
delete core.codexHashes;
|
|
1040
|
+
|
|
1041
|
+
const metadata = {
|
|
1042
|
+
version: STORE_VERSION,
|
|
1043
|
+
id: generationId,
|
|
1044
|
+
createdAt: new Date().toISOString(),
|
|
1045
|
+
coreFile: "core.json",
|
|
1046
|
+
codexFiles: {},
|
|
1047
|
+
codexEvents: {},
|
|
1048
|
+
counts: {},
|
|
1049
|
+
};
|
|
1050
|
+
|
|
1051
|
+
for (const [shardKey, data] of codexFilesByShard.entries()) {
|
|
1052
|
+
const relativeFile = path.join("codex-files", `${safeShardFilename(shardKey)}.json`);
|
|
1053
|
+
const serialized = `${JSON.stringify(data)}\n`;
|
|
1054
|
+
await fs.writeFile(
|
|
1055
|
+
path.join(generationDir, relativeFile),
|
|
1056
|
+
serialized,
|
|
1057
|
+
"utf8",
|
|
1058
|
+
);
|
|
1059
|
+
metadata.codexFiles[shardKey] = buildCodexFileShardMetadata({
|
|
1060
|
+
shardKey,
|
|
1061
|
+
data,
|
|
1062
|
+
relativeFile,
|
|
1063
|
+
serialized,
|
|
1064
|
+
dayInventoryCache: cursors.codexDayInventoryCache,
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
for (const [shardKey, values] of eventShards.entries()) {
|
|
1069
|
+
const relativeFile = path.join("codex-events", `${safeShardFilename(shardKey)}.txt`);
|
|
1070
|
+
const serialized = serializeEventSet(values);
|
|
1071
|
+
await fs.writeFile(
|
|
1072
|
+
path.join(generationDir, relativeFile),
|
|
1073
|
+
serialized,
|
|
1074
|
+
"utf8",
|
|
1075
|
+
);
|
|
1076
|
+
metadata.codexEvents[shardKey] = buildEventShardMetadata({
|
|
1077
|
+
relativeFile,
|
|
1078
|
+
values,
|
|
1079
|
+
serialized,
|
|
1080
|
+
});
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
await fs.writeFile(
|
|
1084
|
+
path.join(generationDir, metadata.coreFile),
|
|
1085
|
+
`${JSON.stringify(core)}\n`,
|
|
1086
|
+
"utf8",
|
|
1087
|
+
);
|
|
1088
|
+
metadata.counts = calculateGenerationCounts(metadata, coreFiles);
|
|
1089
|
+
await fs.writeFile(
|
|
1090
|
+
path.join(generationDir, "generation.json"),
|
|
1091
|
+
`${JSON.stringify(metadata, null, 2)}\n`,
|
|
1092
|
+
"utf8",
|
|
1093
|
+
);
|
|
1094
|
+
|
|
1095
|
+
await maybeInjectFailure(failureInjector, "beforeMigrationManifestSwap");
|
|
1096
|
+
const manifest = {
|
|
1097
|
+
version: STORE_VERSION,
|
|
1098
|
+
current: generationId,
|
|
1099
|
+
previous: isManifest(previousManifest) ? previousManifest.current : null,
|
|
1100
|
+
legacyFingerprint: legacyFingerprint || null,
|
|
1101
|
+
updatedAt: new Date().toISOString(),
|
|
1102
|
+
};
|
|
1103
|
+
await writeJson(path.join(storeRoot, MANIFEST_FILENAME), manifest);
|
|
1104
|
+
await cleanupGenerations(storeRoot, new Set([
|
|
1105
|
+
manifest.current,
|
|
1106
|
+
manifest.previous,
|
|
1107
|
+
].filter(Boolean)));
|
|
1108
|
+
return manifest;
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
async function readLegacyCursorStateForMigration(cursorsPath) {
|
|
1112
|
+
const result = await readJsonStrict(cursorsPath);
|
|
1113
|
+
if (result.status === "missing") return defaultCursorState();
|
|
1114
|
+
if (result.status !== "ok") {
|
|
1115
|
+
throw cursorStoreCorruption(
|
|
1116
|
+
`Unable to read legacy cursor state ${cursorsPath}`,
|
|
1117
|
+
result.error,
|
|
1118
|
+
);
|
|
1119
|
+
}
|
|
1120
|
+
if (!isCursorStateRoot(result.value)) {
|
|
1121
|
+
throw cursorStoreCorruption(`Invalid legacy cursor state ${cursorsPath}`);
|
|
1122
|
+
}
|
|
1123
|
+
if (
|
|
1124
|
+
result.value.codexHashes !== undefined &&
|
|
1125
|
+
!Array.isArray(result.value.codexHashes)
|
|
1126
|
+
) {
|
|
1127
|
+
throw cursorStoreCorruption(`Invalid legacy Codex event state ${cursorsPath}`);
|
|
1128
|
+
}
|
|
1129
|
+
return result.value;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
async function openManifestGeneration({
|
|
1133
|
+
cursorsPath,
|
|
1134
|
+
storeRoot,
|
|
1135
|
+
manifest,
|
|
1136
|
+
codexRoots,
|
|
1137
|
+
failureInjector,
|
|
1138
|
+
}) {
|
|
1139
|
+
const generationIds = Array.from(new Set([
|
|
1140
|
+
manifest.current,
|
|
1141
|
+
manifest.rollback || manifest.previous,
|
|
1142
|
+
].filter(Boolean)));
|
|
1143
|
+
for (let index = 0; index < generationIds.length; index += 1) {
|
|
1144
|
+
const generationId = generationIds[index];
|
|
1145
|
+
const generation = await readGeneration({ storeRoot, generationId });
|
|
1146
|
+
if (!generation) continue;
|
|
1147
|
+
return new V2CursorStore({
|
|
1148
|
+
cursorsPath,
|
|
1149
|
+
storeRoot,
|
|
1150
|
+
manifest,
|
|
1151
|
+
generation,
|
|
1152
|
+
fallbackGenerationIds: generationIds.slice(index + 1),
|
|
1153
|
+
codexRoots,
|
|
1154
|
+
failureInjector,
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
1157
|
+
return null;
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
async function readGeneration({ storeRoot, generationId }) {
|
|
1161
|
+
if (typeof generationId !== "string" || generationId.length === 0) return null;
|
|
1162
|
+
const directory = generationDirectory(storeRoot, generationId);
|
|
1163
|
+
const metadata = await readJson(path.join(directory, "generation.json"));
|
|
1164
|
+
if (!isGenerationMetadata(metadata, generationId)) {
|
|
1165
|
+
return null;
|
|
1166
|
+
}
|
|
1167
|
+
const core = await readJson(path.join(directory, metadata.coreFile));
|
|
1168
|
+
if (!isCursorStateRoot(core)) return null;
|
|
1169
|
+
if (!(await generationReferencesExist(directory, metadata))) return null;
|
|
1170
|
+
const counts = calculateGenerationCounts(metadata, core.files);
|
|
1171
|
+
if (!sameGenerationCounts(counts, metadata.counts)) return null;
|
|
1172
|
+
return { directory, metadata, core };
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
function readGenerationSync({ storeRoot, generationId }) {
|
|
1176
|
+
if (typeof generationId !== "string" || generationId.length === 0) return null;
|
|
1177
|
+
const directory = generationDirectory(storeRoot, generationId);
|
|
1178
|
+
try {
|
|
1179
|
+
const metadata = JSON.parse(
|
|
1180
|
+
fssync.readFileSync(path.join(directory, "generation.json"), "utf8"),
|
|
1181
|
+
);
|
|
1182
|
+
if (!isGenerationMetadata(metadata, generationId)) return null;
|
|
1183
|
+
const core = JSON.parse(
|
|
1184
|
+
fssync.readFileSync(path.join(directory, metadata.coreFile), "utf8"),
|
|
1185
|
+
);
|
|
1186
|
+
if (!isCursorStateRoot(core)) return null;
|
|
1187
|
+
if (!generationReferencesExistSync(directory, metadata)) return null;
|
|
1188
|
+
const counts = calculateGenerationCounts(metadata, core.files);
|
|
1189
|
+
if (!sameGenerationCounts(counts, metadata.counts)) return null;
|
|
1190
|
+
return { directory, metadata, core };
|
|
1191
|
+
} catch (_error) {
|
|
1192
|
+
return null;
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
function isCursorStateRoot(value) {
|
|
1197
|
+
return Boolean(
|
|
1198
|
+
value &&
|
|
1199
|
+
typeof value === "object" &&
|
|
1200
|
+
!Array.isArray(value) &&
|
|
1201
|
+
value.files &&
|
|
1202
|
+
typeof value.files === "object" &&
|
|
1203
|
+
!Array.isArray(value.files)
|
|
1204
|
+
);
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
function isGenerationMetadata(metadata, generationId) {
|
|
1208
|
+
const shardedOpencode = metadata?.coreFile === SHARDED_CORE_FILENAME;
|
|
1209
|
+
if (
|
|
1210
|
+
!metadata ||
|
|
1211
|
+
typeof metadata !== "object" ||
|
|
1212
|
+
Array.isArray(metadata) ||
|
|
1213
|
+
metadata.version !== STORE_VERSION ||
|
|
1214
|
+
metadata.id !== generationId ||
|
|
1215
|
+
(metadata.coreFile !== "core.json" && !shardedOpencode) ||
|
|
1216
|
+
!metadata.codexFiles ||
|
|
1217
|
+
typeof metadata.codexFiles !== "object" ||
|
|
1218
|
+
Array.isArray(metadata.codexFiles) ||
|
|
1219
|
+
!metadata.codexEvents ||
|
|
1220
|
+
typeof metadata.codexEvents !== "object" ||
|
|
1221
|
+
Array.isArray(metadata.codexEvents) ||
|
|
1222
|
+
!metadata.counts ||
|
|
1223
|
+
typeof metadata.counts !== "object" ||
|
|
1224
|
+
Array.isArray(metadata.counts)
|
|
1225
|
+
) {
|
|
1226
|
+
return false;
|
|
1227
|
+
}
|
|
1228
|
+
if (
|
|
1229
|
+
shardedOpencode &&
|
|
1230
|
+
(!metadata.opencodeMessages ||
|
|
1231
|
+
typeof metadata.opencodeMessages !== "object" ||
|
|
1232
|
+
Array.isArray(metadata.opencodeMessages) ||
|
|
1233
|
+
!metadata.opencodeFingerprints ||
|
|
1234
|
+
typeof metadata.opencodeFingerprints !== "object" ||
|
|
1235
|
+
Array.isArray(metadata.opencodeFingerprints))
|
|
1236
|
+
) {
|
|
1237
|
+
return false;
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
return (
|
|
1241
|
+
validShardMetadataMap(metadata.codexFiles, "codex-files") &&
|
|
1242
|
+
validShardMetadataMap(metadata.codexEvents, "codex-events") &&
|
|
1243
|
+
(!shardedOpencode || (
|
|
1244
|
+
validShardMetadataMap(metadata.opencodeMessages, OPENCODE_MESSAGES_DIRNAME) &&
|
|
1245
|
+
validShardMetadataMap(metadata.opencodeFingerprints, OPENCODE_FINGERPRINTS_DIRNAME)
|
|
1246
|
+
))
|
|
1247
|
+
);
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
function validShardMetadataMap(shards, expectedDirectory) {
|
|
1251
|
+
const files = new Set();
|
|
1252
|
+
for (const entry of Object.values(shards)) {
|
|
1253
|
+
const reference = normalizeShardReference(entry?.file, expectedDirectory);
|
|
1254
|
+
const count = Number(entry?.count);
|
|
1255
|
+
const bytes = Number(entry?.bytes);
|
|
1256
|
+
if (
|
|
1257
|
+
!reference ||
|
|
1258
|
+
files.has(reference.name) ||
|
|
1259
|
+
!Number.isSafeInteger(count) ||
|
|
1260
|
+
count < 0 ||
|
|
1261
|
+
!Number.isSafeInteger(bytes) ||
|
|
1262
|
+
bytes < 0 ||
|
|
1263
|
+
typeof entry?.sha256 !== "string" ||
|
|
1264
|
+
!/^[a-f0-9]{64}$/.test(entry.sha256)
|
|
1265
|
+
) {
|
|
1266
|
+
return false;
|
|
1267
|
+
}
|
|
1268
|
+
files.add(reference.name);
|
|
1269
|
+
}
|
|
1270
|
+
return true;
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
async function generationReferencesExist(directory, metadata) {
|
|
1274
|
+
for (const [expectedDirectory, shards] of generationShardMaps(metadata)) {
|
|
1275
|
+
const expectedNames = Object.values(shards)
|
|
1276
|
+
.map((entry) => normalizeShardReference(entry.file, expectedDirectory).name);
|
|
1277
|
+
if (expectedNames.length === 0) continue;
|
|
1278
|
+
|
|
1279
|
+
let entries;
|
|
1280
|
+
try {
|
|
1281
|
+
entries = await fs.readdir(path.join(directory, expectedDirectory), {
|
|
1282
|
+
withFileTypes: true,
|
|
1283
|
+
});
|
|
1284
|
+
} catch (_e) {
|
|
1285
|
+
return false;
|
|
1286
|
+
}
|
|
1287
|
+
const files = new Map(entries.map((entry) => [entry.name, entry]));
|
|
1288
|
+
if (expectedNames.some((name) => !files.get(name)?.isFile())) return false;
|
|
1289
|
+
}
|
|
1290
|
+
return true;
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1293
|
+
function generationReferencesExistSync(directory, metadata) {
|
|
1294
|
+
for (const [expectedDirectory, shards] of generationShardMaps(metadata)) {
|
|
1295
|
+
const expectedNames = Object.values(shards)
|
|
1296
|
+
.map((entry) => normalizeShardReference(entry.file, expectedDirectory).name);
|
|
1297
|
+
if (expectedNames.length === 0) continue;
|
|
1298
|
+
|
|
1299
|
+
let entries;
|
|
1300
|
+
try {
|
|
1301
|
+
entries = fssync.readdirSync(path.join(directory, expectedDirectory), {
|
|
1302
|
+
withFileTypes: true,
|
|
1303
|
+
});
|
|
1304
|
+
} catch (_error) {
|
|
1305
|
+
return false;
|
|
1306
|
+
}
|
|
1307
|
+
const files = new Map(entries.map((entry) => [entry.name, entry]));
|
|
1308
|
+
if (expectedNames.some((name) => !files.get(name)?.isFile())) return false;
|
|
1309
|
+
}
|
|
1310
|
+
return true;
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
function normalizeShardReference(relativeFile, expectedDirectory) {
|
|
1314
|
+
if (typeof relativeFile !== "string" || relativeFile.length === 0) return null;
|
|
1315
|
+
const normalized = relativeFile.replace(/\\/g, "/");
|
|
1316
|
+
const parts = normalized.split("/");
|
|
1317
|
+
if (
|
|
1318
|
+
parts.length !== 2 ||
|
|
1319
|
+
parts[0] !== expectedDirectory ||
|
|
1320
|
+
!parts[1] ||
|
|
1321
|
+
parts[1] === "." ||
|
|
1322
|
+
parts[1] === ".."
|
|
1323
|
+
) {
|
|
1324
|
+
return null;
|
|
1325
|
+
}
|
|
1326
|
+
return { directory: expectedDirectory, name: parts[1] };
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
function generationShardMaps(metadata) {
|
|
1330
|
+
return [
|
|
1331
|
+
["codex-files", metadata.codexFiles || {}],
|
|
1332
|
+
["codex-events", metadata.codexEvents || {}],
|
|
1333
|
+
[OPENCODE_MESSAGES_DIRNAME, metadata.opencodeMessages || {}],
|
|
1334
|
+
[OPENCODE_FINGERPRINTS_DIRNAME, metadata.opencodeFingerprints || {}],
|
|
1335
|
+
];
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
function sameGenerationCounts(left, right) {
|
|
1339
|
+
return [
|
|
1340
|
+
"nonCodexFiles",
|
|
1341
|
+
"codexFiles",
|
|
1342
|
+
"totalFiles",
|
|
1343
|
+
"codexEvents",
|
|
1344
|
+
"opencodeMessages",
|
|
1345
|
+
"opencodeFingerprints",
|
|
1346
|
+
].every((key) => Number(left?.[key] || 0) === Number(right?.[key] || 0));
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
async function cloneGenerationFiles({ fromDirectory, toDirectory, metadata }) {
|
|
1350
|
+
for (const [, shards] of generationShardMaps(metadata)) {
|
|
1351
|
+
for (const entry of Object.values(shards)) {
|
|
1352
|
+
if (entry?.file) await cloneOrCopy(fromDirectory, toDirectory, entry.file);
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
async function cloneOrCopy(fromDirectory, toDirectory, relativeFile) {
|
|
1358
|
+
const source = path.join(fromDirectory, relativeFile);
|
|
1359
|
+
const target = path.join(toDirectory, relativeFile);
|
|
1360
|
+
await ensureDir(path.dirname(target));
|
|
1361
|
+
await fs.copyFile(source, target, fssync.constants.COPYFILE_FICLONE);
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
async function rebuildEventShards({ generationDir, metadata, hashes }) {
|
|
1365
|
+
for (const entry of Object.values(metadata.codexEvents || {})) {
|
|
1366
|
+
if (entry?.file) await fs.unlink(path.join(generationDir, entry.file)).catch(() => {});
|
|
1367
|
+
}
|
|
1368
|
+
metadata.codexEvents = {};
|
|
1369
|
+
for (const [shardKey, values] of partitionEventHashes(hashes).entries()) {
|
|
1370
|
+
const relativeFile = path.join("codex-events", `${safeShardFilename(shardKey)}.txt`);
|
|
1371
|
+
const serialized = serializeEventSet(values);
|
|
1372
|
+
await fs.writeFile(
|
|
1373
|
+
path.join(generationDir, relativeFile),
|
|
1374
|
+
serialized,
|
|
1375
|
+
"utf8",
|
|
1376
|
+
);
|
|
1377
|
+
metadata.codexEvents[shardKey] = buildEventShardMetadata({
|
|
1378
|
+
relativeFile,
|
|
1379
|
+
values,
|
|
1380
|
+
serialized,
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
function buildCodexFileShardMetadata({
|
|
1386
|
+
shardKey,
|
|
1387
|
+
data,
|
|
1388
|
+
relativeFile,
|
|
1389
|
+
serialized,
|
|
1390
|
+
dayInventoryCache,
|
|
1391
|
+
}) {
|
|
1392
|
+
const directories = {};
|
|
1393
|
+
const configs = new Map();
|
|
1394
|
+
let absentFreshUntil = null;
|
|
1395
|
+
|
|
1396
|
+
for (const [filePath, cursor] of Object.entries(data || {})) {
|
|
1397
|
+
const dayDir = codexDayDirectory(filePath);
|
|
1398
|
+
if (dayDir) {
|
|
1399
|
+
if (!directories[dayDir]) directories[dayDir] = { paths: [] };
|
|
1400
|
+
directories[dayDir].paths.push(filePath);
|
|
1401
|
+
}
|
|
1402
|
+
const context = cursor?.projectFileContext;
|
|
1403
|
+
if (context?.absent === true) {
|
|
1404
|
+
const checkedAtMs = Number(context.checkedAtMs);
|
|
1405
|
+
if (Number.isFinite(checkedAtMs) && checkedAtMs > 0) {
|
|
1406
|
+
const expiresAt = checkedAtMs + PROJECT_ABSENT_CONTEXT_RESCAN_MS;
|
|
1407
|
+
absentFreshUntil = absentFreshUntil == null
|
|
1408
|
+
? expiresAt
|
|
1409
|
+
: Math.min(absentFreshUntil, expiresAt);
|
|
1410
|
+
} else {
|
|
1411
|
+
absentFreshUntil = 0;
|
|
1412
|
+
}
|
|
1413
|
+
continue;
|
|
1414
|
+
}
|
|
1415
|
+
for (const config of projectConfigs(context)) {
|
|
1416
|
+
configs.set(config.configPath, config);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
for (const [dayDir, value] of Object.entries(directories)) {
|
|
1421
|
+
const inventory = dayInventoryCache?.days?.[dayDir];
|
|
1422
|
+
const inventoryFiles = Array.isArray(inventory?.files) ? inventory.files : [];
|
|
1423
|
+
const complete = Boolean(
|
|
1424
|
+
typeof inventory?.statKey === "string" &&
|
|
1425
|
+
inventoryFiles.length > 0 &&
|
|
1426
|
+
inventoryFiles.every((name) => {
|
|
1427
|
+
const filePath = path.join(dayDir, name);
|
|
1428
|
+
const cursor = data[filePath];
|
|
1429
|
+
const offset = Number(cursor?.offset);
|
|
1430
|
+
const projectOffset = Number(cursor?.projectOffset);
|
|
1431
|
+
return (
|
|
1432
|
+
Number.isFinite(offset) && offset > 0 &&
|
|
1433
|
+
Number.isFinite(projectOffset) && projectOffset >= offset &&
|
|
1434
|
+
cursor?.projectFileContext && typeof cursor.projectFileContext === "object"
|
|
1435
|
+
);
|
|
1436
|
+
})
|
|
1437
|
+
);
|
|
1438
|
+
directories[dayDir] = {
|
|
1439
|
+
statKey: typeof inventory?.statKey === "string" ? inventory.statKey : null,
|
|
1440
|
+
fileCount: inventoryFiles.length,
|
|
1441
|
+
complete,
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
return {
|
|
1446
|
+
file: relativeFile,
|
|
1447
|
+
shardKey,
|
|
1448
|
+
count: Object.keys(data || {}).length,
|
|
1449
|
+
...buildShardIntegrity(serialized),
|
|
1450
|
+
directories,
|
|
1451
|
+
projectSummary: {
|
|
1452
|
+
configs: Array.from(configs.values()),
|
|
1453
|
+
absentFreshUntil,
|
|
1454
|
+
},
|
|
1455
|
+
};
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
function buildEventShardMetadata({ relativeFile, values, serialized }) {
|
|
1459
|
+
return {
|
|
1460
|
+
file: relativeFile,
|
|
1461
|
+
count: values.size,
|
|
1462
|
+
...buildShardIntegrity(serialized),
|
|
1463
|
+
};
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
function buildShardIntegrity(serialized) {
|
|
1467
|
+
return {
|
|
1468
|
+
bytes: Buffer.byteLength(serialized),
|
|
1469
|
+
sha256: crypto.createHash("sha256").update(serialized).digest("hex"),
|
|
1470
|
+
};
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
function assertShardIntegrity(raw, metadata, label) {
|
|
1474
|
+
const integrity = buildShardIntegrity(raw);
|
|
1475
|
+
if (
|
|
1476
|
+
integrity.bytes !== metadata?.bytes ||
|
|
1477
|
+
integrity.sha256 !== metadata?.sha256
|
|
1478
|
+
) {
|
|
1479
|
+
throw cursorStoreCorruption(`Integrity check failed for ${label}`);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
async function readJsonObjectShard({
|
|
1484
|
+
generationDir,
|
|
1485
|
+
metadata,
|
|
1486
|
+
label,
|
|
1487
|
+
validate = (value) => value && typeof value === "object" && !Array.isArray(value),
|
|
1488
|
+
}) {
|
|
1489
|
+
if (!metadata?.file) return {};
|
|
1490
|
+
let raw;
|
|
1491
|
+
try {
|
|
1492
|
+
raw = await fs.readFile(path.join(generationDir, metadata.file), "utf8");
|
|
1493
|
+
} catch (error) {
|
|
1494
|
+
throw cursorStoreCorruption(`Unable to read ${label}`, error);
|
|
1495
|
+
}
|
|
1496
|
+
assertShardIntegrity(raw, metadata, label);
|
|
1497
|
+
let data;
|
|
1498
|
+
try {
|
|
1499
|
+
data = JSON.parse(raw);
|
|
1500
|
+
} catch (error) {
|
|
1501
|
+
throw cursorStoreCorruption(`Invalid ${label}`, error);
|
|
1502
|
+
}
|
|
1503
|
+
if (
|
|
1504
|
+
!data ||
|
|
1505
|
+
typeof data !== "object" ||
|
|
1506
|
+
Array.isArray(data) ||
|
|
1507
|
+
Object.keys(data).length !== metadata.count ||
|
|
1508
|
+
Object.values(data).some((value) => !validate(value))
|
|
1509
|
+
) {
|
|
1510
|
+
throw cursorStoreCorruption(`Invalid ${label}`);
|
|
1511
|
+
}
|
|
1512
|
+
return data;
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
async function writeOpencodeShard({
|
|
1516
|
+
generationDir,
|
|
1517
|
+
metadataMap,
|
|
1518
|
+
directory,
|
|
1519
|
+
shardKey,
|
|
1520
|
+
data,
|
|
1521
|
+
}) {
|
|
1522
|
+
const previousFile = metadataMap[shardKey]?.file;
|
|
1523
|
+
if (previousFile) await fs.unlink(path.join(generationDir, previousFile)).catch(() => {});
|
|
1524
|
+
if (Object.keys(data).length === 0) {
|
|
1525
|
+
delete metadataMap[shardKey];
|
|
1526
|
+
return;
|
|
1527
|
+
}
|
|
1528
|
+
const relativeFile = path.join(directory, `${shardKey}.json`);
|
|
1529
|
+
const serialized = `${JSON.stringify(data)}\n`;
|
|
1530
|
+
await fs.writeFile(path.join(generationDir, relativeFile), serialized, "utf8");
|
|
1531
|
+
metadataMap[shardKey] = {
|
|
1532
|
+
file: relativeFile,
|
|
1533
|
+
count: Object.keys(data).length,
|
|
1534
|
+
...buildShardIntegrity(serialized),
|
|
1535
|
+
};
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
async function removeOpencodeNamespaceShards({ generationDir, metadata, namespace }) {
|
|
1539
|
+
for (const shardMap of [metadata.opencodeMessages, metadata.opencodeFingerprints]) {
|
|
1540
|
+
for (const [shardKey, entry] of Object.entries(shardMap)) {
|
|
1541
|
+
if (opencodeShardNamespace(shardKey) !== namespace) continue;
|
|
1542
|
+
if (entry?.file) await fs.unlink(path.join(generationDir, entry.file)).catch(() => {});
|
|
1543
|
+
delete shardMap[shardKey];
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
function normalizeOpencodeNamespace(value) {
|
|
1549
|
+
return value === "native" || value === "wsl" ? value : "flat";
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
function opencodeNamespaceStates(state) {
|
|
1553
|
+
const out = new Map();
|
|
1554
|
+
if (!state || typeof state !== "object" || Array.isArray(state)) return out;
|
|
1555
|
+
if (state.native !== undefined || state.wsl !== undefined) {
|
|
1556
|
+
for (const namespace of ["native", "wsl"]) {
|
|
1557
|
+
const child = state[namespace];
|
|
1558
|
+
if (child && typeof child === "object" && !Array.isArray(child)) {
|
|
1559
|
+
out.set(namespace, child);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
} else {
|
|
1563
|
+
out.set("flat", state);
|
|
1564
|
+
}
|
|
1565
|
+
return out;
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1568
|
+
function opencodeStateForNamespace(cursors, namespace, create = false) {
|
|
1569
|
+
if (!cursors.opencode || typeof cursors.opencode !== "object" || Array.isArray(cursors.opencode)) {
|
|
1570
|
+
if (!create) return null;
|
|
1571
|
+
cursors.opencode = namespace === "flat" ? {} : { native: {}, wsl: {} };
|
|
1572
|
+
}
|
|
1573
|
+
const root = cursors.opencode;
|
|
1574
|
+
const namespaced = root.native !== undefined || root.wsl !== undefined;
|
|
1575
|
+
if (namespace === "flat") return namespaced ? null : root;
|
|
1576
|
+
if (!namespaced) return null;
|
|
1577
|
+
if (!root[namespace] || typeof root[namespace] !== "object" || Array.isArray(root[namespace])) {
|
|
1578
|
+
if (!create) return null;
|
|
1579
|
+
root[namespace] = {};
|
|
1580
|
+
}
|
|
1581
|
+
return root[namespace];
|
|
1582
|
+
}
|
|
1583
|
+
|
|
1584
|
+
function opencodeShardHash(value) {
|
|
1585
|
+
return crypto.createHash("sha256").update(value).digest("hex").slice(0, 2);
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
function opencodeMessageShardKey(messageKey, namespace = "flat") {
|
|
1589
|
+
return `${normalizeOpencodeNamespace(namespace)}-${opencodeShardHash(messageKey)}`;
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
function opencodeFingerprintShardKey(fingerprint, namespace = "flat") {
|
|
1593
|
+
return `${normalizeOpencodeNamespace(namespace)}-${opencodeShardHash(fingerprint)}`;
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
function opencodeShardNamespace(shardKey) {
|
|
1597
|
+
const namespace = typeof shardKey === "string" ? shardKey.split("-", 1)[0] : "";
|
|
1598
|
+
return namespace === "native" || namespace === "wsl" ? namespace : "flat";
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
function opencodeShardKeysForNamespace(namespace, metadataMap, loadedMap) {
|
|
1602
|
+
return Array.from(new Set([
|
|
1603
|
+
...Object.keys(metadataMap || {}),
|
|
1604
|
+
...Array.from(loadedMap?.keys?.() || []),
|
|
1605
|
+
].filter((shardKey) => opencodeShardNamespace(shardKey) === namespace)));
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
function hasOpencodeNamespaceShards({
|
|
1609
|
+
namespace,
|
|
1610
|
+
metadata,
|
|
1611
|
+
loadedMessages,
|
|
1612
|
+
loadedFingerprints,
|
|
1613
|
+
}) {
|
|
1614
|
+
return (
|
|
1615
|
+
opencodeShardKeysForNamespace(namespace, metadata.opencodeMessages, loadedMessages).length > 0 ||
|
|
1616
|
+
opencodeShardKeysForNamespace(namespace, metadata.opencodeFingerprints, loadedFingerprints).length > 0
|
|
1617
|
+
);
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
function opencodeNamespacesForStore({
|
|
1621
|
+
state,
|
|
1622
|
+
metadata,
|
|
1623
|
+
loadedMessages,
|
|
1624
|
+
loadedFingerprints,
|
|
1625
|
+
}) {
|
|
1626
|
+
const namespaces = new Set(opencodeNamespaceStates(state).keys());
|
|
1627
|
+
for (const shardKey of [
|
|
1628
|
+
...Object.keys(metadata.opencodeMessages || {}),
|
|
1629
|
+
...Object.keys(metadata.opencodeFingerprints || {}),
|
|
1630
|
+
...Array.from(loadedMessages.keys()),
|
|
1631
|
+
...Array.from(loadedFingerprints.keys()),
|
|
1632
|
+
]) {
|
|
1633
|
+
namespaces.add(opencodeShardNamespace(shardKey));
|
|
1634
|
+
}
|
|
1635
|
+
return namespaces;
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
function partitionOpencodeMessages(messages, namespace) {
|
|
1639
|
+
const messageShards = new Map();
|
|
1640
|
+
const fingerprintShards = new Map();
|
|
1641
|
+
for (const messageKey in messages || {}) {
|
|
1642
|
+
const entry = messages[messageKey];
|
|
1643
|
+
const messageShard = opencodeMessageShardKey(messageKey, namespace);
|
|
1644
|
+
if (!messageShards.has(messageShard)) messageShards.set(messageShard, {});
|
|
1645
|
+
messageShards.get(messageShard)[messageKey] = entry;
|
|
1646
|
+
const fingerprint = typeof entry?.fingerprint === "string" ? entry.fingerprint : null;
|
|
1647
|
+
if (!fingerprint || entry?.dedupedForkCopy === true) continue;
|
|
1648
|
+
const fingerprintShard = opencodeFingerprintShardKey(fingerprint, namespace);
|
|
1649
|
+
if (!fingerprintShards.has(fingerprintShard)) fingerprintShards.set(fingerprintShard, {});
|
|
1650
|
+
const shard = fingerprintShards.get(fingerprintShard);
|
|
1651
|
+
if (!shard[fingerprint]) shard[fingerprint] = [];
|
|
1652
|
+
shard[fingerprint].push(messageKey);
|
|
1653
|
+
}
|
|
1654
|
+
return { messageShards, fingerprintShards };
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
function cloneOpencodeCoreState(state) {
|
|
1658
|
+
if (!state || typeof state !== "object" || Array.isArray(state)) return state;
|
|
1659
|
+
if (state.native !== undefined || state.wsl !== undefined) {
|
|
1660
|
+
const clone = { ...state };
|
|
1661
|
+
for (const namespace of ["native", "wsl"]) {
|
|
1662
|
+
if (!state[namespace] || typeof state[namespace] !== "object") continue;
|
|
1663
|
+
clone[namespace] = { ...state[namespace] };
|
|
1664
|
+
delete clone[namespace].messages;
|
|
1665
|
+
}
|
|
1666
|
+
return clone;
|
|
1667
|
+
}
|
|
1668
|
+
const clone = { ...state };
|
|
1669
|
+
delete clone.messages;
|
|
1670
|
+
return clone;
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
async function validateProjectSummary(summary, nowMs) {
|
|
1674
|
+
if (!summary || typeof summary !== "object") return false;
|
|
1675
|
+
const absentFreshUntil = Number(summary.absentFreshUntil);
|
|
1676
|
+
if (Number.isFinite(absentFreshUntil) && absentFreshUntil > 0 && nowMs >= absentFreshUntil) {
|
|
1677
|
+
return false;
|
|
1678
|
+
}
|
|
1679
|
+
if (summary.absentFreshUntil === 0) return false;
|
|
1680
|
+
for (const config of Array.isArray(summary.configs) ? summary.configs : []) {
|
|
1681
|
+
const configPath = typeof config?.configPath === "string" ? config.configPath : null;
|
|
1682
|
+
if (!configPath) return false;
|
|
1683
|
+
const stat = await fs.stat(configPath).catch(() => null);
|
|
1684
|
+
if (
|
|
1685
|
+
!stat?.isFile() ||
|
|
1686
|
+
stat.mtimeMs !== config.configMtimeMs ||
|
|
1687
|
+
stat.size !== config.configSize
|
|
1688
|
+
) {
|
|
1689
|
+
return false;
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
return true;
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
function projectConfigs(context) {
|
|
1696
|
+
if (!context || typeof context !== "object") return [];
|
|
1697
|
+
const raw = Array.isArray(context.configs) ? context.configs : [context];
|
|
1698
|
+
const out = [];
|
|
1699
|
+
for (const config of raw) {
|
|
1700
|
+
const configPath = typeof config?.configPath === "string" ? config.configPath : null;
|
|
1701
|
+
if (!configPath) continue;
|
|
1702
|
+
out.push({
|
|
1703
|
+
configPath,
|
|
1704
|
+
configMtimeMs: Number.isFinite(config.configMtimeMs) ? config.configMtimeMs : null,
|
|
1705
|
+
configSize: Number.isFinite(config.configSize) ? config.configSize : null,
|
|
1706
|
+
});
|
|
1707
|
+
}
|
|
1708
|
+
return out;
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
function partitionCursorFiles(files, codexRoots = []) {
|
|
1712
|
+
const coreFiles = {};
|
|
1713
|
+
const codexFilesByShard = new Map();
|
|
1714
|
+
for (const [filePath, cursor] of Object.entries(files || {})) {
|
|
1715
|
+
if (!isCodexSessionCursorPathForRoots(filePath, codexRoots)) {
|
|
1716
|
+
coreFiles[filePath] = cursor;
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1719
|
+
const shardKey = codexFileShardKey(filePath);
|
|
1720
|
+
if (!codexFilesByShard.has(shardKey)) codexFilesByShard.set(shardKey, {});
|
|
1721
|
+
codexFilesByShard.get(shardKey)[filePath] = cursor;
|
|
1722
|
+
}
|
|
1723
|
+
return { coreFiles, codexFilesByShard };
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
function partitionEventHashes(hashes) {
|
|
1727
|
+
const shards = new Map();
|
|
1728
|
+
for (const key of Array.isArray(hashes) ? hashes : []) {
|
|
1729
|
+
if (typeof key !== "string" || key.length === 0) continue;
|
|
1730
|
+
const shardKey = codexEventShardKey(key);
|
|
1731
|
+
if (!shards.has(shardKey)) shards.set(shardKey, new Set());
|
|
1732
|
+
shards.get(shardKey).add(key);
|
|
1733
|
+
}
|
|
1734
|
+
return shards;
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
function mergeFileShardIntoCursors(data, cursors) {
|
|
1738
|
+
if (!cursors.files || typeof cursors.files !== "object") cursors.files = {};
|
|
1739
|
+
for (const [filePath, cursor] of Object.entries(data || {})) {
|
|
1740
|
+
if (!(filePath in cursors.files)) cursors.files[filePath] = cursor;
|
|
1741
|
+
}
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
function calculateGenerationCounts(metadata, coreFiles) {
|
|
1745
|
+
const nonCodexFiles = Object.keys(coreFiles || {}).length;
|
|
1746
|
+
const codexFiles = Object.values(metadata.codexFiles || {})
|
|
1747
|
+
.reduce((sum, entry) => sum + Number(entry?.count || 0), 0);
|
|
1748
|
+
const codexEvents = Object.values(metadata.codexEvents || {})
|
|
1749
|
+
.reduce((sum, entry) => sum + Number(entry?.count || 0), 0);
|
|
1750
|
+
const opencodeMessages = Object.values(metadata.opencodeMessages || {})
|
|
1751
|
+
.reduce((sum, entry) => sum + Number(entry?.count || 0), 0);
|
|
1752
|
+
const opencodeFingerprints = Object.values(metadata.opencodeFingerprints || {})
|
|
1753
|
+
.reduce((sum, entry) => sum + Number(entry?.count || 0), 0);
|
|
1754
|
+
return {
|
|
1755
|
+
nonCodexFiles,
|
|
1756
|
+
codexFiles,
|
|
1757
|
+
totalFiles: nonCodexFiles + codexFiles,
|
|
1758
|
+
codexEvents,
|
|
1759
|
+
opencodeMessages,
|
|
1760
|
+
opencodeFingerprints,
|
|
1761
|
+
};
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
function isCodexSessionCursorPath(filePath, codexRoots = []) {
|
|
1765
|
+
return isCodexSessionCursorPathForRoots(
|
|
1766
|
+
filePath,
|
|
1767
|
+
normalizeCodexRoots(codexRoots),
|
|
1768
|
+
);
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
function isCodexSessionCursorPathForRoots(filePath, codexRoots) {
|
|
1772
|
+
if (typeof filePath !== "string") return false;
|
|
1773
|
+
const normalized = filePath.replace(/\\/g, "/");
|
|
1774
|
+
if (/\/\.codex\/(?:archived_)?sessions\//.test(normalized)) return true;
|
|
1775
|
+
return codexRoots.some((root) => (
|
|
1776
|
+
normalized.startsWith(`${root}/sessions/`) ||
|
|
1777
|
+
normalized.startsWith(`${root}/archived_sessions/`)
|
|
1778
|
+
));
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
function normalizeCodexRoots(roots) {
|
|
1782
|
+
const values = Array.isArray(roots) ? roots : [roots];
|
|
1783
|
+
return Array.from(new Set(values
|
|
1784
|
+
.filter((root) => typeof root === "string" && root.trim().length > 0)
|
|
1785
|
+
.map((root) => root.replace(/\\/g, "/").replace(/\/+$/, ""))));
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
function codexFileShardKey(filePath) {
|
|
1789
|
+
if (typeof filePath !== "string") return "misc";
|
|
1790
|
+
const normalized = filePath.replace(/\\/g, "/");
|
|
1791
|
+
const match = normalized.match(/\/(\d{4})\/(\d{2})\/(\d{2})\//);
|
|
1792
|
+
return match ? `${match[1]}-${match[2]}-${match[3]}` : "misc";
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
function codexDayDirectory(filePath) {
|
|
1796
|
+
if (codexFileShardKey(filePath) === "misc") return null;
|
|
1797
|
+
return path.dirname(filePath);
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
function codexEventShardKey(key) {
|
|
1801
|
+
if (typeof key !== "string") return "misc";
|
|
1802
|
+
const separator = key.indexOf(":");
|
|
1803
|
+
const day = separator >= 0 ? key.slice(separator + 1, separator + 11) : "";
|
|
1804
|
+
return /^\d{4}-\d{2}-\d{2}$/.test(day) ? day : "misc";
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
function safeShardFilename(shardKey) {
|
|
1808
|
+
return /^\d{4}-\d{2}-\d{2}$/.test(shardKey) ? shardKey : "misc";
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
function serializeEventSet(values) {
|
|
1812
|
+
const rows = Array.from(values || []);
|
|
1813
|
+
return rows.length > 0 ? `${rows.join("\n")}\n` : "";
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
function defaultCursorState() {
|
|
1817
|
+
return { version: 1, files: {}, updatedAt: null };
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
function newGenerationId() {
|
|
1821
|
+
return `gen-${Date.now()}-${crypto.randomUUID().slice(0, 8)}`;
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
function generationDirectory(storeRoot, generationId) {
|
|
1825
|
+
return path.join(storeRoot, GENERATIONS_DIRNAME, generationId);
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
function isManifest(value) {
|
|
1829
|
+
return Boolean(
|
|
1830
|
+
value &&
|
|
1831
|
+
value.version === STORE_VERSION &&
|
|
1832
|
+
typeof value.current === "string" &&
|
|
1833
|
+
value.current.length > 0
|
|
1834
|
+
);
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
async function fingerprintFile(filePath) {
|
|
1838
|
+
try {
|
|
1839
|
+
const stat = await fs.stat(filePath, { bigint: true });
|
|
1840
|
+
return {
|
|
1841
|
+
dev: String(stat.dev),
|
|
1842
|
+
ino: String(stat.ino),
|
|
1843
|
+
size: String(stat.size),
|
|
1844
|
+
mtimeNs: String(stat.mtimeNs),
|
|
1845
|
+
ctimeNs: String(stat.ctimeNs),
|
|
1846
|
+
};
|
|
1847
|
+
} catch (e) {
|
|
1848
|
+
if (e?.code === "ENOENT") return null;
|
|
1849
|
+
throw e;
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
function sameFingerprint(left, right) {
|
|
1854
|
+
if (!left || !right) return left === right;
|
|
1855
|
+
return ["dev", "ino", "size", "mtimeNs", "ctimeNs"]
|
|
1856
|
+
.every((key) => String(left[key]) === String(right[key]));
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
async function cleanupGenerations(storeRoot, keep) {
|
|
1860
|
+
const root = path.join(storeRoot, GENERATIONS_DIRNAME);
|
|
1861
|
+
let entries = [];
|
|
1862
|
+
try {
|
|
1863
|
+
entries = await fs.readdir(root, { withFileTypes: true });
|
|
1864
|
+
} catch (e) {
|
|
1865
|
+
if (e?.code !== "ENOENT") throw e;
|
|
1866
|
+
return;
|
|
1867
|
+
}
|
|
1868
|
+
for (const entry of entries) {
|
|
1869
|
+
if (!entry.isDirectory() || keep.has(entry.name)) continue;
|
|
1870
|
+
await fs.rm(path.join(root, entry.name), { recursive: true, force: true });
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
function cloneJson(value) {
|
|
1875
|
+
return JSON.parse(JSON.stringify(value));
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
function replaceObjectContents(target, source) {
|
|
1879
|
+
for (const key of Object.keys(target)) delete target[key];
|
|
1880
|
+
Object.assign(target, source);
|
|
1881
|
+
}
|
|
1882
|
+
|
|
1883
|
+
function cursorStoreCorruption(message, cause = null) {
|
|
1884
|
+
const error = new Error(message, cause ? { cause } : undefined);
|
|
1885
|
+
error.code = "TOKENUSAGE_CURSOR_STORE_CORRUPT";
|
|
1886
|
+
return error;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
function cursorStoreRetry(cause) {
|
|
1890
|
+
const error = new Error(
|
|
1891
|
+
"Cursor store generation changed; restart the operation",
|
|
1892
|
+
cause ? { cause } : undefined,
|
|
1893
|
+
);
|
|
1894
|
+
error.code = CURSOR_STORE_RETRY_CODE;
|
|
1895
|
+
return error;
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
function isCursorStoreRetry(error) {
|
|
1899
|
+
return error?.code === CURSOR_STORE_RETRY_CODE;
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
async function maybeInjectFailure(injector, stage) {
|
|
1903
|
+
if (typeof injector === "function") await injector(stage);
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
module.exports = {
|
|
1907
|
+
DEFAULT_ACTIVATION_BYTES,
|
|
1908
|
+
STORE_DIRNAME,
|
|
1909
|
+
STORE_VERSION,
|
|
1910
|
+
codexEventShardKey,
|
|
1911
|
+
codexFileShardKey,
|
|
1912
|
+
opencodeFingerprintShardKey,
|
|
1913
|
+
opencodeMessageShardKey,
|
|
1914
|
+
isCodexSessionCursorPath,
|
|
1915
|
+
isCursorStoreRetry,
|
|
1916
|
+
openCursorStore,
|
|
1917
|
+
readCursorStateSummary,
|
|
1918
|
+
};
|