prime-agent-dsh 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/LICENSE +21 -0
- package/README.md +121 -0
- package/SECURITY.md +27 -0
- package/THIRD_PARTY_NOTICES.md +3 -0
- package/docs/context-spill.md +7 -0
- package/docs/durable-context-query.md +34 -0
- package/docs/getting-started.md +253 -0
- package/docs/security.md +78 -0
- package/docs/shadow-telemetry-validation.md +18 -0
- package/docs/single-window-cache-architecture.md +24 -0
- package/extensions/index.ts +161 -0
- package/extensions/shadow-context.ts +168 -0
- package/package.json +109 -0
- package/scripts/package-smoke.mjs +148 -0
- package/scripts/patch-pi-ai-partial-json.mjs +14 -0
- package/skills/dsh-context/SKILL.md +62 -0
- package/skills/dsh-context/pyproject.toml +13 -0
- package/skills/dsh-context/src/dsh_context/__init__.py +573 -0
- package/src/context-converter.ts +176 -0
- package/src/context-objects.ts +247 -0
- package/src/context-protocol.ts +37 -0
- package/src/context-spill.ts +109 -0
- package/src/dsh-context-service.ts +38 -0
- package/src/dsh-image-attachments.ts +99 -0
- package/src/durable-context-query.ts +233 -0
- package/src/durable-context-store.ts +838 -0
- package/src/durable-file-attachments.ts +186 -0
- package/src/prefix-metrics.ts +91 -0
- package/src/provider-cache-series.ts +56 -0
- package/src/recursive-context-loader.ts +215 -0
- package/src/rlm-context-bootstrap.ts +333 -0
- package/src/rlm-context-inheritance.ts +389 -0
- package/src/shadow-telemetry.ts +84 -0
|
@@ -0,0 +1,838 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
import {
|
|
3
|
+
chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readSync,
|
|
4
|
+
realpathSync, readdirSync, renameSync, rmSync, statfsSync, statSync, unlinkSync, writeFileSync,
|
|
5
|
+
} from "node:fs";
|
|
6
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
7
|
+
|
|
8
|
+
export const DURABLE_STORE_VERSION = "prime-agent-dsh/durable-store-v3-reference" as const;
|
|
9
|
+
export const DURABLE_OBJECT_VERSION = "prime-agent-dsh/derived-object-v3-reference" as const;
|
|
10
|
+
export const DURABLE_COMMIT_VERSION = "prime-agent-dsh/derived-commit-v1" as const;
|
|
11
|
+
const SHA256 = /^[a-f0-9]{64}$/;
|
|
12
|
+
|
|
13
|
+
type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
|
|
14
|
+
|
|
15
|
+
export interface StoreBinding {
|
|
16
|
+
readonly sessionId: string;
|
|
17
|
+
/** Path to Prime's canonical JSONL. It is identified by its real path, never read or modified. */
|
|
18
|
+
readonly primeSessionFile: string;
|
|
19
|
+
}
|
|
20
|
+
export interface PublishInput {
|
|
21
|
+
/** Exact Prime records used to derive this view. */
|
|
22
|
+
readonly source: readonly unknown[];
|
|
23
|
+
/** Exact converted records consumed by the derived context reader. */
|
|
24
|
+
readonly effective: readonly unknown[];
|
|
25
|
+
/** Prime source entry for each effective message; null only when no public mapping exists. */
|
|
26
|
+
readonly effectiveSourceIndexes?: readonly (number | null)[];
|
|
27
|
+
readonly converterVersion: string;
|
|
28
|
+
readonly schemaVersion: string;
|
|
29
|
+
readonly compatibilityMetrics?: Readonly<Record<string, number>>;
|
|
30
|
+
readonly cropped?: boolean;
|
|
31
|
+
/** Optional exact leaf identifier for diagnostic/compatibility views. */
|
|
32
|
+
readonly branchId?: string;
|
|
33
|
+
readonly observedAt?: number;
|
|
34
|
+
}
|
|
35
|
+
export interface CompatibilityView {
|
|
36
|
+
readonly version: typeof DURABLE_STORE_VERSION;
|
|
37
|
+
readonly sessionId: string;
|
|
38
|
+
readonly branchId: string;
|
|
39
|
+
readonly revision: number;
|
|
40
|
+
readonly messageCount: number;
|
|
41
|
+
readonly entries: readonly [];
|
|
42
|
+
readonly cropped: boolean;
|
|
43
|
+
readonly sourceDigest: string;
|
|
44
|
+
readonly effectiveDigest: string;
|
|
45
|
+
readonly converterVersion: string;
|
|
46
|
+
readonly schemaVersion: string;
|
|
47
|
+
readonly metrics?: Readonly<Record<string, number>>;
|
|
48
|
+
}
|
|
49
|
+
export interface SourceLocator {
|
|
50
|
+
readonly index: number;
|
|
51
|
+
readonly byteOffset: number;
|
|
52
|
+
readonly byteLength: number;
|
|
53
|
+
readonly line: number;
|
|
54
|
+
readonly entryDigest: string;
|
|
55
|
+
readonly entryId?: string;
|
|
56
|
+
}
|
|
57
|
+
export interface EffectiveReference {
|
|
58
|
+
readonly entryDigest: string;
|
|
59
|
+
readonly sourceIndex: number | null;
|
|
60
|
+
readonly role?: string;
|
|
61
|
+
}
|
|
62
|
+
export interface DerivedObject {
|
|
63
|
+
readonly version: typeof DURABLE_OBJECT_VERSION;
|
|
64
|
+
readonly bindingDigest: string;
|
|
65
|
+
readonly sourceDigest: string;
|
|
66
|
+
readonly effectiveDigest: string;
|
|
67
|
+
readonly sourceEntryDigests: readonly string[];
|
|
68
|
+
readonly effectiveEntryDigests: readonly string[];
|
|
69
|
+
readonly sourceLocators: readonly SourceLocator[];
|
|
70
|
+
readonly effectiveReferences: readonly EffectiveReference[];
|
|
71
|
+
readonly compatibility: CompatibilityView;
|
|
72
|
+
}
|
|
73
|
+
export type EffectiveProjectionRebuildReason = "initial" | "none" | "converter-change" | "source-diverged" | "source-append-effective-projection-change" | "effective-diverged";
|
|
74
|
+
export interface PrimeIndexDiagnostics {
|
|
75
|
+
readonly mode: "full" | "tail" | "cache";
|
|
76
|
+
readonly bytesProcessed: number;
|
|
77
|
+
readonly linesProcessed: number;
|
|
78
|
+
}
|
|
79
|
+
export interface PublicationDiagnostics {
|
|
80
|
+
readonly source: { readonly mode: "append" | "rebuild" | "noop"; readonly reused: number; readonly new: number; readonly reindexed: number };
|
|
81
|
+
readonly index?: PrimeIndexDiagnostics;
|
|
82
|
+
readonly effective: { readonly reused: number; readonly new: number; readonly reindexed: number; readonly rebuildReason: EffectiveProjectionRebuildReason };
|
|
83
|
+
}
|
|
84
|
+
export interface DerivedCommit {
|
|
85
|
+
readonly version: typeof DURABLE_COMMIT_VERSION;
|
|
86
|
+
readonly bindingDigest: string;
|
|
87
|
+
readonly generation: number;
|
|
88
|
+
readonly parent: string | null;
|
|
89
|
+
readonly object: string;
|
|
90
|
+
readonly sourceDigest: string;
|
|
91
|
+
readonly effectiveDigest: string;
|
|
92
|
+
readonly converterVersion: string;
|
|
93
|
+
readonly schemaVersion: string;
|
|
94
|
+
readonly mode: "append" | "rebuild";
|
|
95
|
+
readonly commonPrefix: number;
|
|
96
|
+
readonly observedAt: number;
|
|
97
|
+
readonly publication?: PublicationDiagnostics;
|
|
98
|
+
}
|
|
99
|
+
export interface RecoveredGeneration {
|
|
100
|
+
readonly commitDigest: string;
|
|
101
|
+
readonly commit: DerivedCommit;
|
|
102
|
+
readonly object: DerivedObject;
|
|
103
|
+
}
|
|
104
|
+
export interface PublishResult extends RecoveredGeneration { readonly mode: "append" | "rebuild" | "noop"; readonly publication: PublicationDiagnostics }
|
|
105
|
+
export type FaultBoundary = "object-durable" | "commit-durable" | "head-durable" | "current-temp-durable" | "current-replaced";
|
|
106
|
+
|
|
107
|
+
export class DurablePublicationUnavailableError extends Error {
|
|
108
|
+
readonly code = "DURABLE_PUBLICATION_UNAVAILABLE";
|
|
109
|
+
constructor(message: string) { super(message); this.name = "DurablePublicationUnavailableError"; }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface DurableContextStoreOptions {
|
|
113
|
+
readonly root: string;
|
|
114
|
+
readonly binding: StoreBinding;
|
|
115
|
+
readonly recoveryScanLimit?: number;
|
|
116
|
+
readonly maxObjectBytes?: number;
|
|
117
|
+
/** Maximum bytes allowed for the private, content-free Prime JSONL index. */
|
|
118
|
+
readonly maxIndexBytes?: number;
|
|
119
|
+
/** Maximum bytes owned by the rebuildable derived store. */
|
|
120
|
+
readonly maxStoreBytes?: number;
|
|
121
|
+
/** Refuse a publication that would leave less filesystem space than this. */
|
|
122
|
+
readonly minFreeBytes?: number;
|
|
123
|
+
/** Complete generations retained after a successful publication. */
|
|
124
|
+
readonly retainGenerations?: number;
|
|
125
|
+
readonly lockTimeoutMs?: number;
|
|
126
|
+
readonly staleLockMs?: number;
|
|
127
|
+
readonly fault?: (boundary: FaultBoundary) => void;
|
|
128
|
+
readonly now?: () => number;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function wellFormed(value: string): boolean {
|
|
132
|
+
for (let i = 0; i < value.length; i++) { const code = value.charCodeAt(i); if (code >= 0xd800 && code <= 0xdbff) { const next = value.charCodeAt(++i); if (!(next >= 0xdc00 && next <= 0xdfff)) return false; } else if (code >= 0xdc00 && code <= 0xdfff) return false; } return true;
|
|
133
|
+
}
|
|
134
|
+
function canonical(value: unknown, seen = new Set<object>()): string {
|
|
135
|
+
if (value === null) return "null";
|
|
136
|
+
if (typeof value === "string") { if (!wellFormed(value)) throw new TypeError("derived context strings must be well-formed Unicode"); return JSON.stringify(value); }
|
|
137
|
+
if (typeof value === "boolean") return JSON.stringify(value);
|
|
138
|
+
if (typeof value === "number") {
|
|
139
|
+
if (!Number.isFinite(value)) throw new TypeError("derived context must contain finite JSON numbers");
|
|
140
|
+
return JSON.stringify(Object.is(value, -0) ? 0 : value);
|
|
141
|
+
}
|
|
142
|
+
if (Array.isArray(value)) {
|
|
143
|
+
if (seen.has(value)) throw new TypeError("derived context must not contain cycles");
|
|
144
|
+
seen.add(value); const result = `[${value.map((item) => canonical(item, seen)).join(",")}]`; seen.delete(value); return result;
|
|
145
|
+
}
|
|
146
|
+
if (typeof value === "object") {
|
|
147
|
+
if (seen.has(value)) throw new TypeError("derived context must not contain cycles");
|
|
148
|
+
const record = value as Record<string, unknown>;
|
|
149
|
+
const prototype = Reflect.getPrototypeOf(record);
|
|
150
|
+
if (prototype !== Object.prototype && prototype !== null) throw new TypeError("derived context must contain plain JSON objects");
|
|
151
|
+
seen.add(record);
|
|
152
|
+
const fields: string[] = [];
|
|
153
|
+
for (const key of Object.keys(record).sort()) {
|
|
154
|
+
const item = record[key];
|
|
155
|
+
if (item === undefined || typeof item === "function" || typeof item === "symbol" || typeof item === "bigint") {
|
|
156
|
+
throw new TypeError(`derived context field ${key} is not JSON`);
|
|
157
|
+
}
|
|
158
|
+
fields.push(`${JSON.stringify(key)}:${canonical(item, seen)}`);
|
|
159
|
+
}
|
|
160
|
+
seen.delete(record); return `{${fields.join(",")}}`;
|
|
161
|
+
}
|
|
162
|
+
throw new TypeError("derived context must be JSON");
|
|
163
|
+
}
|
|
164
|
+
function digest(value: unknown): string { return createHash("sha256").update(canonical(value)).digest("hex"); }
|
|
165
|
+
function safeFileText(path: string, maximum = 32 * 1024 * 1024): string {
|
|
166
|
+
const stat = lstatSync(path);
|
|
167
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > maximum) throw new Error(`unsafe or oversized durable store file: ${path}`);
|
|
168
|
+
return readFileSync(path, "utf8");
|
|
169
|
+
}
|
|
170
|
+
function parseJson(path: string): unknown { return JSON.parse(safeFileText(path)) as unknown; }
|
|
171
|
+
function object(value: unknown): Record<string, unknown> | undefined { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined; }
|
|
172
|
+
function safeInteger(value: unknown): value is number { return typeof value === "number" && Number.isSafeInteger(value) && value > 0; }
|
|
173
|
+
function syncDirectory(path: string): void {
|
|
174
|
+
let fd: number | undefined;
|
|
175
|
+
try { fd = openSync(path, constants.O_RDONLY); fsyncSync(fd); } catch (error) {
|
|
176
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
177
|
+
if (code !== "EINVAL" && code !== "ENOTSUP" && code !== "EBADF") throw error;
|
|
178
|
+
} finally { if (fd !== undefined) closeSync(fd); }
|
|
179
|
+
}
|
|
180
|
+
function canonicalNewPath(path: string): string {
|
|
181
|
+
let cursor = resolve(path); const suffix: string[] = [];
|
|
182
|
+
while (!existsSync(cursor)) { suffix.unshift(cursor.slice(dirname(cursor).length + (dirname(cursor) === sep ? 0 : 1))); cursor = dirname(cursor); }
|
|
183
|
+
return join(realpathSync(cursor), ...suffix);
|
|
184
|
+
}
|
|
185
|
+
function ensureNoSymlink(path: string): void {
|
|
186
|
+
const absolute = resolve(path); let cursor = absolute;
|
|
187
|
+
const pending: string[] = [];
|
|
188
|
+
while (!existsSync(cursor)) { pending.push(cursor); const parent = dirname(cursor); if (parent === cursor) break; cursor = parent; }
|
|
189
|
+
while (true) {
|
|
190
|
+
const stat = lstatSync(cursor);
|
|
191
|
+
if (stat.isSymbolicLink()) throw new Error(`symlink is not allowed in durable store path: ${cursor}`);
|
|
192
|
+
const parent = dirname(cursor); if (parent === cursor) break; cursor = parent;
|
|
193
|
+
}
|
|
194
|
+
void pending;
|
|
195
|
+
}
|
|
196
|
+
function privateDirectory(path: string): void {
|
|
197
|
+
ensureNoSymlink(path); mkdirSync(path, { recursive: true, mode: 0o700 });
|
|
198
|
+
const stat = lstatSync(path); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error(`unsafe durable store directory: ${path}`);
|
|
199
|
+
chmodSync(path, 0o700);
|
|
200
|
+
}
|
|
201
|
+
function assertChild(root: string, path: string): void {
|
|
202
|
+
const rel = relative(root, path);
|
|
203
|
+
if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw new Error(`path escapes durable store: ${path}`);
|
|
204
|
+
}
|
|
205
|
+
function atomicWrite(path: string, contents: string, replace: boolean, beforeRename?: () => void, syncAfter = true): void {
|
|
206
|
+
const directory = dirname(path); const temporary = join(directory, `.tmp-${process.pid}-${randomBytes(12).toString("hex")}`);
|
|
207
|
+
const fd = openSync(temporary, "wx", 0o600);
|
|
208
|
+
try { writeFileSync(fd, contents, "utf8"); fsyncSync(fd); } finally { closeSync(fd); }
|
|
209
|
+
beforeRename?.();
|
|
210
|
+
try {
|
|
211
|
+
if (!replace && existsSync(path)) { unlinkSync(temporary); return; }
|
|
212
|
+
renameSync(temporary, path); if (syncAfter) syncDirectory(directory);
|
|
213
|
+
} catch (error) { try { unlinkSync(temporary); } catch { /* best effort */ } throw error; }
|
|
214
|
+
}
|
|
215
|
+
function sleep(ms: number): Promise<void> { return new Promise((accept) => setTimeout(accept, ms)); }
|
|
216
|
+
|
|
217
|
+
interface PrimeLine { offset: number; length: number; line: number; digest: string; id?: string }
|
|
218
|
+
interface PrimeIndexCursor {
|
|
219
|
+
version: "prime-agent-dsh/prime-jsonl-index-v1";
|
|
220
|
+
bindingDigest: string;
|
|
221
|
+
identity: { dev: number; ino: number };
|
|
222
|
+
byteOffset: number;
|
|
223
|
+
nextLine: number;
|
|
224
|
+
mtimeMs: number;
|
|
225
|
+
sourceDigest: string;
|
|
226
|
+
entries: PrimeLine[];
|
|
227
|
+
checksum: string;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function readRange(fd: number, offset: number, length: number): Buffer {
|
|
231
|
+
const result = Buffer.alloc(length); let done = 0;
|
|
232
|
+
while (done < length) { const count = readSync(fd, result, done, length - done, offset + done); if (count === 0) throw new Error("Prime JSONL changed while it was read"); done += count; }
|
|
233
|
+
return result;
|
|
234
|
+
}
|
|
235
|
+
function indexedDigest(entries: readonly PrimeLine[]): string {
|
|
236
|
+
return digest(entries.map(({ offset, length, line, digest: entryDigest, id }) => ({ offset, length, line, digest: entryDigest, ...(id === undefined ? {} : { id }) })));
|
|
237
|
+
}
|
|
238
|
+
function parsePrimeBytes(raw: Buffer, baseOffset: number, firstLine: number): { entries: PrimeLine[]; nextLine: number } {
|
|
239
|
+
const entries: PrimeLine[] = []; let start = 0, line = firstLine;
|
|
240
|
+
const add = (endExclusive: number): void => {
|
|
241
|
+
let end = endExclusive; if (end > start && raw[end - 1] === 0x0d) end--;
|
|
242
|
+
if (end > start) {
|
|
243
|
+
const bytes = raw.subarray(start, end); let value: unknown;
|
|
244
|
+
try { value = JSON.parse(bytes.toString("utf8")); } catch { throw new Error(`invalid Prime JSONL at line ${line}`); }
|
|
245
|
+
const normalized = JSON.parse(canonical(value)) as Json; const record = object(normalized);
|
|
246
|
+
entries.push({ offset: baseOffset + start, length: end - start, line, digest: digest(normalized), ...(typeof record?.id === "string" ? { id: record.id } : {}) });
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
for (let cursor = 0; cursor < raw.length; cursor++) {
|
|
250
|
+
if (raw[cursor] !== 0x0a) continue;
|
|
251
|
+
add(cursor); start = cursor + 1; line++;
|
|
252
|
+
}
|
|
253
|
+
if (start < raw.length) { add(raw.length); line++; }
|
|
254
|
+
return { entries, nextLine: line };
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function validPrimeLine(value: unknown): value is PrimeLine {
|
|
258
|
+
const item = object(value);
|
|
259
|
+
return !!item && Number.isSafeInteger(item.offset) && (item.offset as number) >= 0
|
|
260
|
+
&& safeInteger(item.length) && safeInteger(item.line) && typeof item.digest === "string" && SHA256.test(item.digest)
|
|
261
|
+
&& (item.id === undefined || typeof item.id === "string");
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** A rebuildable, content-addressed publication store. Prime's JSONL remains authoritative. */
|
|
265
|
+
function reuseCounts(previous: readonly string[], next: readonly string[]): { reused: number; new: number; reindexed: number } {
|
|
266
|
+
const remaining = new Map<string, number[]>();
|
|
267
|
+
previous.forEach((value, index) => { const indexes = remaining.get(value) ?? []; indexes.push(index); remaining.set(value, indexes); });
|
|
268
|
+
let reused = 0, added = 0, reindexed = 0;
|
|
269
|
+
next.forEach((value, index) => {
|
|
270
|
+
const indexes = remaining.get(value);
|
|
271
|
+
if (!indexes?.length) { added++; return; }
|
|
272
|
+
const same = indexes.indexOf(index);
|
|
273
|
+
if (same >= 0) { indexes.splice(same, 1); reused++; } else { indexes.shift(); reindexed++; }
|
|
274
|
+
});
|
|
275
|
+
return { reused, new: added, reindexed };
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
export class DurableContextStore {
|
|
279
|
+
readonly root: string;
|
|
280
|
+
readonly binding: Readonly<StoreBinding & { primeSessionFile: string }>;
|
|
281
|
+
readonly bindingDigest: string;
|
|
282
|
+
private readonly scanLimit: number;
|
|
283
|
+
private readonly maxObjectBytes: number;
|
|
284
|
+
private readonly maxIndexBytes: number;
|
|
285
|
+
private readonly maxStoreBytes: number;
|
|
286
|
+
private readonly minFreeBytes: number;
|
|
287
|
+
private readonly retainGenerations: number;
|
|
288
|
+
private readonly lockTimeout: number;
|
|
289
|
+
private readonly staleLock: number;
|
|
290
|
+
private readonly fault?: (boundary: FaultBoundary) => void;
|
|
291
|
+
private readonly now: () => number;
|
|
292
|
+
|
|
293
|
+
constructor(options: DurableContextStoreOptions) {
|
|
294
|
+
if (!isAbsolute(options.root)) throw new Error("durable store root must be absolute");
|
|
295
|
+
if (!/^[A-Za-z0-9._-]{1,128}$/.test(options.binding.sessionId)) throw new Error("invalid session id");
|
|
296
|
+
if (!isAbsolute(options.binding.primeSessionFile)) throw new Error("Prime session file must be absolute");
|
|
297
|
+
if (existsSync(options.root) && lstatSync(options.root).isSymbolicLink()) throw new Error(`symlink is not allowed in durable store path: ${options.root}`);
|
|
298
|
+
const canonicalRoot = canonicalNewPath(options.root);
|
|
299
|
+
ensureNoSymlink(canonicalRoot);
|
|
300
|
+
const sessionFile = realpathSync(options.binding.primeSessionFile);
|
|
301
|
+
const sessionStat = statSync(sessionFile); if (!sessionStat.isFile()) throw new Error("Prime session binding is not a file");
|
|
302
|
+
this.root = canonicalRoot;
|
|
303
|
+
this.binding = Object.freeze({ sessionId: options.binding.sessionId, primeSessionFile: sessionFile });
|
|
304
|
+
this.bindingDigest = digest({ sessionId: this.binding.sessionId, primeSessionFile: sessionFile });
|
|
305
|
+
this.scanLimit = options.recoveryScanLimit ?? 128;
|
|
306
|
+
this.maxObjectBytes = options.maxObjectBytes ?? 16 * 1024 * 1024;
|
|
307
|
+
this.maxIndexBytes = options.maxIndexBytes ?? 64 * 1024 * 1024;
|
|
308
|
+
this.maxStoreBytes = options.maxStoreBytes ?? 64 * 1024 * 1024;
|
|
309
|
+
this.minFreeBytes = options.minFreeBytes ?? 128 * 1024 * 1024;
|
|
310
|
+
this.retainGenerations = options.retainGenerations ?? 2;
|
|
311
|
+
this.lockTimeout = options.lockTimeoutMs ?? 10_000;
|
|
312
|
+
this.staleLock = options.staleLockMs ?? 120_000;
|
|
313
|
+
this.fault = options.fault; this.now = options.now ?? Date.now;
|
|
314
|
+
if (![this.scanLimit, this.maxObjectBytes, this.maxIndexBytes, this.maxStoreBytes, this.minFreeBytes, this.retainGenerations, this.lockTimeout, this.staleLock].every((v) => Number.isSafeInteger(v) && v > 0)) throw new Error("store limits must be positive integers");
|
|
315
|
+
this.initialize();
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
private initialize(): void {
|
|
319
|
+
privateDirectory(this.root);
|
|
320
|
+
if (this.needsSchemaReset()) this.resetDerivedCache();
|
|
321
|
+
for (const name of ["objects", "commits", "heads", "quarantine", "indexes"]) privateDirectory(join(this.root, name));
|
|
322
|
+
const bindingPath = join(this.root, "BINDING");
|
|
323
|
+
const value = `${canonical({ version: DURABLE_STORE_VERSION, ...this.binding, bindingDigest: this.bindingDigest })}\n`;
|
|
324
|
+
if (existsSync(bindingPath)) {
|
|
325
|
+
const stat = lstatSync(bindingPath);
|
|
326
|
+
if (!stat.isFile() || stat.isSymbolicLink() || readFileSync(bindingPath, "utf8") !== value) {
|
|
327
|
+
throw new Error("durable store is bound to another Prime session");
|
|
328
|
+
}
|
|
329
|
+
} else atomicWrite(bindingPath, value, false);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** Unsupported derived schemas are disposable caches. User artifacts and grants are never touched. */
|
|
333
|
+
private needsSchemaReset(): boolean {
|
|
334
|
+
const bindingPath = join(this.root, "BINDING");
|
|
335
|
+
// Pre-store context objects used snapshots/ + manifest.json without a
|
|
336
|
+
// BINDING. V2 used bodies/. Both are disposable derived caches.
|
|
337
|
+
if (existsSync(join(this.root, "snapshots")) || existsSync(join(this.root, "bodies"))) return true;
|
|
338
|
+
if (!existsSync(bindingPath) && existsSync(join(this.root, "manifest.json"))) return true;
|
|
339
|
+
if (existsSync(bindingPath)) {
|
|
340
|
+
try {
|
|
341
|
+
const binding = object(parseJson(bindingPath));
|
|
342
|
+
if (binding?.version !== DURABLE_STORE_VERSION) return true;
|
|
343
|
+
} catch { return true; }
|
|
344
|
+
}
|
|
345
|
+
const objectsPath = join(this.root, "objects");
|
|
346
|
+
if (!existsSync(objectsPath)) return false;
|
|
347
|
+
try {
|
|
348
|
+
for (const name of requireDirectory(objectsPath)) {
|
|
349
|
+
if (!/^[a-f0-9]{64}\.json$/.test(name)) continue;
|
|
350
|
+
try { if (object(parseJson(join(objectsPath, name)))?.version !== DURABLE_OBJECT_VERSION) return true; }
|
|
351
|
+
catch { /* normal recovery skips corrupt current-schema cache objects */ }
|
|
352
|
+
}
|
|
353
|
+
} catch { return true; }
|
|
354
|
+
return false;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
private resetDerivedCache(): void {
|
|
358
|
+
for (const name of ["heads", "commits", "objects", "bodies", "snapshots", "indexes"]) {
|
|
359
|
+
rmSync(join(this.root, name), { recursive: true, force: true });
|
|
360
|
+
}
|
|
361
|
+
for (const name of requireDirectory(this.root)) {
|
|
362
|
+
if (name === "BINDING" || name === "CURRENT" || name === "GENERATION" || name === "index.json" || name === "index.sqlite"
|
|
363
|
+
|| name === "manifest.json" || /^manifest-[A-Za-z0-9._-]+\.json$/.test(name)) {
|
|
364
|
+
rmSync(join(this.root, name), { recursive: true, force: true });
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
syncDirectory(this.root);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
private currentCandidate(): string | undefined {
|
|
371
|
+
const path = join(this.root, "CURRENT");
|
|
372
|
+
try { const value = safeFileText(path).trim(); return SHA256.test(value) ? value : undefined; } catch { return undefined; }
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
private validate(commitDigest: string): RecoveredGeneration | undefined {
|
|
376
|
+
if (!SHA256.test(commitDigest)) return undefined;
|
|
377
|
+
try {
|
|
378
|
+
const commitPath = join(this.root, "commits", `${commitDigest}.json`); assertChild(this.root, commitPath);
|
|
379
|
+
const commitRaw = safeFileText(commitPath).trim();
|
|
380
|
+
if (createHash("sha256").update(commitRaw).digest("hex") !== commitDigest) return undefined;
|
|
381
|
+
const c = object(JSON.parse(commitRaw));
|
|
382
|
+
if (!c || c.version !== DURABLE_COMMIT_VERSION || c.bindingDigest !== this.bindingDigest || !safeInteger(c.generation)
|
|
383
|
+
|| typeof c.object !== "string" || !SHA256.test(c.object) || typeof c.sourceDigest !== "string" || typeof c.effectiveDigest !== "string"
|
|
384
|
+
|| !SHA256.test(c.sourceDigest) || !SHA256.test(c.effectiveDigest) || typeof c.converterVersion !== "string" || !c.converterVersion
|
|
385
|
+
|| typeof c.schemaVersion !== "string" || !c.schemaVersion || (c.mode !== "append" && c.mode !== "rebuild")
|
|
386
|
+
|| !Number.isSafeInteger(c.commonPrefix) || (c.parent !== null && (typeof c.parent !== "string" || !SHA256.test(c.parent)))) return undefined;
|
|
387
|
+
const objectPath = join(this.root, "objects", `${c.object}.json`); assertChild(this.root, objectPath);
|
|
388
|
+
const objectRaw = safeFileText(objectPath).trim();
|
|
389
|
+
if (createHash("sha256").update(objectRaw).digest("hex") !== c.object) return undefined;
|
|
390
|
+
const o = object(JSON.parse(objectRaw));
|
|
391
|
+
if (!o || o.version !== DURABLE_OBJECT_VERSION || o.bindingDigest !== this.bindingDigest
|
|
392
|
+
|| o.sourceDigest !== c.sourceDigest || o.effectiveDigest !== c.effectiveDigest
|
|
393
|
+
|| !Array.isArray(o.sourceEntryDigests) || !Array.isArray(o.effectiveEntryDigests) || !object(o.compatibility)) return undefined;
|
|
394
|
+
const sourceEntryDigests = o.sourceEntryDigests as unknown[];
|
|
395
|
+
const effectiveEntryDigests = o.effectiveEntryDigests as unknown[];
|
|
396
|
+
if (!sourceEntryDigests.every((item) => typeof item === "string" && SHA256.test(item))
|
|
397
|
+
|| !effectiveEntryDigests.every((item) => typeof item === "string" && SHA256.test(item))) return undefined;
|
|
398
|
+
if ("effective" in o || !Array.isArray(o.sourceLocators) || !Array.isArray(o.effectiveReferences)
|
|
399
|
+
|| o.sourceLocators.length !== sourceEntryDigests.length || o.effectiveReferences.length !== effectiveEntryDigests.length) return undefined;
|
|
400
|
+
const source: Json[] = []; const fd = openSync(this.binding.primeSessionFile, constants.O_RDONLY);
|
|
401
|
+
try {
|
|
402
|
+
const file = fstatSync(fd);
|
|
403
|
+
for (let index = 0; index < o.sourceLocators.length; index++) {
|
|
404
|
+
const locator = object(o.sourceLocators[index]);
|
|
405
|
+
if (!locator || locator.index !== index || !Number.isSafeInteger(locator.byteOffset) || (locator.byteOffset as number) < 0
|
|
406
|
+
|| !safeInteger(locator.byteLength) || !safeInteger(locator.line) || locator.entryDigest !== sourceEntryDigests[index]
|
|
407
|
+
|| (locator.entryId !== undefined && typeof locator.entryId !== "string")) return undefined;
|
|
408
|
+
const start = Number(locator.byteOffset), end = start + Number(locator.byteLength);
|
|
409
|
+
if (end > file.size || (start > 0 && readRange(fd, start - 1, 1)[0] !== 0x0a)) return undefined;
|
|
410
|
+
if (end < file.size) {
|
|
411
|
+
const boundary = readRange(fd, end, Math.min(2, file.size - end));
|
|
412
|
+
if (boundary[0] !== 0x0a && !(boundary[0] === 0x0d && boundary[1] === 0x0a)) return undefined;
|
|
413
|
+
}
|
|
414
|
+
const value = JSON.parse(readRange(fd, start, Number(locator.byteLength)).toString("utf8")) as Json;
|
|
415
|
+
if (digest(value) !== locator.entryDigest) return undefined;
|
|
416
|
+
const valueId = object(value)?.id;
|
|
417
|
+
if (locator.entryId !== undefined && valueId !== locator.entryId) return undefined;
|
|
418
|
+
source.push(value);
|
|
419
|
+
}
|
|
420
|
+
} finally { closeSync(fd); }
|
|
421
|
+
if (digest(source) !== o.sourceDigest) return undefined;
|
|
422
|
+
for (let index = 0; index < o.effectiveReferences.length; index++) {
|
|
423
|
+
const reference = object(o.effectiveReferences[index]);
|
|
424
|
+
if (!reference || reference.entryDigest !== effectiveEntryDigests[index]
|
|
425
|
+
|| (reference.sourceIndex !== null && (!Number.isSafeInteger(reference.sourceIndex) || (reference.sourceIndex as number) < 0 || (reference.sourceIndex as number) >= source.length))
|
|
426
|
+
|| (reference.role !== undefined && typeof reference.role !== "string")) return undefined;
|
|
427
|
+
}
|
|
428
|
+
const compatibility = object(o.compatibility);
|
|
429
|
+
if (!compatibility || compatibility.version !== DURABLE_STORE_VERSION || compatibility.sessionId !== this.binding.sessionId
|
|
430
|
+
|| typeof compatibility.branchId !== "string" || compatibility.branchId.length === 0 || compatibility.branchId.length > 512
|
|
431
|
+
|| compatibility.revision !== c.generation || compatibility.messageCount !== effectiveEntryDigests.length || !Array.isArray(compatibility.entries) || compatibility.entries.length !== 0
|
|
432
|
+
|| compatibility.messages !== undefined
|
|
433
|
+
|| typeof compatibility.cropped !== "boolean" || compatibility.sourceDigest !== c.sourceDigest || compatibility.effectiveDigest !== c.effectiveDigest
|
|
434
|
+
|| compatibility.converterVersion !== c.converterVersion || compatibility.schemaVersion !== c.schemaVersion
|
|
435
|
+
|| typeof c.observedAt !== "number" || !Number.isFinite(c.observedAt) || c.observedAt < 0
|
|
436
|
+
|| (c.commonPrefix as number) < 0 || (c.commonPrefix as number) > sourceEntryDigests.length) return undefined;
|
|
437
|
+
return { commitDigest, commit: c as unknown as DerivedCommit, object: o as unknown as DerivedObject };
|
|
438
|
+
} catch { return undefined; }
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
private headGenerationNumbers(): number[] {
|
|
442
|
+
try { return requireDirectory(join(this.root, "heads")).flatMap(name => /^([0-9]{16})-[a-f0-9]{64}$/.exec(name)?.[1]).map(Number).filter(Number.isSafeInteger); }
|
|
443
|
+
catch { return []; }
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
private recoveredGenerations(): RecoveredGeneration[] {
|
|
447
|
+
const names = (() => { try { return requireDirectory(join(this.root, "heads")); } catch { return []; } })()
|
|
448
|
+
.filter(name => /^\d{16}-[a-f0-9]{64}$/.test(name)).sort().reverse();
|
|
449
|
+
const values: RecoveredGeneration[] = [];
|
|
450
|
+
for (const name of names) try {
|
|
451
|
+
const digestValue = name.slice(17); if (safeFileText(join(this.root, "heads", name), 1024).trim() !== digestValue) continue;
|
|
452
|
+
const value = this.validate(digestValue); if (value && value.commit.generation === Number(name.slice(0, 16))) { values.push(value); if (values.length >= this.scanLimit) break; }
|
|
453
|
+
} catch { /* skip poisoned head */ }
|
|
454
|
+
return values;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/** CURRENT is tried only as an optimization. Immutable heads are the recovery authority. */
|
|
458
|
+
recover(branchId?: string): RecoveredGeneration | undefined {
|
|
459
|
+
const hinted = this.currentCandidate();
|
|
460
|
+
const hintedGeneration = hinted ? this.validate(hinted) : undefined;
|
|
461
|
+
let hint: RecoveredGeneration | undefined;
|
|
462
|
+
if (hintedGeneration) {
|
|
463
|
+
const head = join(this.root, "heads", `${String(hintedGeneration.commit.generation).padStart(16, "0")}-${hintedGeneration.commitDigest}`);
|
|
464
|
+
try { if (safeFileText(head).trim() === hintedGeneration.commitDigest) hint = hintedGeneration; } catch { /* CURRENT is not authority */ }
|
|
465
|
+
}
|
|
466
|
+
const names = (() => { try { return requireDirectory(join(this.root, "heads")); } catch { return []; } })()
|
|
467
|
+
.filter((name) => /^\d{16}-[a-f0-9]{64}$/.test(name)).sort().reverse();
|
|
468
|
+
let best = hint && (!branchId || hint.object.compatibility.branchId === branchId) ? hint : undefined;
|
|
469
|
+
for (const name of names) {
|
|
470
|
+
const namedGeneration = Number(name.slice(0, 16));
|
|
471
|
+
const namedDigest = name.slice(17);
|
|
472
|
+
let headValid = false;
|
|
473
|
+
try { headValid = safeFileText(join(this.root, "heads", name)).trim() === namedDigest; } catch { /* ignored corruption */ }
|
|
474
|
+
const candidate = headValid ? this.validate(namedDigest) : undefined;
|
|
475
|
+
if (candidate && candidate.commit.generation === namedGeneration && (!branchId || candidate.object.compatibility.branchId === branchId) && (!best || candidate.commit.generation > best.commit.generation)) best = candidate;
|
|
476
|
+
}
|
|
477
|
+
return best;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
async publish(input: PublishInput): Promise<PublishResult> {
|
|
481
|
+
const release = await this.acquire();
|
|
482
|
+
try {
|
|
483
|
+
this.cleanup();
|
|
484
|
+
const current = this.recover();
|
|
485
|
+
if (current) this.pruneGenerations(current.commitDigest);
|
|
486
|
+
return this.publishLocked(input);
|
|
487
|
+
} finally { release(); }
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
private writeImmutable(path: string, contents: string, syncAfter = true): void {
|
|
491
|
+
if (existsSync(path)) {
|
|
492
|
+
try { if (safeFileText(path) === contents) return; } catch { /* quarantine below */ }
|
|
493
|
+
const quarantined = join(this.root, "quarantine", `${this.now()}-${randomBytes(8).toString("hex")}-${path.slice(path.lastIndexOf(sep) + 1)}`);
|
|
494
|
+
renameSync(path, quarantined); syncDirectory(dirname(path)); syncDirectory(join(this.root, "quarantine"));
|
|
495
|
+
}
|
|
496
|
+
atomicWrite(path, contents, false, undefined, syncAfter);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
private readPrimeIndex(): PrimeIndexCursor | undefined {
|
|
500
|
+
const path = join(this.root, "indexes", "prime-jsonl.json");
|
|
501
|
+
try {
|
|
502
|
+
const raw = safeFileText(path, this.maxIndexBytes); const parsed = object(JSON.parse(raw));
|
|
503
|
+
if (!parsed || parsed.version !== "prime-agent-dsh/prime-jsonl-index-v1" || parsed.bindingDigest !== this.bindingDigest
|
|
504
|
+
|| !object(parsed.identity) || !Number.isSafeInteger((parsed.identity as Record<string, unknown>).dev)
|
|
505
|
+
|| !Number.isSafeInteger((parsed.identity as Record<string, unknown>).ino)
|
|
506
|
+
|| !Number.isSafeInteger(parsed.byteOffset) || (parsed.byteOffset as number) < 0
|
|
507
|
+
|| !safeInteger(parsed.nextLine) || typeof parsed.mtimeMs !== "number" || !Number.isFinite(parsed.mtimeMs)
|
|
508
|
+
|| typeof parsed.sourceDigest !== "string" || !SHA256.test(parsed.sourceDigest)
|
|
509
|
+
|| !Array.isArray(parsed.entries) || !parsed.entries.every(validPrimeLine)
|
|
510
|
+
|| typeof parsed.checksum !== "string" || !SHA256.test(parsed.checksum)) return undefined;
|
|
511
|
+
const { checksum, ...unsigned } = parsed;
|
|
512
|
+
if (digest(unsigned) !== checksum || indexedDigest(parsed.entries) !== parsed.sourceDigest) return undefined;
|
|
513
|
+
const entries = parsed.entries;
|
|
514
|
+
let previousEnd = 0, previousLine = 0;
|
|
515
|
+
for (const entry of entries) {
|
|
516
|
+
if (entry.offset < previousEnd || entry.line <= previousLine || entry.offset + entry.length > Number(parsed.byteOffset)) return undefined;
|
|
517
|
+
previousEnd = entry.offset + entry.length; previousLine = entry.line;
|
|
518
|
+
}
|
|
519
|
+
return parsed as unknown as PrimeIndexCursor;
|
|
520
|
+
} catch { return undefined; }
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
private primeIndex(): { entries: PrimeLine[]; diagnostics: PrimeIndexDiagnostics } {
|
|
524
|
+
const path = this.binding.primeSessionFile; const fd = openSync(path, constants.O_RDONLY);
|
|
525
|
+
try {
|
|
526
|
+
const before = fstatSync(fd);
|
|
527
|
+
if (!before.isFile() || !Number.isSafeInteger(before.size)) throw new Error("unsafe or oversized Prime session file");
|
|
528
|
+
const prior = this.readPrimeIndex();
|
|
529
|
+
const identityMatches = !!prior && prior.identity.dev === before.dev && prior.identity.ino === before.ino;
|
|
530
|
+
let mode: PrimeIndexDiagnostics["mode"] = "full", offset = 0, firstLine = 1, entries: PrimeLine[] = [];
|
|
531
|
+
if (identityMatches && prior.byteOffset === before.size && prior.mtimeMs === before.mtimeMs) {
|
|
532
|
+
const after = fstatSync(fd);
|
|
533
|
+
if (after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeMs !== before.mtimeMs) {
|
|
534
|
+
throw new Error("Prime JSONL changed while locators were built");
|
|
535
|
+
}
|
|
536
|
+
return { entries: prior.entries, diagnostics: { mode: "cache", bytesProcessed: 0, linesProcessed: 0 } };
|
|
537
|
+
}
|
|
538
|
+
if (identityMatches && prior.byteOffset < before.size) {
|
|
539
|
+
const endedAtLineBoundary = prior.byteOffset === 0 || readRange(fd, prior.byteOffset - 1, 1)[0] === 0x0a;
|
|
540
|
+
if (endedAtLineBoundary) { mode = "tail"; offset = prior.byteOffset; firstLine = prior.nextLine; entries = prior.entries.slice(); }
|
|
541
|
+
}
|
|
542
|
+
const raw = readRange(fd, offset, before.size - offset);
|
|
543
|
+
const parsed = parsePrimeBytes(raw, offset, firstLine); entries.push(...parsed.entries);
|
|
544
|
+
const after = fstatSync(fd);
|
|
545
|
+
if (after.dev !== before.dev || after.ino !== before.ino || after.size !== before.size || after.mtimeMs !== before.mtimeMs) {
|
|
546
|
+
throw new Error("Prime JSONL changed while locators were built");
|
|
547
|
+
}
|
|
548
|
+
const unsigned = {
|
|
549
|
+
version: "prime-agent-dsh/prime-jsonl-index-v1" as const, bindingDigest: this.bindingDigest,
|
|
550
|
+
identity: { dev: before.dev, ino: before.ino }, byteOffset: before.size, nextLine: parsed.nextLine,
|
|
551
|
+
mtimeMs: before.mtimeMs, sourceDigest: indexedDigest(entries), entries,
|
|
552
|
+
};
|
|
553
|
+
const cursor: PrimeIndexCursor = { ...unsigned, checksum: digest(unsigned) };
|
|
554
|
+
const text = `${canonical(cursor)}\n`;
|
|
555
|
+
if (Buffer.byteLength(text, "utf8") > this.maxIndexBytes) throw new DurablePublicationUnavailableError(`Prime JSONL index exceeds ${this.maxIndexBytes} bytes`);
|
|
556
|
+
atomicWrite(join(this.root, "indexes", "prime-jsonl.json"), text, true);
|
|
557
|
+
return { entries, diagnostics: { mode, bytesProcessed: raw.length, linesProcessed: parsed.entries.length } };
|
|
558
|
+
} finally { closeSync(fd); }
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
private locateSource(source: readonly Json[]): { locators: SourceLocator[]; diagnostics: PrimeIndexDiagnostics } {
|
|
562
|
+
let indexed = this.primeIndex();
|
|
563
|
+
const locate = (): SourceLocator[] | undefined => {
|
|
564
|
+
const used = new Set<number>(); const result: SourceLocator[] = [];
|
|
565
|
+
for (let index = 0; index < source.length; index++) {
|
|
566
|
+
const entry = source[index]!; const entryDigest = digest(entry); const entryRecord = object(entry);
|
|
567
|
+
const entryId = typeof entryRecord?.id === "string" ? entryRecord.id : undefined;
|
|
568
|
+
let found = -1;
|
|
569
|
+
if (entryId !== undefined) found = indexed.entries.findIndex((line, n) => !used.has(n) && line.id === entryId && line.digest === entryDigest);
|
|
570
|
+
if (found < 0) found = indexed.entries.findIndex((line, n) => !used.has(n) && line.digest === entryDigest);
|
|
571
|
+
if (found < 0) return undefined;
|
|
572
|
+
used.add(found); const line = indexed.entries[found];
|
|
573
|
+
result.push({ index, byteOffset: line.offset, byteLength: line.length, line: line.line, entryDigest, ...(entryId === undefined ? {} : { entryId }) });
|
|
574
|
+
}
|
|
575
|
+
return result;
|
|
576
|
+
};
|
|
577
|
+
let locators = locate();
|
|
578
|
+
if (!locators && indexed.diagnostics.mode !== "full") {
|
|
579
|
+
try { unlinkSync(join(this.root, "indexes", "prime-jsonl.json")); } catch { /* absent index */ }
|
|
580
|
+
indexed = this.primeIndex(); locators = locate();
|
|
581
|
+
}
|
|
582
|
+
if (!locators) throw new Error("source entry is not present in the bound Prime JSONL");
|
|
583
|
+
return { locators, diagnostics: indexed.diagnostics };
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
private publishLocked(input: PublishInput): PublishResult {
|
|
587
|
+
if (!input.converterVersion || !input.schemaVersion) throw new Error("converterVersion and schemaVersion are required");
|
|
588
|
+
const source = input.source.map((item) => JSON.parse(canonical(item)) as Json);
|
|
589
|
+
const effective = input.effective.map((item) => JSON.parse(canonical(item)) as Json);
|
|
590
|
+
const sourceDigest = digest(source), effectiveDigest = digest(effective);
|
|
591
|
+
const sourceEntryDigests = source.map(digest);
|
|
592
|
+
const effectiveEntryDigests = effective.map(digest);
|
|
593
|
+
const located = this.locateSource(source);
|
|
594
|
+
const sourceLocators = located.locators;
|
|
595
|
+
const allRecovered = this.recoveredGenerations();
|
|
596
|
+
const branchId = input.branchId ?? "root";
|
|
597
|
+
const isAncestor = (value: RecoveredGeneration): boolean => value.object.sourceEntryDigests.length <= sourceEntryDigests.length
|
|
598
|
+
&& value.object.effectiveEntryDigests.length <= effectiveEntryDigests.length
|
|
599
|
+
&& value.object.sourceEntryDigests.every((item, index) => item === sourceEntryDigests[index])
|
|
600
|
+
&& value.object.effectiveEntryDigests.every((item, index) => item === effectiveEntryDigests[index]);
|
|
601
|
+
const ancestors = allRecovered.filter(isAncestor).sort((a, b) => b.object.sourceEntryDigests.length - a.object.sourceEntryDigests.length || b.commit.generation - a.commit.generation);
|
|
602
|
+
const prior = ancestors[0] ?? allRecovered.find(value => value.object.compatibility.branchId === branchId);
|
|
603
|
+
const highWaterPath = join(this.root, "GENERATION");
|
|
604
|
+
let highWater = 0;
|
|
605
|
+
try { const value = Number(safeFileText(highWaterPath, 128).trim()); if (Number.isSafeInteger(value) && value >= 0) highWater = value; } catch { /* migrate from pre-high-water stores */ }
|
|
606
|
+
const nextGeneration = Math.max(highWater, 0, ...this.headGenerationNumbers()) + 1;
|
|
607
|
+
const same = prior && prior.commit.sourceDigest === sourceDigest && prior.commit.effectiveDigest === effectiveDigest
|
|
608
|
+
&& prior.commit.converterVersion === input.converterVersion && prior.commit.schemaVersion === input.schemaVersion
|
|
609
|
+
&& prior.object.compatibility.branchId === branchId;
|
|
610
|
+
if (same) return { ...prior, mode: "noop", publication: {
|
|
611
|
+
source: { mode: "noop", reused: sourceEntryDigests.length, new: 0, reindexed: 0 },
|
|
612
|
+
index: located.diagnostics,
|
|
613
|
+
effective: { reused: effectiveEntryDigests.length, new: 0, reindexed: 0, rebuildReason: "none" },
|
|
614
|
+
} };
|
|
615
|
+
let prefix = 0;
|
|
616
|
+
if (prior && prior.commit.converterVersion === input.converterVersion && prior.commit.schemaVersion === input.schemaVersion) {
|
|
617
|
+
const old = prior.object.sourceEntryDigests;
|
|
618
|
+
while (prefix < old.length && prefix < sourceEntryDigests.length && old[prefix] === sourceEntryDigests[prefix]) prefix++;
|
|
619
|
+
}
|
|
620
|
+
let effectivePrefix = 0;
|
|
621
|
+
if (prior) {
|
|
622
|
+
const oldEffective = prior.object.effectiveEntryDigests;
|
|
623
|
+
while (effectivePrefix < oldEffective.length && effectivePrefix < effectiveEntryDigests.length && oldEffective[effectivePrefix] === effectiveEntryDigests[effectivePrefix]) effectivePrefix++;
|
|
624
|
+
}
|
|
625
|
+
const sourceAppend = !!prior && prefix === prior.object.sourceEntryDigests.length && sourceEntryDigests.length > prefix;
|
|
626
|
+
const append = sourceAppend && effectivePrefix === prior.object.effectiveEntryDigests.length;
|
|
627
|
+
const sourceCounts = reuseCounts(prior?.object.sourceEntryDigests ?? [], sourceEntryDigests);
|
|
628
|
+
const effectiveCounts = reuseCounts(prior?.object.effectiveEntryDigests ?? [], effectiveEntryDigests);
|
|
629
|
+
const effectiveReason: EffectiveProjectionRebuildReason = !prior ? "initial"
|
|
630
|
+
: prior.commit.converterVersion !== input.converterVersion || prior.commit.schemaVersion !== input.schemaVersion ? "converter-change"
|
|
631
|
+
: prefix < prior.object.sourceEntryDigests.length ? "source-diverged"
|
|
632
|
+
: sourceAppend && effectivePrefix < prior.object.effectiveEntryDigests.length ? "source-append-effective-projection-change"
|
|
633
|
+
: effectivePrefix < prior.object.effectiveEntryDigests.length ? "effective-diverged" : "none";
|
|
634
|
+
const publication: PublicationDiagnostics = {
|
|
635
|
+
source: { mode: sourceAppend ? "append" : "rebuild", ...sourceCounts },
|
|
636
|
+
index: located.diagnostics,
|
|
637
|
+
effective: { ...effectiveCounts, rebuildReason: effectiveReason },
|
|
638
|
+
};
|
|
639
|
+
// V3 stores only verified references into Prime JSONL. Compatibility input is
|
|
640
|
+
// deliberately ignored because it may contain cropped copies of secret text.
|
|
641
|
+
if (input.effectiveSourceIndexes && input.effectiveSourceIndexes.length !== effective.length) {
|
|
642
|
+
throw new Error("effective source index count does not match effective messages");
|
|
643
|
+
}
|
|
644
|
+
const effectiveReferences: EffectiveReference[] = effective.map((entry, index) => {
|
|
645
|
+
let sourceIndex = input.effectiveSourceIndexes?.[index] ?? null;
|
|
646
|
+
if (sourceIndex !== null && (!Number.isSafeInteger(sourceIndex) || sourceIndex < 0 || sourceIndex >= source.length)) {
|
|
647
|
+
throw new Error(`invalid effective source index at ${index}`);
|
|
648
|
+
}
|
|
649
|
+
if (sourceIndex === null) {
|
|
650
|
+
if (sourceEntryDigests[index] === effectiveEntryDigests[index]) sourceIndex = index;
|
|
651
|
+
else if (source.length === effective.length) {
|
|
652
|
+
const nested = object(source[index])?.message;
|
|
653
|
+
if (nested !== undefined && digest(nested) === effectiveEntryDigests[index]) sourceIndex = index;
|
|
654
|
+
} else {
|
|
655
|
+
const exact = sourceEntryDigests.indexOf(effectiveEntryDigests[index]);
|
|
656
|
+
if (exact >= 0) sourceIndex = exact;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
const role = object(entry)?.role;
|
|
660
|
+
return { entryDigest: effectiveEntryDigests[index], sourceIndex, ...(typeof role === "string" ? { role } : {}) };
|
|
661
|
+
});
|
|
662
|
+
const metrics = input.compatibilityMetrics
|
|
663
|
+
? JSON.parse(canonical(input.compatibilityMetrics)) as Record<string, number>
|
|
664
|
+
: undefined;
|
|
665
|
+
const derived: DerivedObject = {
|
|
666
|
+
version: DURABLE_OBJECT_VERSION, bindingDigest: this.bindingDigest, sourceDigest, effectiveDigest,
|
|
667
|
+
sourceEntryDigests, effectiveEntryDigests, sourceLocators, effectiveReferences,
|
|
668
|
+
compatibility: {
|
|
669
|
+
version: DURABLE_STORE_VERSION, sessionId: this.binding.sessionId, branchId,
|
|
670
|
+
revision: nextGeneration, messageCount: effective.length, entries: [], cropped: input.cropped === true,
|
|
671
|
+
sourceDigest, effectiveDigest, converterVersion: input.converterVersion, schemaVersion: input.schemaVersion,
|
|
672
|
+
...(metrics ? { metrics } : {}),
|
|
673
|
+
},
|
|
674
|
+
};
|
|
675
|
+
const objectText = canonical(derived);
|
|
676
|
+
const objectBytes = Buffer.byteLength(objectText, "utf8") + 1;
|
|
677
|
+
if (objectBytes > this.maxObjectBytes) throw new DurablePublicationUnavailableError(`derived context object exceeds ${this.maxObjectBytes} bytes`);
|
|
678
|
+
this.assertPublicationCapacity(objectBytes + 16 * 1024);
|
|
679
|
+
// Persist the allocation before any generation content. Gaps are safe and
|
|
680
|
+
// ensure cleanup/corruption can never cause a generation identity rewind.
|
|
681
|
+
atomicWrite(highWaterPath, `${nextGeneration}\n`, true);
|
|
682
|
+
const objectDigest = createHash("sha256").update(objectText).digest("hex");
|
|
683
|
+
this.writeImmutable(join(this.root, "objects", `${objectDigest}.json`), `${objectText}\n`); this.fault?.("object-durable");
|
|
684
|
+
const commit: DerivedCommit = {
|
|
685
|
+
version: DURABLE_COMMIT_VERSION, bindingDigest: this.bindingDigest, generation: nextGeneration,
|
|
686
|
+
parent: prior?.commitDigest ?? null, object: objectDigest, sourceDigest, effectiveDigest,
|
|
687
|
+
converterVersion: input.converterVersion, schemaVersion: input.schemaVersion, mode: append ? "append" : "rebuild",
|
|
688
|
+
commonPrefix: prefix, observedAt: input.observedAt ?? 0, publication,
|
|
689
|
+
};
|
|
690
|
+
const commitText = canonical(commit), commitDigest = createHash("sha256").update(commitText).digest("hex");
|
|
691
|
+
this.writeImmutable(join(this.root, "commits", `${commitDigest}.json`), `${commitText}\n`); this.fault?.("commit-durable");
|
|
692
|
+
// Validate the complete reference candidate before publishing its immutable head.
|
|
693
|
+
if (!this.validate(commitDigest)) throw new Error("candidate durable context generation failed validation");
|
|
694
|
+
const generation = String(commit.generation).padStart(16, "0");
|
|
695
|
+
this.writeImmutable(join(this.root, "heads", `${generation}-${commitDigest}`), `${commitDigest}\n`); this.fault?.("head-durable");
|
|
696
|
+
atomicWrite(join(this.root, "CURRENT"), `${commitDigest}\n`, true, () => this.fault?.("current-temp-durable")); this.fault?.("current-replaced");
|
|
697
|
+
this.pruneGenerations(commitDigest);
|
|
698
|
+
return { commitDigest, commit, object: derived, mode: commit.mode, publication };
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
private managedBytes(path = this.root): number {
|
|
702
|
+
let total = 0;
|
|
703
|
+
for (const name of requireDirectory(path)) {
|
|
704
|
+
if (name === "LOCK") continue;
|
|
705
|
+
const child = join(path, name);
|
|
706
|
+
const stat = lstatSync(child);
|
|
707
|
+
if (stat.isSymbolicLink()) throw new DurablePublicationUnavailableError(`symlink is not allowed in durable store: ${child}`);
|
|
708
|
+
if (stat.isDirectory()) total += this.managedBytes(child);
|
|
709
|
+
else if (stat.isFile()) total += stat.size;
|
|
710
|
+
}
|
|
711
|
+
return total;
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
private assertPublicationCapacity(candidateBytes: number): void {
|
|
715
|
+
const used = this.managedBytes();
|
|
716
|
+
if (used + candidateBytes > this.maxStoreBytes) {
|
|
717
|
+
throw new DurablePublicationUnavailableError(`durable context quota exceeded (${used + candidateBytes} > ${this.maxStoreBytes} bytes)`);
|
|
718
|
+
}
|
|
719
|
+
const fs = statfsSync(this.root);
|
|
720
|
+
const free = Number(fs.bavail) * Number(fs.bsize);
|
|
721
|
+
if (!Number.isFinite(free) || free - candidateBytes < this.minFreeBytes) {
|
|
722
|
+
throw new DurablePublicationUnavailableError(`durable context publication requires ${this.minFreeBytes} bytes free after write`);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/** Retire the oldest authoritative generations head-first. */
|
|
727
|
+
private pruneGenerations(currentDigest: string): void {
|
|
728
|
+
const heads = requireDirectory(join(this.root, "heads"))
|
|
729
|
+
.filter((name) => /^\d{16}-[a-f0-9]{64}$/.test(name)).sort().reverse();
|
|
730
|
+
const observedHighWater = heads.reduce((maximum, name) => Math.max(maximum, Number(name.slice(0, 16))), 0);
|
|
731
|
+
const highWaterPath = join(this.root, "GENERATION");
|
|
732
|
+
let recordedHighWater = 0;
|
|
733
|
+
try { recordedHighWater = Number(safeFileText(highWaterPath, 128).trim()) || 0; } catch { /* migration */ }
|
|
734
|
+
if (observedHighWater > recordedHighWater) atomicWrite(highWaterPath, `${observedHighWater}\n`, true);
|
|
735
|
+
const keep = new Set<string>([currentDigest]);
|
|
736
|
+
for (const name of heads) {
|
|
737
|
+
if (keep.size >= this.retainGenerations) break;
|
|
738
|
+
const digestValue = name.slice(17);
|
|
739
|
+
if (this.validate(digestValue)) keep.add(digestValue);
|
|
740
|
+
}
|
|
741
|
+
for (const name of heads) {
|
|
742
|
+
const digestValue = name.slice(17);
|
|
743
|
+
if (keep.has(digestValue)) continue;
|
|
744
|
+
try { unlinkSync(join(this.root, "heads", name)); } catch { /* best effort; retry next publication */ }
|
|
745
|
+
}
|
|
746
|
+
syncDirectory(join(this.root, "heads"));
|
|
747
|
+
const objects = new Set<string>();
|
|
748
|
+
for (const name of requireDirectory(join(this.root, "commits"))) {
|
|
749
|
+
const match = /^([a-f0-9]{64})\.json$/.exec(name);
|
|
750
|
+
if (!match) continue;
|
|
751
|
+
const commitDigest = match[1] ?? "";
|
|
752
|
+
const path = join(this.root, "commits", name);
|
|
753
|
+
if (!keep.has(commitDigest)) { try { unlinkSync(path); } catch { /* best effort */ } continue; }
|
|
754
|
+
try { const value = object(parseJson(path)); if (typeof value?.object === "string" && SHA256.test(value.object)) objects.add(value.object); } catch { /* retained corrupt commit has no object authority */ }
|
|
755
|
+
}
|
|
756
|
+
for (const name of requireDirectory(join(this.root, "objects"))) {
|
|
757
|
+
const match = /^([a-f0-9]{64})\.json$/.exec(name);
|
|
758
|
+
if (match && !objects.has(match[1] ?? "")) try { unlinkSync(join(this.root, "objects", name)); } catch { /* best effort */ }
|
|
759
|
+
}
|
|
760
|
+
syncDirectory(join(this.root, "commits")); syncDirectory(join(this.root, "objects"));
|
|
761
|
+
// Compatibility manifests are also rebuildable derived state. Retire only
|
|
762
|
+
// names from this store's strict manifest namespace.
|
|
763
|
+
for (const name of requireDirectory(this.root)) {
|
|
764
|
+
const immutable = /^manifest-[a-f0-9]{64}-([a-f0-9]{64})\.json$/.exec(name);
|
|
765
|
+
const branch = /^manifest-[a-f0-9]{64}\.json$/.exec(name);
|
|
766
|
+
let remove = !!immutable && !keep.has(immutable[1] ?? "");
|
|
767
|
+
if (branch) {
|
|
768
|
+
try { const value = object(parseJson(join(this.root, name))); remove = typeof value?.commit !== "string" || !keep.has(value.commit); }
|
|
769
|
+
catch { remove = true; }
|
|
770
|
+
}
|
|
771
|
+
if (remove) try { unlinkSync(join(this.root, name)); } catch { /* best effort */ }
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
private async acquire(): Promise<() => void> {
|
|
776
|
+
const path = join(this.root, "LOCK"); const deadline = this.now() + this.lockTimeout;
|
|
777
|
+
for (;;) {
|
|
778
|
+
try {
|
|
779
|
+
mkdirSync(path, { mode: 0o700 });
|
|
780
|
+
writeFileSync(join(path, "owner.json"), canonical({ pid: process.pid, started: this.now(), nonce: randomBytes(16).toString("hex") }), { flag: "wx", mode: 0o600 });
|
|
781
|
+
syncDirectory(this.root);
|
|
782
|
+
return () => { try { rmSync(path, { recursive: true, force: true }); syncDirectory(this.root); } catch { /* process exit also releases by staleness */ } };
|
|
783
|
+
} catch (error) {
|
|
784
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
785
|
+
this.breakStaleLock(path);
|
|
786
|
+
if (this.now() >= deadline) throw new Error("timed out acquiring durable context store lock", { cause: error });
|
|
787
|
+
await sleep(10 + Math.floor(Math.random() * 20));
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
private breakStaleLock(path: string): void {
|
|
792
|
+
try {
|
|
793
|
+
const age = this.now() - lstatSync(path).mtimeMs; if (age < this.staleLock) return;
|
|
794
|
+
const owner = object(parseJson(join(path, "owner.json"))); const pid = owner?.pid;
|
|
795
|
+
if (typeof pid === "number" && Number.isSafeInteger(pid)) {
|
|
796
|
+
try { process.kill(pid, 0); return; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") return; }
|
|
797
|
+
}
|
|
798
|
+
renameSync(path, join(this.root, `quarantine`, `stale-lock-${this.now()}-${randomBytes(4).toString("hex")}`)); syncDirectory(this.root);
|
|
799
|
+
} catch { /* another process won, or the lock cannot safely be proven stale */ }
|
|
800
|
+
}
|
|
801
|
+
private cleanup(): void {
|
|
802
|
+
const cutoff = this.now() - this.staleLock;
|
|
803
|
+
for (const directory of [this.root, join(this.root, "objects"), join(this.root, "commits"), join(this.root, "heads")]) {
|
|
804
|
+
for (const name of requireDirectory(directory)) {
|
|
805
|
+
if (!name.startsWith(".tmp-")) continue;
|
|
806
|
+
const path = join(directory, name);
|
|
807
|
+
try { if (lstatSync(path).mtimeMs < cutoff) unlinkSync(path); } catch { /* best effort */ }
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
// A crash can leave a durable object or commit before its immutable head.
|
|
811
|
+
// Only collect files older than the stale window while holding the writer lock.
|
|
812
|
+
const headed = new Set(requireDirectory(join(this.root, "heads"))
|
|
813
|
+
.filter((name) => /^\d{16}-[a-f0-9]{64}$/.test(name)).map((name) => name.slice(17)));
|
|
814
|
+
const referencedObjects = new Set<string>();
|
|
815
|
+
for (const name of requireDirectory(join(this.root, "commits"))) {
|
|
816
|
+
const match = /^([a-f0-9]{64})\.json$/.exec(name); if (!match) continue;
|
|
817
|
+
const path = join(this.root, "commits", name);
|
|
818
|
+
if (!headed.has(match[1] ?? "")) {
|
|
819
|
+
try { if (lstatSync(path).mtimeMs < cutoff) unlinkSync(path); } catch { /* best effort */ }
|
|
820
|
+
continue;
|
|
821
|
+
}
|
|
822
|
+
try { const value = object(parseJson(path)); if (typeof value?.object === "string" && SHA256.test(value.object)) referencedObjects.add(value.object); } catch { /* corrupt commits are ignored */ }
|
|
823
|
+
}
|
|
824
|
+
for (const name of requireDirectory(join(this.root, "objects"))) {
|
|
825
|
+
const match = /^([a-f0-9]{64})\.json$/.exec(name); if (!match || referencedObjects.has(match[1] ?? "")) continue;
|
|
826
|
+
const path = join(this.root, "objects", name);
|
|
827
|
+
try { if (lstatSync(path).mtimeMs < cutoff) unlinkSync(path); } catch { /* best effort */ }
|
|
828
|
+
}
|
|
829
|
+
const quarantine = requireDirectory(join(this.root, "quarantine")).sort().reverse();
|
|
830
|
+
for (const name of quarantine.slice(8)) try { rmSync(join(this.root, "quarantine", name), { recursive: true, force: true }); } catch { /* best effort */ }
|
|
831
|
+
syncDirectory(join(this.root, "objects")); syncDirectory(join(this.root, "commits")); syncDirectory(join(this.root, "quarantine"));
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
function requireDirectory(path: string): string[] {
|
|
836
|
+
const stat = lstatSync(path); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error(`unsafe durable store directory: ${path}`);
|
|
837
|
+
return readdirSync(path);
|
|
838
|
+
}
|