vouchington-tooling 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +32 -0
  2. package/dist/agent-blackboard/index.d.mts +41 -0
  3. package/dist/agent-blackboard/index.mjs +94 -0
  4. package/dist/agent-blackboard/session-id.d.mts +1 -0
  5. package/dist/agent-blackboard/session-id.mjs +5 -0
  6. package/dist/agent-blackboard/snapshot-cleanup-directory.d.mts +11 -0
  7. package/dist/agent-blackboard/snapshot-cleanup-directory.mjs +66 -0
  8. package/dist/agent-blackboard/snapshot-cleanup-key.d.mts +14 -0
  9. package/dist/agent-blackboard/snapshot-cleanup-key.mjs +129 -0
  10. package/dist/agent-blackboard/snapshot-cleanup-receipt.d.mts +6 -0
  11. package/dist/agent-blackboard/snapshot-cleanup-receipt.mjs +117 -0
  12. package/dist/agent-blackboard/snapshot-cleanup-resume.d.mts +16 -0
  13. package/dist/agent-blackboard/snapshot-cleanup-resume.mjs +98 -0
  14. package/dist/agent-blackboard/snapshot-partition-cleanup.d.mts +13 -0
  15. package/dist/agent-blackboard/snapshot-partition-cleanup.mjs +136 -0
  16. package/dist/agent-blackboard/snapshot-partition-format.d.mts +32 -0
  17. package/dist/agent-blackboard/snapshot-partition-format.mjs +151 -0
  18. package/dist/agent-blackboard/snapshot-partition-io.d.mts +4 -0
  19. package/dist/agent-blackboard/snapshot-partition-io.mjs +37 -0
  20. package/dist/agent-blackboard/snapshot-partition-read.d.mts +9 -0
  21. package/dist/agent-blackboard/snapshot-partition-read.mjs +70 -0
  22. package/dist/agent-blackboard/snapshot-partition-validate.d.mts +4 -0
  23. package/dist/agent-blackboard/snapshot-partition-validate.mjs +40 -0
  24. package/dist/agent-blackboard/snapshot-partition-write.d.mts +2 -0
  25. package/dist/agent-blackboard/snapshot-partition-write.mjs +91 -0
  26. package/dist/agent-blackboard/snapshot-partitions.d.mts +10 -0
  27. package/dist/agent-blackboard/snapshot-partitions.mjs +102 -0
  28. package/dist/agent-blackboard/snapshot-types.d.mts +67 -0
  29. package/dist/agent-blackboard/snapshot-types.mjs +1 -0
  30. package/dist/agent-blackboard/snapshot.d.mts +3 -0
  31. package/dist/agent-blackboard/snapshot.mjs +2 -0
  32. package/dist/cli/commands/agent-blackboard.d.mts +3 -0
  33. package/dist/cli/commands/agent-blackboard.mjs +114 -0
  34. package/dist/cli/commands/retrospective-facts.d.mts +1 -0
  35. package/dist/cli/commands/retrospective-facts.mjs +33 -0
  36. package/dist/cli/index.mjs +11 -0
  37. package/dist/cli/parse.d.mts +7 -1
  38. package/dist/cli/parse.mjs +6 -0
  39. package/dist/cli/usage.d.mts +1 -1
  40. package/dist/cli/usage.mjs +12 -0
  41. package/dist/index.d.mts +4 -0
  42. package/dist/index.mjs +2 -0
  43. package/dist/retrospective-facts/exec.d.mts +3 -0
  44. package/dist/retrospective-facts/exec.mjs +19 -0
  45. package/dist/retrospective-facts/foreign.d.mts +2 -0
  46. package/dist/retrospective-facts/foreign.mjs +36 -0
  47. package/dist/retrospective-facts/format.d.mts +10 -0
  48. package/dist/retrospective-facts/format.mjs +64 -0
  49. package/dist/retrospective-facts/index.d.mts +3 -0
  50. package/dist/retrospective-facts/index.mjs +26 -0
  51. package/dist/retrospective-facts/local.d.mts +2 -0
  52. package/dist/retrospective-facts/local.mjs +144 -0
  53. package/dist/retrospective-facts/shared.d.mts +17 -0
  54. package/dist/retrospective-facts/shared.mjs +1 -0
  55. package/package.json +21 -2
  56. package/scripts/gha/harness-admission-lane.sh +26 -0
  57. package/scripts/gha/harness-assert-gates.sh +24 -0
@@ -0,0 +1,13 @@
1
+ import { lstat, readFile, readdir, rename, rm, rmdir } from 'node:fs/promises';
2
+ import type { SnapshotCleanupOptions } from './snapshot-types.mts';
3
+ declare const defaults: {
4
+ lstat: typeof lstat;
5
+ readFile: typeof readFile;
6
+ readdir: typeof readdir;
7
+ rename: typeof rename;
8
+ rm: typeof rm;
9
+ rmdir: typeof rmdir;
10
+ };
11
+ export declare function setSnapshotCleanupFilesystemForTest(overrides?: Partial<typeof defaults>): void;
12
+ export declare function cleanupSnapshotPartitions(options: SnapshotCleanupOptions): Promise<void>;
13
+ export {};
@@ -0,0 +1,136 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { lstat, readFile, readdir, rename, rm, rmdir } from 'node:fs/promises';
3
+ import { tmpdir } from 'node:os';
4
+ import { basename, dirname, isAbsolute, join, resolve } from 'node:path';
5
+ import { verifyCleanupReceipt } from './snapshot-cleanup-receipt.mjs';
6
+ import { removePartitionDirectory } from './snapshot-cleanup-directory.mjs';
7
+ import { removeResumeReceipt, requireResumeReceipt, writeResumeReceipt, } from './snapshot-cleanup-resume.mjs';
8
+ const SOURCE_NAME = /^agent-blackboard-snapshot-[0-9a-f-]{36}\.jsonl$/;
9
+ const DIRECTORY_NAME = /^agent-blackboard-partitions-[A-Za-z0-9]+$/;
10
+ const defaults = { lstat, readFile, readdir, rename, rm, rmdir };
11
+ let filesystem = defaults;
12
+ export function setSnapshotCleanupFilesystemForTest(overrides) {
13
+ filesystem = { ...defaults, ...overrides };
14
+ }
15
+ function assertGeneratedPath(path, expression, label) {
16
+ if (!isAbsolute(path) ||
17
+ dirname(resolve(path)) !== resolve(tmpdir()) ||
18
+ !expression.test(basename(path)))
19
+ throw new Error(`${label} must be a generated temporary ${label}`);
20
+ }
21
+ async function removeCaptured(path, expression, directory, receipt) {
22
+ const label = directory ? 'partition directory' : 'snapshot path';
23
+ assertGeneratedPath(path, expression, label);
24
+ let initial;
25
+ try {
26
+ initial = await filesystem.lstat(path);
27
+ }
28
+ catch (error) {
29
+ if (error.code === 'ENOENT' && directory && receipt)
30
+ return resumeDirectory(path, receipt);
31
+ if (error.code === 'ENOENT')
32
+ return;
33
+ throw error;
34
+ }
35
+ if (initial.isSymbolicLink() ||
36
+ (!initial.isFile() && !directory) ||
37
+ (directory && !initial.isDirectory()) ||
38
+ (!directory && initial.nlink !== 1))
39
+ throw new Error(`${label} is not a generated ${directory ? 'directory' : 'regular file'}`);
40
+ const tombstone = directory && receipt
41
+ ? tombstonePath(receipt)
42
+ : join(tmpdir(), `.agent-blackboard-cleanup-${process.pid}-${randomUUID()}`);
43
+ if (directory)
44
+ await writeResumeReceipt(optionsReceipt(receipt));
45
+ await filesystem.rename(path, tombstone);
46
+ let deleting = false;
47
+ try {
48
+ const captured = await filesystem.lstat(tombstone);
49
+ if (captured.isSymbolicLink() ||
50
+ captured.dev !== initial.dev ||
51
+ captured.ino !== initial.ino ||
52
+ captured.isDirectory() !== directory ||
53
+ (!directory && (!captured.isFile() || captured.nlink !== 1)))
54
+ throw new Error(`${label} changed while it was being removed`);
55
+ if (directory)
56
+ await removePartitionDirectory(filesystem, tombstone, path, optionsReceipt(receipt), captured, false, () => {
57
+ deleting = true;
58
+ });
59
+ else
60
+ await filesystem.rm(tombstone, { force: true });
61
+ }
62
+ catch (error) {
63
+ if (directory && deleting)
64
+ throw new Error(`${label} cleanup failed; retained tombstone ${tombstone} for retry`, {
65
+ cause: error,
66
+ });
67
+ try {
68
+ await filesystem.rename(tombstone, path);
69
+ }
70
+ catch (rollback) {
71
+ throw new AggregateError([error, rollback], `${label} cleanup failed; tombstone ${tombstone} could not be restored to ${path}`);
72
+ }
73
+ const detail = error instanceof Error ? `: ${error.message}` : '';
74
+ throw new Error(`${label} cleanup failed; restored ${path} for retry${detail}`, {
75
+ cause: error,
76
+ });
77
+ }
78
+ }
79
+ function tombstonePath(receipt) {
80
+ return join(tmpdir(), `.agent-blackboard-cleanup-${receipt.token}`);
81
+ }
82
+ async function resumeDirectory(path, receipt) {
83
+ const tombstone = tombstonePath(receipt);
84
+ let marker = true;
85
+ try {
86
+ await requireResumeReceipt(receipt);
87
+ }
88
+ catch (error) {
89
+ if (error.code !== 'ENOENT')
90
+ throw error;
91
+ marker = false;
92
+ }
93
+ let info;
94
+ try {
95
+ info = await filesystem.lstat(tombstone);
96
+ }
97
+ catch (error) {
98
+ if (error.code !== 'ENOENT')
99
+ throw error;
100
+ if (marker)
101
+ await removeResumeReceipt(receipt);
102
+ return;
103
+ }
104
+ if (!marker)
105
+ throw new Error('partition directory tombstone has no resume metadata');
106
+ if (!info.isDirectory() || info.isSymbolicLink())
107
+ throw new Error('partition directory tombstone is unsafe');
108
+ try {
109
+ await removePartitionDirectory(filesystem, tombstone, path, receipt, info, true, () => undefined);
110
+ }
111
+ catch (error) {
112
+ throw new Error(`partition directory cleanup failed; retained tombstone ${tombstone} for retry`, { cause: error });
113
+ }
114
+ }
115
+ function optionsReceipt(receipt) {
116
+ if (!receipt)
117
+ throw new Error('partition directory cleanup requires a receipt');
118
+ return receipt;
119
+ }
120
+ export async function cleanupSnapshotPartitions(options) {
121
+ if (!options.path && !options.directory)
122
+ throw new Error('cleanup requires a snapshot path or partition directory');
123
+ if (options.directory)
124
+ await verifyCleanupReceipt(optionsReceipt(options.receipt));
125
+ const results = await Promise.allSettled([
126
+ ...(options.path ? [removeCaptured(options.path, SOURCE_NAME, false)] : []),
127
+ ...(options.directory
128
+ ? [removeCaptured(options.directory, DIRECTORY_NAME, true, options.receipt)]
129
+ : []),
130
+ ]);
131
+ const failures = results.filter((result) => result.status === 'rejected');
132
+ if (failures.length) {
133
+ const reasons = failures.map((result) => String(result.reason)).join('; ');
134
+ throw new AggregateError(failures.map((result) => result.reason), `snapshot cleanup failed: ${reasons}`);
135
+ }
136
+ }
@@ -0,0 +1,32 @@
1
+ import type { SnapshotCounts, SnapshotManifest } from './snapshot-types.mts';
2
+ export type SnapshotBlock = {
3
+ sessionId: string;
4
+ path: string;
5
+ bytes: number;
6
+ sessions: number;
7
+ entries: number;
8
+ };
9
+ export type SnapshotRecord = {
10
+ type: 'session';
11
+ session: Record<string, unknown>;
12
+ } | {
13
+ type: 'entry';
14
+ entry: Record<string, unknown>;
15
+ } | {
16
+ type: 'manifest';
17
+ manifest: SnapshotManifest;
18
+ };
19
+ export type SnapshotState = {
20
+ sessions: number;
21
+ entries: number;
22
+ records: number;
23
+ lastSessionCreatedAt?: number;
24
+ lastEntryCreatedAt?: number;
25
+ currentSessionId?: string;
26
+ };
27
+ export declare function snapshotLine(value: unknown): string;
28
+ export declare function countsFor(block: Pick<SnapshotBlock, 'sessions' | 'entries'>, bytes: number): SnapshotCounts;
29
+ export declare function manifestFor(manifest: SnapshotManifest, block: Pick<SnapshotBlock, 'sessions' | 'entries'>): SnapshotManifest;
30
+ export declare function parseSnapshotRecord(line: string): SnapshotRecord;
31
+ export declare function consumeSnapshotRecord(record: SnapshotRecord, state: SnapshotState): void;
32
+ export declare function assertManifest(manifest: SnapshotManifest, state: SnapshotState): void;
@@ -0,0 +1,151 @@
1
+ import { assertSessionId } from './session-id.mjs';
2
+ function isObject(value) {
3
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
4
+ }
5
+ function isTimestamp(value) {
6
+ return typeof value === 'string' && Number.isFinite(Date.parse(value));
7
+ }
8
+ function isCount(value) {
9
+ return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
10
+ }
11
+ function isSelection(value) {
12
+ if (!isObject(value) || value.archived !== false)
13
+ return false;
14
+ if (value.agent !== undefined && typeof value.agent !== 'string')
15
+ return false;
16
+ if (value.version !== undefined && typeof value.version !== 'string')
17
+ return false;
18
+ if (value.parentSessionId !== undefined &&
19
+ value.parentSessionId !== null &&
20
+ typeof value.parentSessionId !== 'string')
21
+ return false;
22
+ if (value.data !== undefined && !isObject(value.data))
23
+ return false;
24
+ return (value.inactiveForHours === undefined ||
25
+ (typeof value.inactiveForHours === 'number' &&
26
+ Number.isFinite(value.inactiveForHours) &&
27
+ value.inactiveForHours > 0));
28
+ }
29
+ function isManifest(value) {
30
+ if (!isObject(value) || value.schemaVersion !== 1 || value.status !== 'complete')
31
+ return false;
32
+ if (!isTimestamp(value.createdAt) ||
33
+ !isTimestamp(value.completedAt) ||
34
+ !isSelection(value.selection))
35
+ return false;
36
+ if (!isObject(value.counts) ||
37
+ !isCount(value.counts.sessions) ||
38
+ !isCount(value.counts.entries) ||
39
+ !isCount(value.counts.records))
40
+ return false;
41
+ return (isObject(value.ordering) &&
42
+ value.ordering.sessions === 'createdAt ascending' &&
43
+ value.ordering.entries === 'createdAt ascending within session' &&
44
+ value.consistency === 'best-effort');
45
+ }
46
+ function isSession(value) {
47
+ if (isObject(value) &&
48
+ typeof value.id === 'string' &&
49
+ value.id.length > 0 &&
50
+ (value.parentSessionId === null || typeof value.parentSessionId === 'string') &&
51
+ typeof value.agent === 'string' &&
52
+ typeof value.version === 'string' &&
53
+ isTimestamp(value.createdAt) &&
54
+ (value.lastEntryAt === null || isTimestamp(value.lastEntryAt)) &&
55
+ value.archivedAt === null &&
56
+ isObject(value.data)) {
57
+ try {
58
+ assertSessionId(value.id);
59
+ if (value.parentSessionId !== null)
60
+ assertSessionId(value.parentSessionId, 'parent session id');
61
+ return true;
62
+ }
63
+ catch {
64
+ return false;
65
+ }
66
+ }
67
+ return false;
68
+ }
69
+ function isEntry(value) {
70
+ if (isObject(value) &&
71
+ typeof value.sessionId === 'string' &&
72
+ value.sessionId.length > 0 &&
73
+ isTimestamp(value.createdAt) &&
74
+ isObject(value.data)) {
75
+ try {
76
+ assertSessionId(value.sessionId);
77
+ return true;
78
+ }
79
+ catch {
80
+ return false;
81
+ }
82
+ }
83
+ return false;
84
+ }
85
+ export function snapshotLine(value) {
86
+ return `${JSON.stringify(value)}\n`;
87
+ }
88
+ export function countsFor(block, bytes) {
89
+ return {
90
+ sessions: block.sessions,
91
+ entries: block.entries,
92
+ records: block.sessions + block.entries + 1,
93
+ bytes,
94
+ };
95
+ }
96
+ export function manifestFor(manifest, block) {
97
+ const counts = countsFor(block, 0);
98
+ return {
99
+ ...manifest,
100
+ counts: { sessions: counts.sessions, entries: counts.entries, records: counts.records },
101
+ };
102
+ }
103
+ export function parseSnapshotRecord(line) {
104
+ let value;
105
+ try {
106
+ value = JSON.parse(line);
107
+ }
108
+ catch {
109
+ throw new Error('snapshot contains invalid JSONL');
110
+ }
111
+ if (!isObject(value))
112
+ throw new Error('snapshot contains an invalid record');
113
+ if (value.type === 'session' && isSession(value.session))
114
+ return { type: 'session', session: value.session };
115
+ if (value.type === 'entry' && isEntry(value.entry))
116
+ return { type: 'entry', entry: value.entry };
117
+ if (value.type === 'manifest' && isObject(value.manifest))
118
+ return { type: 'manifest', manifest: value.manifest };
119
+ throw new Error('snapshot contains an unsupported record');
120
+ }
121
+ export function consumeSnapshotRecord(record, state) {
122
+ if (record.type === 'manifest')
123
+ return;
124
+ if (record.type === 'session') {
125
+ const createdAt = Date.parse(record.session.createdAt);
126
+ if (state.lastSessionCreatedAt !== undefined && createdAt < state.lastSessionCreatedAt)
127
+ throw new Error('snapshot sessions are not ordered');
128
+ state.sessions += 1;
129
+ state.records += 1;
130
+ state.lastSessionCreatedAt = createdAt;
131
+ delete state.lastEntryCreatedAt;
132
+ state.currentSessionId = record.session.id;
133
+ return;
134
+ }
135
+ const createdAt = Date.parse(record.entry.createdAt);
136
+ if (state.currentSessionId !== record.entry.sessionId)
137
+ throw new Error('snapshot entries must follow their session');
138
+ if (state.lastEntryCreatedAt !== undefined && createdAt < state.lastEntryCreatedAt)
139
+ throw new Error('snapshot entries are not ordered');
140
+ state.entries += 1;
141
+ state.records += 1;
142
+ state.lastEntryCreatedAt = createdAt;
143
+ }
144
+ export function assertManifest(manifest, state) {
145
+ if (!isManifest(manifest))
146
+ throw new Error('snapshot is missing a complete terminal manifest');
147
+ if (manifest.counts.sessions !== state.sessions ||
148
+ manifest.counts.entries !== state.entries ||
149
+ manifest.counts.records !== state.records + 1)
150
+ throw new Error('snapshot manifest counts do not match its records');
151
+ }
@@ -0,0 +1,4 @@
1
+ import { createHash } from 'node:crypto';
2
+ import type { FileHandle } from 'node:fs/promises';
3
+ export declare function writeAll(file: FileHandle, bytes: Uint8Array): Promise<void>;
4
+ export declare function readLines(file: FileHandle, hash?: ReturnType<typeof createHash>, onRead?: (bytes: number) => void): AsyncGenerator<string>;
@@ -0,0 +1,37 @@
1
+ import { createHash } from 'node:crypto';
2
+ const CHUNK_SIZE = 64 * 1024;
3
+ export async function writeAll(file, bytes) {
4
+ let offset = 0;
5
+ while (offset < bytes.byteLength) {
6
+ const { bytesWritten } = await file.write(bytes, offset);
7
+ if (bytesWritten === 0)
8
+ throw new Error('could not write snapshot partition');
9
+ offset += bytesWritten;
10
+ }
11
+ }
12
+ export async function* readLines(file, hash, onRead) {
13
+ const buffer = Buffer.allocUnsafe(CHUNK_SIZE);
14
+ const decoder = new TextDecoder('utf-8', { fatal: true });
15
+ let pending = '';
16
+ let position = 0;
17
+ for (;;) {
18
+ const { bytesRead } = await file.read(buffer, 0, buffer.byteLength, position);
19
+ if (!bytesRead)
20
+ break;
21
+ position += bytesRead;
22
+ const bytes = buffer.subarray(0, bytesRead);
23
+ hash?.update(bytes);
24
+ onRead?.(bytesRead);
25
+ pending += decoder.decode(bytes, { stream: true });
26
+ for (;;) {
27
+ const newline = pending.indexOf('\n');
28
+ if (newline < 0)
29
+ break;
30
+ yield pending.slice(0, newline);
31
+ pending = pending.slice(newline + 1);
32
+ }
33
+ }
34
+ pending += decoder.decode();
35
+ if (pending)
36
+ yield pending;
37
+ }
@@ -0,0 +1,9 @@
1
+ import type { FileHandle } from 'node:fs/promises';
2
+ import type { SnapshotManifest } from './snapshot-types.mts';
3
+ export interface StagedSnapshot {
4
+ manifest: SnapshotManifest;
5
+ bytes: number;
6
+ checksum: string;
7
+ index: string;
8
+ }
9
+ export declare function stageSnapshot(source: FileHandle, directory: string): Promise<StagedSnapshot>;
@@ -0,0 +1,70 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { open } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
+ import { assertManifest, consumeSnapshotRecord, parseSnapshotRecord, snapshotLine, } from './snapshot-partition-format.mjs';
5
+ import { readLines, writeAll } from './snapshot-partition-io.mjs';
6
+ export async function stageSnapshot(source, directory) {
7
+ const index = join(directory, 'index.jsonl');
8
+ const indexFile = await open(index, 'wx', 0o600);
9
+ const hash = createHash('sha256');
10
+ const state = { sessions: 0, entries: 0, records: 0 };
11
+ let current;
12
+ let manifest;
13
+ let ordinal = 0;
14
+ let bytes = 0;
15
+ const finish = async () => {
16
+ if (!current)
17
+ return;
18
+ await current.file.sync();
19
+ await current.file.close();
20
+ const { file: _file, ...block } = current;
21
+ await writeAll(indexFile, Buffer.from(snapshotLine(block)));
22
+ current = undefined;
23
+ };
24
+ try {
25
+ for await (const sourceLine of readLines(source, hash, (read) => {
26
+ bytes += read;
27
+ })) {
28
+ if (!sourceLine)
29
+ throw new Error('snapshot contains a blank JSONL record');
30
+ if (manifest)
31
+ throw new Error('snapshot contains records after its manifest');
32
+ const record = parseSnapshotRecord(sourceLine);
33
+ if (record.type === 'manifest') {
34
+ await finish();
35
+ manifest = record.manifest;
36
+ assertManifest(manifest, state);
37
+ continue;
38
+ }
39
+ if (record.type === 'session') {
40
+ await finish();
41
+ ordinal += 1;
42
+ const path = join(directory, `session-${ordinal}.jsonl`);
43
+ current = {
44
+ sessionId: record.session.id,
45
+ path,
46
+ bytes: 0,
47
+ sessions: 1,
48
+ entries: 0,
49
+ file: await open(path, 'wx', 0o600),
50
+ };
51
+ }
52
+ else if (!current)
53
+ throw new Error('snapshot entries must follow their session');
54
+ consumeSnapshotRecord(record, state);
55
+ if (record.type === 'entry')
56
+ current.entries += 1;
57
+ const recordBytes = Buffer.from(`${sourceLine}\n`);
58
+ await writeAll(current.file, recordBytes);
59
+ current.bytes += recordBytes.byteLength;
60
+ }
61
+ if (!manifest)
62
+ throw new Error('snapshot is missing a complete terminal manifest');
63
+ await indexFile.sync();
64
+ return { manifest, bytes, checksum: hash.digest('hex'), index };
65
+ }
66
+ finally {
67
+ await current?.file.close().catch(() => undefined);
68
+ await indexFile.close().catch(() => undefined);
69
+ }
70
+ }
@@ -0,0 +1,4 @@
1
+ import type { Stats } from 'node:fs';
2
+ import type { SnapshotChecksum } from './snapshot-types.mts';
3
+ export declare function assertPartitionFile(info: Stats): void;
4
+ export declare function validatePartition(path: string, expected: Stats, checksum?: SnapshotChecksum): Promise<void>;
@@ -0,0 +1,40 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { constants } from 'node:fs';
3
+ import { open } from 'node:fs/promises';
4
+ import { consumeSnapshotRecord, assertManifest, parseSnapshotRecord, } from './snapshot-partition-format.mjs';
5
+ import { readLines } from './snapshot-partition-io.mjs';
6
+ export function assertPartitionFile(info) {
7
+ if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || (info.mode & 0o222) !== 0)
8
+ throw new Error('partition directory contains an unsafe partition');
9
+ }
10
+ export async function validatePartition(path, expected, checksum) {
11
+ const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
12
+ try {
13
+ const actual = await file.stat();
14
+ assertPartitionFile(actual);
15
+ if (actual.dev !== expected.dev || actual.ino !== expected.ino)
16
+ throw new Error('partition directory changed while it was being removed');
17
+ const state = { sessions: 0, entries: 0, records: 0 };
18
+ const hash = createHash('sha256');
19
+ let manifest = false;
20
+ for await (const line of readLines(file, hash)) {
21
+ if (!line || manifest)
22
+ throw new Error('partition contains invalid JSONL');
23
+ const record = parseSnapshotRecord(line);
24
+ if (record.type === 'manifest') {
25
+ assertManifest(record.manifest, state);
26
+ manifest = true;
27
+ }
28
+ else {
29
+ consumeSnapshotRecord(record, state);
30
+ }
31
+ }
32
+ if (!manifest || state.sessions === 0)
33
+ throw new Error('partition is missing a complete terminal manifest');
34
+ if (checksum && hash.digest('hex') !== checksum.value)
35
+ throw new Error('partition checksum does not match cleanup receipt');
36
+ }
37
+ finally {
38
+ await file.close();
39
+ }
40
+ }
@@ -0,0 +1,2 @@
1
+ import type { SnapshotManifest, SnapshotPartition } from './snapshot-types.mts';
2
+ export declare function writePartitions(index: string, manifest: SnapshotManifest, directory: string, maxSessions: number, maxBytes: number): Promise<SnapshotPartition[]>;
@@ -0,0 +1,91 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { constants } from 'node:fs';
3
+ import { chmod, open, rename } from 'node:fs/promises';
4
+ import { join } from 'node:path';
5
+ import { countsFor, manifestFor, snapshotLine, } from './snapshot-partition-format.mjs';
6
+ import { readLines, writeAll } from './snapshot-partition-io.mjs';
7
+ async function copyBlock(block, output, hash) {
8
+ const input = await open(block.path, constants.O_RDONLY | constants.O_NOFOLLOW);
9
+ const buffer = Buffer.allocUnsafe(64 * 1024);
10
+ try {
11
+ for (;;) {
12
+ const { bytesRead } = await input.read(buffer);
13
+ if (!bytesRead)
14
+ return;
15
+ const bytes = buffer.subarray(0, bytesRead);
16
+ hash.update(bytes);
17
+ await writeAll(output, bytes);
18
+ }
19
+ }
20
+ finally {
21
+ await input.close();
22
+ }
23
+ }
24
+ export async function writePartitions(index, manifest, directory, maxSessions, maxBytes) {
25
+ const partitions = [];
26
+ const indexFile = await open(index, constants.O_RDONLY | constants.O_NOFOLLOW);
27
+ let active;
28
+ const start = async () => {
29
+ const number = partitions.length + 1;
30
+ const temporary = join(directory, `.partition-${number}.tmp`);
31
+ active = {
32
+ block: { sessionId: '', path: '', bytes: 0, sessions: 0, entries: 0 },
33
+ file: await open(temporary, 'wx', 0o600),
34
+ temporary,
35
+ hash: createHash('sha256'),
36
+ };
37
+ return active;
38
+ };
39
+ const finish = async () => {
40
+ if (!active)
41
+ return;
42
+ const partitionManifest = manifestFor(manifest, active.block);
43
+ const terminal = Buffer.from(snapshotLine({ type: 'manifest', manifest: partitionManifest }));
44
+ await writeAll(active.file, terminal);
45
+ active.hash.update(terminal);
46
+ await active.file.sync();
47
+ await active.file.close();
48
+ await chmod(active.temporary, 0o400);
49
+ const path = join(directory, `partition-${partitions.length + 1}.jsonl`);
50
+ await rename(active.temporary, path);
51
+ const bytes = active.block.bytes + terminal.byteLength;
52
+ partitions.push({
53
+ path,
54
+ counts: countsFor(active.block, bytes),
55
+ checksum: { algorithm: 'sha256', value: active.hash.digest('hex') },
56
+ manifest: partitionManifest,
57
+ });
58
+ active = undefined;
59
+ };
60
+ try {
61
+ for await (const sourceLine of readLines(indexFile)) {
62
+ const block = JSON.parse(sourceLine);
63
+ if (active && active.block.sessions + block.sessions > maxSessions)
64
+ await finish();
65
+ let candidate = {
66
+ ...block,
67
+ sessions: (active?.block.sessions ?? 0) + block.sessions,
68
+ entries: (active?.block.entries ?? 0) + block.entries,
69
+ };
70
+ let terminalBytes = Buffer.byteLength(snapshotLine({ type: 'manifest', manifest: manifestFor(manifest, candidate) }));
71
+ if (active && active.block.bytes + block.bytes + terminalBytes > maxBytes) {
72
+ await finish();
73
+ candidate = { ...block };
74
+ terminalBytes = Buffer.byteLength(snapshotLine({ type: 'manifest', manifest: manifestFor(manifest, candidate) }));
75
+ }
76
+ if (block.bytes + terminalBytes > maxBytes)
77
+ throw new Error(`snapshot session ${block.sessionId} is too large for one partition`);
78
+ const target = active ?? (await start());
79
+ target.block.sessions += block.sessions;
80
+ target.block.entries += block.entries;
81
+ target.block.bytes += block.bytes;
82
+ await copyBlock(block, target.file, target.hash);
83
+ }
84
+ await finish();
85
+ return partitions;
86
+ }
87
+ finally {
88
+ await indexFile.close();
89
+ await active?.file.close().catch(() => undefined);
90
+ }
91
+ }
@@ -0,0 +1,10 @@
1
+ import { chmod, mkdtemp, open } from 'node:fs/promises';
2
+ import type { SnapshotPartitionOptions, SnapshotPartitionResult } from './snapshot-types.mts';
3
+ declare const defaults: {
4
+ open: typeof open;
5
+ mkdtemp: typeof mkdtemp;
6
+ chmod: typeof chmod;
7
+ };
8
+ export declare function setSnapshotFilesystemForTest(overrides?: Partial<typeof defaults>): void;
9
+ export declare function partitionSnapshot(options: SnapshotPartitionOptions): Promise<SnapshotPartitionResult>;
10
+ export {};