sealkeep 0.11.5 → 0.11.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/CONTROL_PLANE.md +2 -0
- package/dist/src/audit.d.ts +1 -1
- package/dist/src/chunk-store.js +104 -82
- package/dist/src/cli.js +150 -5
- package/dist/src/control-plane/auth.d.ts +9 -0
- package/dist/src/control-plane/auth.js +19 -2
- package/dist/src/control-plane/server.js +47 -0
- package/dist/src/control-plane/store.d.ts +32 -0
- package/dist/src/control-plane/store.js +33 -0
- package/dist/src/daemon.js +2 -0
- package/dist/src/errors.d.ts +1 -1
- package/dist/src/leakscan.d.ts +101 -9
- package/dist/src/leakscan.js +481 -23
- package/dist/src/local-api.js +16 -33
- package/dist/src/machine-settings.d.ts +26 -0
- package/dist/src/machine-settings.js +23 -0
- package/dist/src/queue.d.ts +11 -1
- package/dist/src/queue.js +2 -2
- package/dist/src/stream-to-cloud.d.ts +2 -0
- package/dist/src/stream-to-cloud.js +80 -59
- package/dist/src/types.d.ts +18 -0
- package/dist/src/vault.js +38 -8
- package/docs/README.md +21 -0
- package/docs/secret-scanning.md +81 -0
- package/package.json +3 -2
- package/web/app.js +24 -6
|
@@ -103,6 +103,11 @@ export function defaultLocalSettings() {
|
|
|
103
103
|
watchedPaths: [],
|
|
104
104
|
exclusions: [],
|
|
105
105
|
reclaimEnabled: false,
|
|
106
|
+
secretScanPolicy: "off",
|
|
107
|
+
secretScanFailClosed: false,
|
|
108
|
+
secretScanOrgRollup: false,
|
|
109
|
+
secretScanControlPlaneUrl: null,
|
|
110
|
+
secretScanPatterns: [],
|
|
106
111
|
trashStrategy: "auto",
|
|
107
112
|
schedule: { intervalMinutes: 5 },
|
|
108
113
|
bandwidth: { uploadLimitMbps: null },
|
|
@@ -144,6 +149,24 @@ export async function readLocalSettings(dataDir) {
|
|
|
144
149
|
return {
|
|
145
150
|
...fallback,
|
|
146
151
|
...saved,
|
|
152
|
+
secretScanPolicy: saved?.secretScanPolicy === "block-high" || saved?.secretScanPolicy === "record" ? saved.secretScanPolicy : "off",
|
|
153
|
+
secretScanFailClosed: saved?.secretScanFailClosed === true,
|
|
154
|
+
secretScanOrgRollup: saved?.secretScanOrgRollup === true,
|
|
155
|
+
secretScanControlPlaneUrl: typeof saved?.secretScanControlPlaneUrl === "string" && /^https?:\/\//.test(saved.secretScanControlPlaneUrl)
|
|
156
|
+
? saved.secretScanControlPlaneUrl.replace(/\/+$/, "")
|
|
157
|
+
: null,
|
|
158
|
+
secretScanPatterns: Array.isArray(saved?.secretScanPatterns)
|
|
159
|
+
? saved.secretScanPatterns.flatMap((pattern) => {
|
|
160
|
+
if (!pattern || typeof pattern !== "object")
|
|
161
|
+
return [];
|
|
162
|
+
const entry = pattern;
|
|
163
|
+
if (typeof entry.kind !== "string" || typeof entry.source !== "string")
|
|
164
|
+
return [];
|
|
165
|
+
if (entry.severity !== "high" && entry.severity !== "medium")
|
|
166
|
+
return [];
|
|
167
|
+
return [{ kind: entry.kind, severity: entry.severity, source: entry.source }];
|
|
168
|
+
}).slice(0, 32)
|
|
169
|
+
: [],
|
|
147
170
|
watchedAgents: Array.isArray(saved?.watchedAgents)
|
|
148
171
|
? saved.watchedAgents.filter((agent) => agent === "codex" || agent === "claude")
|
|
149
172
|
: fallback.watchedAgents,
|
package/dist/src/queue.d.ts
CHANGED
|
@@ -46,6 +46,16 @@ export type EnqueueRequest = {
|
|
|
46
46
|
event: string;
|
|
47
47
|
sessionId?: string;
|
|
48
48
|
};
|
|
49
|
+
export type EnqueueOptions = {
|
|
50
|
+
/**
|
|
51
|
+
* Skip the queue-wide stale-snapshot reconciliation. Lifecycle hooks use
|
|
52
|
+
* this after durably recording one row so a mature queue cannot hold the
|
|
53
|
+
* agent open while thousands of unrelated rows are scanned. The daemon
|
|
54
|
+
* performs that reconciliation asynchronously as it discovers claimable
|
|
55
|
+
* work.
|
|
56
|
+
*/
|
|
57
|
+
deferSupersede?: boolean;
|
|
58
|
+
};
|
|
49
59
|
export type EnqueueResult = {
|
|
50
60
|
job: ArchiveJob;
|
|
51
61
|
deduped: boolean;
|
|
@@ -218,7 +228,7 @@ export declare class ArchiveQueue {
|
|
|
218
228
|
private describe;
|
|
219
229
|
private persist;
|
|
220
230
|
/** Records intent to archive a transcript. It never reads transcript contents. */
|
|
221
|
-
enqueue(request: EnqueueRequest): Promise<EnqueueResult>;
|
|
231
|
+
enqueue(request: EnqueueRequest, options?: EnqueueOptions): Promise<EnqueueResult>;
|
|
222
232
|
/**
|
|
223
233
|
* Batch form used by startup/rescan discovery. Individual unreadable sources
|
|
224
234
|
* are reported without aborting the rest of the pass, matching the watcher's
|
package/dist/src/queue.js
CHANGED
|
@@ -740,7 +740,7 @@ export class ArchiveQueue {
|
|
|
740
740
|
return { job: await this.require(job.id), deduped: true };
|
|
741
741
|
}
|
|
742
742
|
/** Records intent to archive a transcript. It never reads transcript contents. */
|
|
743
|
-
async enqueue(request) {
|
|
743
|
+
async enqueue(request, options = {}) {
|
|
744
744
|
const described = await this.describe(request);
|
|
745
745
|
await mkdir(this.root, { recursive: true, mode: 0o700 });
|
|
746
746
|
let persisted = await this.persist(described);
|
|
@@ -762,7 +762,7 @@ export class ArchiveQueue {
|
|
|
762
762
|
});
|
|
763
763
|
persisted = { ...persisted, job: promoted };
|
|
764
764
|
}
|
|
765
|
-
const retired = await this.supersedeMany([persisted.job]);
|
|
765
|
+
const retired = options.deferSupersede ? new Map() : await this.supersedeMany([persisted.job]);
|
|
766
766
|
return { ...persisted, superseded: retired.get(persisted.job.id) ?? [] };
|
|
767
767
|
}
|
|
768
768
|
/**
|
|
@@ -55,6 +55,8 @@ type StreamClient = {
|
|
|
55
55
|
checksum?: string;
|
|
56
56
|
}>;
|
|
57
57
|
abortMultipart?(objectKey: string, uploadId: string): Promise<void>;
|
|
58
|
+
/** Removes a completed object when a seal is refused after the bytes were uploaded. */
|
|
59
|
+
deleteObject?(objectKey: string): Promise<void>;
|
|
58
60
|
/** The provider's own account of an in-flight multipart — resume's source of truth. */
|
|
59
61
|
listParts?(objectKey: string, uploadId: string): Promise<{
|
|
60
62
|
partNumber: number;
|
|
@@ -18,6 +18,7 @@ import { DEFAULT_CHUNK_BYTES, ENVELOPE_VERSION, KEY_BYTES, StreamingSha256, TAG_
|
|
|
18
18
|
// seals to the bytes the provider already holds. Reaching into the package for
|
|
19
19
|
// that beats re-implementing the cipher setup here and drifting from it.
|
|
20
20
|
import { aeadCipher, aeadDecipher } from "../packages/sealkeep-crypto/src/aead.js";
|
|
21
|
+
import { assertScanCoversSeal, deliverOrgRollup, discardScanSnapshot, scanBeforeSeal } from "./leakscan.js";
|
|
21
22
|
const STREAMABLE = new Set(["s3", "r2", "b2", "gcs"]);
|
|
22
23
|
const SEAL_SUITE = "chacha20-poly1305";
|
|
23
24
|
const sha256hex = (input) => createHash("sha256").update(input).digest("hex");
|
|
@@ -178,63 +179,71 @@ export async function sealArchiveToCloud(dataDir, sourcePath, rawPhrase, agent,
|
|
|
178
179
|
const source = await stat(absolute).catch(() => null);
|
|
179
180
|
if (!source || !source.isFile())
|
|
180
181
|
fail("source_unreadable", `Transcript is not readable: ${absolute}`, { sourcePath: absolute });
|
|
181
|
-
const
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
// source or abandons it with the reason and looks again, so a stack of
|
|
190
|
-
// crashed attempts drains to the one worth continuing — or to none.
|
|
191
|
-
for (;;) {
|
|
192
|
-
const journal = await findSpoolForSource(dataDir, absolute);
|
|
193
|
-
if (!journal)
|
|
194
|
-
break;
|
|
195
|
-
if (options.resume === false) {
|
|
196
|
-
await abandonJournal(dataDir, client, journal, "a fresh start was requested (--fresh)");
|
|
197
|
-
continue;
|
|
198
|
-
}
|
|
199
|
-
if (journal.provider !== remoteStorage.provider || journal.bucket !== remoteStorage.bucket
|
|
200
|
-
|| journal.source.totalBytes !== source.size
|
|
201
|
-
|| !Number.isInteger(journal.source.chunkBytes) || journal.source.chunkBytes <= 0
|
|
202
|
-
|| !Number.isInteger(journal.nextChunkIndex) || journal.nextChunkIndex < 0) {
|
|
203
|
-
await abandonJournal(dataDir, client, journal, "the source file or the storage target is no longer the one the journal describes");
|
|
204
|
-
continue;
|
|
205
|
-
}
|
|
206
|
-
// THE SAFETY GATE, before anything else: the same (archiveKey,
|
|
207
|
-
// noncePrefix, chunkIndex) triple must never encrypt two different
|
|
208
|
-
// plaintexts, and every resume decision must be provably on the safe side
|
|
209
|
-
// of that line. Chunks below `nextChunkIndex` were already encrypted and
|
|
210
|
-
// handed over under this journal's key, so this seal may only continue if
|
|
211
|
-
// the bytes they covered are still byte-for-byte the bytes on disk. Hash
|
|
212
|
-
// the consumed prefix and compare; on any mismatch the journal — key,
|
|
213
|
-
// nonce prefix, provider session, all of it — is dead, and the archive
|
|
214
|
-
// starts over with a fresh identity.
|
|
215
|
-
const consumed = Math.min(journal.nextChunkIndex * journal.source.chunkBytes, source.size);
|
|
216
|
-
const gate = await hashFileRange(absolute, 0, consumed);
|
|
217
|
-
if (gate.sha256 !== journal.consumedPrefixSha256) {
|
|
218
|
-
await abandonJournal(dataDir, client, journal, "the source bytes under the already-uploaded chunks changed, so the sealed prefix no longer describes this file");
|
|
219
|
-
continue;
|
|
220
|
-
}
|
|
221
|
-
// One process per journal, held across the whole resume including its
|
|
222
|
-
// abandon path — a second process asking meanwhile is told to wait.
|
|
223
|
-
const lock = await acquireSpoolLock(dataDir, journal.archiveId, { now: options.now });
|
|
224
|
-
try {
|
|
225
|
-
return await resumeSeal(shared, journal);
|
|
226
|
-
}
|
|
227
|
-
catch (error) {
|
|
228
|
-
if (!resumeRefused(error))
|
|
229
|
-
throw error;
|
|
230
|
-
await abandonJournal(dataDir, client, journal, error instanceof Error ? error.message : "the journaled provider state was not resumable");
|
|
231
|
-
continue;
|
|
182
|
+
const sealScan = await scanBeforeSeal(dataDir, absolute, undefined, undefined, source.size, { snapshot: true });
|
|
183
|
+
try {
|
|
184
|
+
const secretScan = sealScan?.metadata;
|
|
185
|
+
const plaintextPath = sealScan?.snapshotPath ?? absolute;
|
|
186
|
+
const phrase = canonicalPhrase(rawPhrase);
|
|
187
|
+
const rawClient = options.client ?? await uploadClientFromStore(dataDir, config.vaultId, remoteStorage);
|
|
188
|
+
if (!("uploadStream" in rawClient) || typeof rawClient.uploadStream !== "function") {
|
|
189
|
+
fail("invalid_argument", `The ${remoteStorage.provider} client cannot stream uploads`);
|
|
232
190
|
}
|
|
233
|
-
|
|
234
|
-
|
|
191
|
+
const client = rawClient;
|
|
192
|
+
const shared = { dataDir, client, config, remoteStorage, absolute, plaintextPath, totalBytes: source.size, phrase, agent, project: options.project ?? null, projectKey: options.projectKey ?? null, targetId: options.targetId, now: options.now, secretScan, sealScan };
|
|
193
|
+
// Journal triage. Each pass either resumes the newest journal for this
|
|
194
|
+
// source or abandons it with the reason and looks again, so a stack of
|
|
195
|
+
// crashed attempts drains to the one worth continuing — or to none.
|
|
196
|
+
for (;;) {
|
|
197
|
+
const journal = await findSpoolForSource(dataDir, absolute);
|
|
198
|
+
if (!journal)
|
|
199
|
+
break;
|
|
200
|
+
if (options.resume === false) {
|
|
201
|
+
await abandonJournal(dataDir, client, journal, "a fresh start was requested (--fresh)");
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
if (journal.provider !== remoteStorage.provider || journal.bucket !== remoteStorage.bucket
|
|
205
|
+
|| journal.source.totalBytes !== source.size
|
|
206
|
+
|| !Number.isInteger(journal.source.chunkBytes) || journal.source.chunkBytes <= 0
|
|
207
|
+
|| !Number.isInteger(journal.nextChunkIndex) || journal.nextChunkIndex < 0) {
|
|
208
|
+
await abandonJournal(dataDir, client, journal, "the source file or the storage target is no longer the one the journal describes");
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
// THE SAFETY GATE, before anything else: the same (archiveKey,
|
|
212
|
+
// noncePrefix, chunkIndex) triple must never encrypt two different
|
|
213
|
+
// plaintexts, and every resume decision must be provably on the safe side
|
|
214
|
+
// of that line. Chunks below `nextChunkIndex` were already encrypted and
|
|
215
|
+
// handed over under this journal's key, so this seal may only continue if
|
|
216
|
+
// the bytes they covered are still byte-for-byte the bytes on disk. Hash
|
|
217
|
+
// the consumed prefix and compare; on any mismatch the journal — key,
|
|
218
|
+
// nonce prefix, provider session, all of it — is dead, and the archive
|
|
219
|
+
// starts over with a fresh identity.
|
|
220
|
+
const consumed = Math.min(journal.nextChunkIndex * journal.source.chunkBytes, source.size);
|
|
221
|
+
const gate = await hashFileRange(plaintextPath, 0, consumed);
|
|
222
|
+
if (gate.sha256 !== journal.consumedPrefixSha256) {
|
|
223
|
+
await abandonJournal(dataDir, client, journal, "the source bytes under the already-uploaded chunks changed, so the sealed prefix no longer describes this file");
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
// One process per journal, held across the whole resume including its
|
|
227
|
+
// abandon path — a second process asking meanwhile is told to wait.
|
|
228
|
+
const lock = await acquireSpoolLock(dataDir, journal.archiveId, { now: options.now });
|
|
229
|
+
try {
|
|
230
|
+
return await resumeSeal(shared, journal);
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
if (!resumeRefused(error))
|
|
234
|
+
throw error;
|
|
235
|
+
await abandonJournal(dataDir, client, journal, error instanceof Error ? error.message : "the journaled provider state was not resumable");
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
finally {
|
|
239
|
+
await lock.release();
|
|
240
|
+
}
|
|
235
241
|
}
|
|
242
|
+
return await freshSeal(shared);
|
|
243
|
+
}
|
|
244
|
+
finally {
|
|
245
|
+
await discardScanSnapshot(sealScan);
|
|
236
246
|
}
|
|
237
|
-
return freshSeal(shared);
|
|
238
247
|
}
|
|
239
248
|
/**
|
|
240
249
|
* The seal↔upload bridge both the fresh and the resumed path run: seal chunks
|
|
@@ -328,7 +337,7 @@ async function streamSealCore(ctx, args) {
|
|
|
328
337
|
providerState: progress.journalable
|
|
329
338
|
});
|
|
330
339
|
};
|
|
331
|
-
const sealPromise = sealChunksToSink({ path: ctx.
|
|
340
|
+
const sealPromise = sealChunksToSink({ path: ctx.plaintextPath, start: args.firstChunkIndex * args.chunkBytes, end: ctx.totalBytes }, sink, {
|
|
332
341
|
recipients: args.recipients, archiveId: args.archiveId, adapter: { agent: ctx.agent, version: ADAPTER_VERSION },
|
|
333
342
|
suite: args.suite, chunkBytes: args.chunkBytes, archiveKey: args.archiveKey, noncePrefix: args.noncePrefix,
|
|
334
343
|
firstChunkIndex: args.firstChunkIndex, hashers: args.hashers,
|
|
@@ -394,6 +403,16 @@ async function verifyAndRecord(ctx, flight) {
|
|
|
394
403
|
await shredSpool(ctx.dataDir, archiveId);
|
|
395
404
|
fail("ciphertext_integrity_failed", `Refusing to record the streamed archive as durable: ${problems.join("; ")}. Nothing was written locally; re-run the archive.`, { archiveId, objectKey });
|
|
396
405
|
}
|
|
406
|
+
const plaintextSha256 = flight.hashers.plaintext.digestHex();
|
|
407
|
+
try {
|
|
408
|
+
assertScanCoversSeal(ctx.sealScan, plaintextSha256, ctx.absolute);
|
|
409
|
+
}
|
|
410
|
+
catch (error) {
|
|
411
|
+
await abortProviderState(ctx.client, objectKey, flight.progress);
|
|
412
|
+
await ctx.client.deleteObject?.(objectKey).catch(() => undefined);
|
|
413
|
+
await shredSpool(ctx.dataDir, archiveId);
|
|
414
|
+
throw error;
|
|
415
|
+
}
|
|
397
416
|
const now = new Date(ctx.now ?? Date.now()).toISOString();
|
|
398
417
|
// `remote.checksum` stays OUR whole-object stored SHA-256 (base64), resumed
|
|
399
418
|
// or not — the resumable hasher covered every stored byte across processes.
|
|
@@ -409,7 +428,7 @@ async function verifyAndRecord(ctx, flight) {
|
|
|
409
428
|
const completed = {
|
|
410
429
|
version: 2, id: archiveId, createdAt: flight.envelope.manifest.createdAt,
|
|
411
430
|
source: {
|
|
412
|
-
path: ctx.absolute, agent: ctx.agent, bytes: ctx.totalBytes, sha256:
|
|
431
|
+
path: ctx.absolute, agent: ctx.agent, bytes: ctx.totalBytes, sha256: plaintextSha256,
|
|
413
432
|
...(ctx.project ? { project: ctx.project } : {}),
|
|
414
433
|
...(ctx.projectKey ? { projectKey: ctx.projectKey } : {}),
|
|
415
434
|
...(teamSpaceOf(ctx.config, ctx.project, ctx.projectKey)?.spaceKey ? { projectScope: teamSpaceOf(ctx.config, ctx.project, ctx.projectKey).spaceKey } : {}),
|
|
@@ -421,7 +440,8 @@ async function verifyAndRecord(ctx, flight) {
|
|
|
421
440
|
// an uploaded-then-offloaded archive reaches, arrived at without the detour.
|
|
422
441
|
objectPath: join(ctx.config.storage.root, `${archiveId}.skarchive`),
|
|
423
442
|
remote,
|
|
424
|
-
offloaded: { at: now, provider: ctx.remoteStorage.provider, bucket: ctx.remoteStorage.bucket, objectKey }
|
|
443
|
+
offloaded: { at: now, provider: ctx.remoteStorage.provider, bucket: ctx.remoteStorage.bucket, objectKey },
|
|
444
|
+
...(ctx.secretScan ? { secretScan: ctx.secretScan } : {})
|
|
425
445
|
};
|
|
426
446
|
const record = await mutateArchiveRecord(ctx.config.storage.root, archiveId, (current) => {
|
|
427
447
|
if (!isV2(current))
|
|
@@ -429,7 +449,8 @@ async function verifyAndRecord(ctx, flight) {
|
|
|
429
449
|
const { remote: _remote, copies: _copies, ...completedFacts } = completed;
|
|
430
450
|
return appendArchiveCopy({ ...current, ...completedFacts }, remote, { makePrimary: true });
|
|
431
451
|
}, { initial: completed });
|
|
432
|
-
await recordAudit(ctx.dataDir, "archive.create", "allowed", { archiveId, agent: ctx.agent, bytes: ctx.totalBytes, recipients: flight.envelope.wrappedKeys.length, streamed: true, ...(flight.resumed ? { resumed: true } : {}) });
|
|
452
|
+
await recordAudit(ctx.dataDir, "archive.create", "allowed", { archiveId, agent: ctx.agent, bytes: ctx.totalBytes, recipients: flight.envelope.wrappedKeys.length, streamed: true, ...(flight.resumed ? { resumed: true } : {}), secretFindings: ctx.secretScan?.findings.reduce((total, item) => total + item.count, 0) ?? null });
|
|
453
|
+
await deliverOrgRollup(ctx.dataDir, archiveId, ctx.secretScan);
|
|
433
454
|
await recordAudit(ctx.dataDir, "upload.verify", "allowed", { archiveId, objectKey, bytes: flight.wholeStoredBytes, provider: ctx.remoteStorage.provider, streamed: true, ...(flight.resumed ? { resumed: true } : {}) });
|
|
434
455
|
await shredSpool(ctx.dataDir, archiveId);
|
|
435
456
|
await sweepStaleJournals(ctx.dataDir, ctx.client, ctx.absolute);
|
|
@@ -587,7 +608,7 @@ async function resumeSeal(ctx, journal) {
|
|
|
587
608
|
}
|
|
588
609
|
};
|
|
589
610
|
if (count > 0 || (feedPartialTo > 0 && trueParts)) {
|
|
590
|
-
const handle = await open(ctx.
|
|
611
|
+
const handle = await open(ctx.plaintextPath, "r");
|
|
591
612
|
try {
|
|
592
613
|
const buffer = Buffer.allocUnsafe(Math.min(chunkBytes, Math.max(ctx.totalBytes, 1)));
|
|
593
614
|
for (let index = 0; index < count; index += 1) {
|
package/dist/src/types.d.ts
CHANGED
|
@@ -203,6 +203,24 @@ export type ArchiveRecordV2 = {
|
|
|
203
203
|
baseArchiveId: string;
|
|
204
204
|
baseBytes: number;
|
|
205
205
|
};
|
|
206
|
+
/**
|
|
207
|
+
* Seal-time secret scan, stored beside the archive so "which archives hold
|
|
208
|
+
* credentials?" does not require opening them. Kind, severity, count, and
|
|
209
|
+
* line numbers only — never matched text, previews, or paths.
|
|
210
|
+
*/
|
|
211
|
+
secretScan?: ArchiveSecretScan;
|
|
212
|
+
};
|
|
213
|
+
/** Local-only scan summary. Line numbers stay on this machine. */
|
|
214
|
+
export type ArchiveSecretScan = {
|
|
215
|
+
version: 1;
|
|
216
|
+
scannedAt: string;
|
|
217
|
+
policy: "record" | "block-high";
|
|
218
|
+
findings: Array<{
|
|
219
|
+
kind: string;
|
|
220
|
+
severity: "high" | "medium";
|
|
221
|
+
count: number;
|
|
222
|
+
lines: number[];
|
|
223
|
+
}>;
|
|
206
224
|
};
|
|
207
225
|
export type ArchiveRecord = ArchiveRecordV1 | ArchiveRecordV2;
|
|
208
226
|
export type StorageTargetConfig = {
|
package/dist/src/vault.js
CHANGED
|
@@ -15,6 +15,7 @@ import { isV2, teamCustodyRecordIsValid } from "./types.js";
|
|
|
15
15
|
import { envVar } from "./env.js";
|
|
16
16
|
import { hasCompleteVerifiedCopyChain, hasVerifiedCopy } from "./archive-copies.js";
|
|
17
17
|
import { acquireDurableTicketLock } from "./durable-ticket-lock.js";
|
|
18
|
+
import { assertScanCoversSeal, deliverOrgRollup, scanBeforeSeal } from "./leakscan.js";
|
|
18
19
|
// Agent hook stdout/stderr is a protocol boundary. A damaged historical
|
|
19
20
|
// sidecar must remain visible in Doctor/Health, but the low-level warning that
|
|
20
21
|
// `listArchives` emits for an interactive process must not make an otherwise
|
|
@@ -986,6 +987,9 @@ export async function archiveFile(dataDir, sourcePath, rawPhrase, agent = "custo
|
|
|
986
987
|
// both decide the answer is no.
|
|
987
988
|
return withSourceLock(dataDir, absolute, async () => {
|
|
988
989
|
hooks.signal?.throwIfAborted();
|
|
990
|
+
// Shared archive boundary: daemon, CLI, and MCP seals use this path.
|
|
991
|
+
const sealScan = await scanBeforeSeal(dataDir, absolute, hooks.signal, undefined, source.size);
|
|
992
|
+
const secretScan = sealScan?.metadata;
|
|
989
993
|
// Hashing the source to answer "have I already archived exactly this?" is a
|
|
990
994
|
// whole extra read of a file that may be gigabytes, so it is only worth doing
|
|
991
995
|
// when some archive of this path already claims the same length. Anything of
|
|
@@ -1023,7 +1027,7 @@ export async function archiveFile(dataDir, sourcePath, rawPhrase, agent = "custo
|
|
|
1023
1027
|
// limit, fall through to the ordinary full-snapshot writer. That snapshot
|
|
1024
1028
|
// is independently restorable, and later appends start a short new chain.
|
|
1025
1029
|
if (resolveDeltaChain(archives, baseRecord).length < MAX_DELTA_CHAIN_LINKS) {
|
|
1026
|
-
return sealDeltaArchive(dataDir, config, absolute, agent, phrase, source.size, { ...base, sha256: baseRecord.source.sha256 }, hooks);
|
|
1030
|
+
return sealDeltaArchive(dataDir, config, absolute, agent, phrase, source.size, { ...base, sha256: baseRecord.source.sha256 }, hooks, sealScan);
|
|
1027
1031
|
}
|
|
1028
1032
|
}
|
|
1029
1033
|
const id = randomUUID();
|
|
@@ -1050,8 +1054,15 @@ export async function archiveFile(dataDir, sourcePath, rawPhrase, agent = "custo
|
|
|
1050
1054
|
// a project's membership needs the answer the seal already had.
|
|
1051
1055
|
source: { path: absolute, agent, bytes: plain.originalBytes, sha256: plain.originalSha256, ...(hooks.project ? { project: hooks.project } : {}), ...(hooks.projectKey ? { projectKey: hooks.projectKey } : {}), ...(projectScope ? { projectScope } : {}) },
|
|
1052
1056
|
cipher: { algorithm: "none", ciphertextSha256: plain.storedSha256, storedBytes: plain.storedBytes, chunks: 0 },
|
|
1053
|
-
objectPath, ...(base ? { supersedes: base } : {})
|
|
1057
|
+
objectPath, ...(base ? { supersedes: base } : {}), ...(secretScan ? { secretScan } : {})
|
|
1054
1058
|
};
|
|
1059
|
+
try {
|
|
1060
|
+
assertScanCoversSeal(sealScan, plain.originalSha256, absolute);
|
|
1061
|
+
}
|
|
1062
|
+
catch (error) {
|
|
1063
|
+
await rm(staged, { force: true });
|
|
1064
|
+
throw error;
|
|
1065
|
+
}
|
|
1055
1066
|
try {
|
|
1056
1067
|
await rename(staged, objectPath);
|
|
1057
1068
|
}
|
|
@@ -1060,7 +1071,8 @@ export async function archiveFile(dataDir, sourcePath, rawPhrase, agent = "custo
|
|
|
1060
1071
|
throw error;
|
|
1061
1072
|
}
|
|
1062
1073
|
await writeRecord(config.storage.root, record);
|
|
1063
|
-
await recordAudit(dataDir, "archive.create", "allowed", { archiveId: id, agent, bytes: plain.originalBytes, recipients: 0, supersedes: base?.archiveId ?? null });
|
|
1074
|
+
await recordAudit(dataDir, "archive.create", "allowed", { archiveId: id, agent, bytes: plain.originalBytes, recipients: 0, supersedes: base?.archiveId ?? null, secretFindings: secretScan?.findings.reduce((total, item) => total + item.count, 0) ?? null });
|
|
1075
|
+
await deliverOrgRollup(dataDir, id, secretScan);
|
|
1064
1076
|
return { ...record, deduplicated: false };
|
|
1065
1077
|
}
|
|
1066
1078
|
// Compression happens before the seal because ciphertext is incompressible
|
|
@@ -1095,10 +1107,17 @@ export async function archiveFile(dataDir, sourcePath, rawPhrase, agent = "custo
|
|
|
1095
1107
|
version: 2, id, createdAt: sealed.envelope.manifest.createdAt,
|
|
1096
1108
|
source: { path: absolute, agent, bytes: sealed.originalBytes, sha256: sealed.originalSha256, ...(hooks.project ? { project: hooks.project } : {}), ...(hooks.projectKey ? { projectKey: hooks.projectKey } : {}), ...(projectScope ? { projectScope } : {}) },
|
|
1097
1109
|
cipher: { algorithm: sealed.envelope.suite, ciphertextSha256: sealed.ciphertextSha256, storedBytes: sealed.storedBytes, chunks: sealed.envelope.chunks.length },
|
|
1098
|
-
envelope: sealed.envelope, objectPath, ...(base ? { supersedes: base } : {})
|
|
1110
|
+
envelope: sealed.envelope, objectPath, ...(base ? { supersedes: base } : {}), ...(secretScan ? { secretScan } : {})
|
|
1099
1111
|
};
|
|
1100
1112
|
// The object only becomes an archive once it is whole: the manifest hashes are
|
|
1101
1113
|
// proved by the last chunk, so a crash mid-seal leaves a .partial nobody reads.
|
|
1114
|
+
try {
|
|
1115
|
+
assertScanCoversSeal(sealScan, sealed.originalSha256, absolute);
|
|
1116
|
+
}
|
|
1117
|
+
catch (error) {
|
|
1118
|
+
await rm(staged, { force: true });
|
|
1119
|
+
throw error;
|
|
1120
|
+
}
|
|
1102
1121
|
try {
|
|
1103
1122
|
await rename(staged, objectPath);
|
|
1104
1123
|
}
|
|
@@ -1107,7 +1126,8 @@ export async function archiveFile(dataDir, sourcePath, rawPhrase, agent = "custo
|
|
|
1107
1126
|
throw error;
|
|
1108
1127
|
}
|
|
1109
1128
|
await writeRecord(config.storage.root, record);
|
|
1110
|
-
await recordAudit(dataDir, "archive.create", "allowed", { archiveId: id, agent, bytes: sealed.originalBytes, recipients: sealed.envelope.wrappedKeys.length, supersedes: base?.archiveId ?? null });
|
|
1129
|
+
await recordAudit(dataDir, "archive.create", "allowed", { archiveId: id, agent, bytes: sealed.originalBytes, recipients: sealed.envelope.wrappedKeys.length, supersedes: base?.archiveId ?? null, secretFindings: secretScan?.findings.reduce((total, item) => total + item.count, 0) ?? null });
|
|
1130
|
+
await deliverOrgRollup(dataDir, id, secretScan);
|
|
1111
1131
|
if (tokenCollector)
|
|
1112
1132
|
await indexSealInline(dataDir, phrase, record, tokenCollector.finish(sealed.envelope.chunks.length), hooks.signal);
|
|
1113
1133
|
return { ...record, deduplicated: false };
|
|
@@ -1214,11 +1234,12 @@ async function hashDeltaSnapshot(sourcePath, size, prefixBytes, onActivity, sign
|
|
|
1214
1234
|
tailSha256: tail.digest("hex")
|
|
1215
1235
|
};
|
|
1216
1236
|
}
|
|
1217
|
-
async function sealDeltaArchive(dataDir, config, absolute, agent, phrase, size, base, hooks) {
|
|
1237
|
+
async function sealDeltaArchive(dataDir, config, absolute, agent, phrase, size, base, hooks, sealScan) {
|
|
1218
1238
|
const id = randomUUID();
|
|
1219
1239
|
const objectPath = join(config.storage.root, `${id}.skarchive`);
|
|
1220
1240
|
const staged = `${objectPath}.${randomUUID()}.partial`;
|
|
1221
1241
|
const projectScope = teamSpaceOf(config, hooks.project, hooks.projectKey)?.spaceKey;
|
|
1242
|
+
const secretScan = sealScan?.metadata;
|
|
1222
1243
|
// Same inline-indexing gate a full seal uses, sized to what THIS seal
|
|
1223
1244
|
// actually covers: the delta's own appended range, not the whole
|
|
1224
1245
|
// transcript. That range is exactly what the delta's own chunk numbers
|
|
@@ -1265,12 +1286,20 @@ async function sealDeltaArchive(dataDir, config, absolute, agent, phrase, size,
|
|
|
1265
1286
|
await rm(staged, { force: true });
|
|
1266
1287
|
fail("source_unreadable", `${absolute} changed while its append was being sealed; it will re-archive whole on the next pass`, { sourcePath: absolute, bytes: size });
|
|
1267
1288
|
}
|
|
1289
|
+
try {
|
|
1290
|
+
assertScanCoversSeal(sealScan, snapshot.wholeSha256, absolute);
|
|
1291
|
+
}
|
|
1292
|
+
catch (error) {
|
|
1293
|
+
await rm(staged, { force: true });
|
|
1294
|
+
throw error;
|
|
1295
|
+
}
|
|
1268
1296
|
const record = {
|
|
1269
1297
|
version: 2, id, createdAt: sealed.envelope.manifest.createdAt,
|
|
1270
1298
|
source: { path: absolute, agent, bytes: size, sha256: snapshot.wholeSha256, ...(hooks.project ? { project: hooks.project } : {}), ...(hooks.projectKey ? { projectKey: hooks.projectKey } : {}), ...(projectScope ? { projectScope } : {}) },
|
|
1271
1299
|
cipher: { algorithm: sealed.envelope.suite, ciphertextSha256: sealed.ciphertextSha256, storedBytes: sealed.storedBytes, chunks: sealed.envelope.chunks.length },
|
|
1272
1300
|
envelope: sealed.envelope, objectPath,
|
|
1273
|
-
delta: { baseArchiveId: base.archiveId, baseBytes: base.bytes }
|
|
1301
|
+
delta: { baseArchiveId: base.archiveId, baseBytes: base.bytes },
|
|
1302
|
+
...(secretScan ? { secretScan } : {})
|
|
1274
1303
|
};
|
|
1275
1304
|
try {
|
|
1276
1305
|
await rename(staged, objectPath);
|
|
@@ -1280,7 +1309,8 @@ async function sealDeltaArchive(dataDir, config, absolute, agent, phrase, size,
|
|
|
1280
1309
|
throw error;
|
|
1281
1310
|
}
|
|
1282
1311
|
await writeRecord(config.storage.root, record);
|
|
1283
|
-
await recordAudit(dataDir, "archive.create", "allowed", { archiveId: id, agent, bytes: size, appendedBytes: sealed.originalBytes, recipients: sealed.envelope.wrappedKeys.length, supersedes: null, deltaBase: base.archiveId });
|
|
1312
|
+
await recordAudit(dataDir, "archive.create", "allowed", { archiveId: id, agent, bytes: size, appendedBytes: sealed.originalBytes, recipients: sealed.envelope.wrappedKeys.length, supersedes: null, deltaBase: base.archiveId, secretFindings: secretScan?.findings.reduce((total, item) => total + item.count, 0) ?? null });
|
|
1313
|
+
await deliverOrgRollup(dataDir, id, secretScan);
|
|
1284
1314
|
// The seal is durable either way; indexing it is advisory (see
|
|
1285
1315
|
// indexSealInline) and, unlike a full seal, covers only the appended range.
|
|
1286
1316
|
if (tokenCollector)
|
package/docs/README.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# The documentation map
|
|
2
|
+
|
|
3
|
+
**Published with the package** (repo root): [README](../README.md) —
|
|
4
|
+
[ARCHITECTURE](../ARCHITECTURE.md) — [CONTROL_PLANE](../CONTROL_PLANE.md) —
|
|
5
|
+
[THREAT_MODEL](../THREAT_MODEL.md) — [THIRD_PARTY](../THIRD_PARTY.md).
|
|
6
|
+
|
|
7
|
+
**Living internal docs** (repo root, referenced from code): PRODUCT,
|
|
8
|
+
PRODUCT_ARCHITECTURE, CONTROL_PANEL, SPALA_BACKEND, STORAGE, BORROWED.
|
|
9
|
+
|
|
10
|
+
**This directory — designs and operations:**
|
|
11
|
+
|
|
12
|
+
| Page | What it holds |
|
|
13
|
+
| --- | --- |
|
|
14
|
+
| [RUNBOOK.md](./RUNBOOK.md) | Operating the control plane |
|
|
15
|
+
| [secret-scanning.md](./secret-scanning.md) | Seal-time secret scan: defaults, what stays local, what a rollup sends |
|
|
16
|
+
| [legal/](./legal/) | Draft DPA, privacy, security, retention, rights, breach, self-hosting, and DPIA procurement pack |
|
|
17
|
+
| [managed-streaming-contract.md](./managed-streaming-contract.md) | The CREDENTIALS lease contract, written before B2 exists |
|
|
18
|
+
| [sharing-and-spaces.md](./sharing-and-spaces.md) | Sharing today, and the full spaces/org design — **designed, not built** |
|
|
19
|
+
| [history/](./history/) | Dated snapshots, kept but not maintained |
|
|
20
|
+
- [Browser QA checklist](qa-browser-checklist.md) — the clickable scenarios, what has been run against the live service, and what needs a human.
|
|
21
|
+
- [Sharing: one person, and one company](sharing-b2c-b2b.md) — the B2C hand-off that ships, the B2B model that does not, and what each costs.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Secret scanning
|
|
2
|
+
|
|
3
|
+
Seal-time scanning is off by default. Turn it on per machine:
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
sealkeep scan policy record # scan, store findings, do not block
|
|
7
|
+
sealkeep scan policy block-high # also refuse high-confidence findings
|
|
8
|
+
sealkeep scan policy off
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
`record` and `block-high` run before encryption or upload on every new seal:
|
|
12
|
+
the background worker, `sealkeep archive`, the MCP archive tool, delta seals,
|
|
13
|
+
and both streamed upload paths. A transcript that changes during the scan is
|
|
14
|
+
not sealed. If the scanner cannot finish, `block-high` stops the seal.
|
|
15
|
+
`record` stops it only after `sealkeep scan policy fail-closed on`.
|
|
16
|
+
|
|
17
|
+
Medium findings, including the generic entropy detector, are advisory in every
|
|
18
|
+
mode. High findings are the built-in credential shapes plus any custom pattern
|
|
19
|
+
you mark `high`.
|
|
20
|
+
|
|
21
|
+
## What stays on the machine
|
|
22
|
+
|
|
23
|
+
The archive record stores detector kind, severity, count, and up to 32 line
|
|
24
|
+
numbers per kind. It does not store matched text or the masked preview.
|
|
25
|
+
`sealkeep scan <file>` still prints that preview locally. Logs and audit
|
|
26
|
+
entries store counts and kinds, not values.
|
|
27
|
+
|
|
28
|
+
## What can leave the machine
|
|
29
|
+
|
|
30
|
+
Nothing, until this machine opts in:
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
sealkeep scan policy plane https://control-plane.example
|
|
34
|
+
sealkeep scan policy rollup on
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Policy pull and rollup delivery are signed device requests. Without a
|
|
38
|
+
control-plane device key on this machine, nothing is sent. The control
|
|
39
|
+
plane then receives kind, severity, and count for each scanned archive.
|
|
40
|
+
It does not receive transcript content, matched values, paths, or line
|
|
41
|
+
numbers. Rollups older than 90 days are dropped on write and on read.
|
|
42
|
+
Turning rollup off, or clearing
|
|
43
|
+
the plane with `sealkeep scan policy plane off`, stops further delivery.
|
|
44
|
+
Importing a policy file does not turn rollup on.
|
|
45
|
+
|
|
46
|
+
## Custom patterns and shared policy
|
|
47
|
+
|
|
48
|
+
```sh
|
|
49
|
+
sealkeep scan pattern add acme-token high 'ACME[0-9]{8}'
|
|
50
|
+
sealkeep scan pattern list
|
|
51
|
+
sealkeep scan pattern remove acme-token
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
A pattern must be a valid regular expression of 4–200 characters, must not
|
|
55
|
+
match the empty string, and must contain four consecutive letters or digits.
|
|
56
|
+
Alternation, unbounded quantifiers (`*`, `+`, `?`), lookaround, and
|
|
57
|
+
backreferences are rejected. A closed bound such as `{8}` or `{1,16}` is
|
|
58
|
+
allowed. Each line is matched on a short deadline so a pattern cannot stall
|
|
59
|
+
a seal. Kinds cannot reuse a built-in name. At most 32 patterns are kept.
|
|
60
|
+
|
|
61
|
+
The scan hashes the exact byte range the seal will read. If that digest
|
|
62
|
+
does not match the sealed plaintext, a local archive is not published.
|
|
63
|
+
Streamed and chunked seals upload an immutable copy of the scanned bytes,
|
|
64
|
+
so a later edit of the live transcript is not what reaches the bucket. That
|
|
65
|
+
copy is removed when the seal finishes, including when later setup fails.
|
|
66
|
+
A copy left by a crash is removed on the next daemon start, or on the next
|
|
67
|
+
scanned seal, once the process that wrote it is gone.
|
|
68
|
+
Custom-pattern startup may take a few seconds; each line still has a 50ms
|
|
69
|
+
deadline.
|
|
70
|
+
|
|
71
|
+
`sealkeep scan policy export` prints a file with the mode, the fail-closed
|
|
72
|
+
flag, and the patterns. `import <file>` applies that file on another machine.
|
|
73
|
+
`pull` does the same from `GET /v1/secret-scan/policy`. Consent and the
|
|
74
|
+
control-plane URL stay on the machine that set them.
|
|
75
|
+
|
|
76
|
+
## Limits
|
|
77
|
+
|
|
78
|
+
The built-in detectors look for AWS key ids, AWS secrets in an assignment,
|
|
79
|
+
GitHub tokens, Google API keys, Slack tokens, private-key headers, JWTs with
|
|
80
|
+
a JSON header, and long high-entropy assignments. A secret in another shape
|
|
81
|
+
is not detected. A clean scan is not proof that the transcript is safe.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sealkeep",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.7",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Sealkeep by SPALA AI — your AI coding-agent history, sealed, searchable, and shared across your machines.",
|
|
6
6
|
"repository": {
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
"CONTROL_PLANE.md",
|
|
30
30
|
"THREAT_MODEL.md",
|
|
31
31
|
"THIRD_PARTY.md",
|
|
32
|
-
"CHANGELOG.md"
|
|
32
|
+
"CHANGELOG.md",
|
|
33
|
+
"docs/secret-scanning.md"
|
|
33
34
|
],
|
|
34
35
|
"scripts": {
|
|
35
36
|
"build": "tsc -p tsconfig.json",
|