threadwire 0.1.5 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +17 -4
- package/TELEGRAM-INGRESS.md +8 -2
- package/bin/isolated-runtime.js +5 -0
- package/bin/model-broker.js +5 -0
- package/docs/card-10520-plan.md +45 -0
- package/docs/container-runtime.md +3 -2
- package/docs/delegated-result-protocol.md +161 -0
- package/docs/evidence-artifacts.md +106 -0
- package/docs/isolated-provider-runtime.md +137 -0
- package/package.json +5 -1
- package/scripts/provider-shims/front-door.sh.template +18 -0
- package/scripts/verify-package.js +22 -1
- package/src/absolute-deadline.js +94 -0
- package/src/cli.js +362 -42
- package/src/context-budget-metrics.js +180 -0
- package/src/delegated-result-admission.js +377 -0
- package/src/docker-api.js +131 -0
- package/src/evidence-store.js +1472 -0
- package/src/isolated-runtime-client.js +149 -0
- package/src/isolated-runtime.js +982 -0
- package/src/isolated-state.js +409 -0
- package/src/isolated-worker.js +123 -0
- package/src/model-broker-policy.js +139 -0
- package/src/model-broker.js +313 -0
- package/src/mount-policy.js +28 -0
- package/src/normalized-output.js +68 -0
- package/src/notice-queue.js +22 -7
- package/src/providers/opencode.js +42 -18
- package/src/relay-write.js +44 -0
- package/src/relay.js +6 -6
- package/src/run-worker.js +158 -40
- package/src/telegram-ingress/command.js +16 -0
- package/src/telegram-ingress/config.js +96 -11
- package/src/telegram-ingress/core.js +201 -89
- package/src/telegram-ingress/http.js +17 -1
- package/src/telegram-webhook.js +22 -0
- package/src/types.js +2 -2
- package/src/worker-control.js +294 -0
- package/src/hermes-protocol.js +0 -126
|
@@ -0,0 +1,1472 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {createCipheriv, createDecipheriv, createHash, createHmac, randomBytes, timingSafeEqual} from "node:crypto"
|
|
4
|
+
import {spawnSync} from "node:child_process"
|
|
5
|
+
import {constants as fsConstants} from "node:fs"
|
|
6
|
+
import {mkdir, open, readdir, rename, unlink} from "node:fs/promises"
|
|
7
|
+
import {basename, isAbsolute, join, parse, relative} from "node:path"
|
|
8
|
+
import {StringDecoder} from "node:string_decoder"
|
|
9
|
+
|
|
10
|
+
const HANDLE_PATTERN = /^evidence_([A-Za-z0-9_-]{43})$/u
|
|
11
|
+
const META_SUFFIX = ".meta.json"
|
|
12
|
+
const PAYLOAD_SUFFIX = ".payload"
|
|
13
|
+
const PENDING_SUFFIX = ".pending"
|
|
14
|
+
const LOCK_FILE = ".threadwire-evidence.lock"
|
|
15
|
+
const ROOT_MARKER = ".threadwire-evidence-root.json"
|
|
16
|
+
const FLOCK = "/usr/bin/flock"
|
|
17
|
+
const ARTIFACT_DIRECTORY_OVERHEAD = 768
|
|
18
|
+
/** @type {EvidenceLimits} */
|
|
19
|
+
const DEFAULT_LIMITS = Object.freeze({
|
|
20
|
+
maxEventBytes: 1_048_576,
|
|
21
|
+
maxArtifactBytes: 64 * 1_048_576,
|
|
22
|
+
maxArtifactEvents: 100_000,
|
|
23
|
+
maxRunBytes: 128 * 1_048_576,
|
|
24
|
+
maxRunEvents: 200_000,
|
|
25
|
+
maxStoreBytes: 1024 * 1_048_576,
|
|
26
|
+
maxStoreEvents: 1_000_000,
|
|
27
|
+
maxArtifacts: 10_000,
|
|
28
|
+
maxReadBytes: 64 * 1024,
|
|
29
|
+
maxReadLines: 1_000,
|
|
30
|
+
retentionMs: 7 * 24 * 60 * 60 * 1000
|
|
31
|
+
})
|
|
32
|
+
const TEXT_CONTENT_TYPES = new Set(["application/json", "application/x-ndjson"])
|
|
33
|
+
|
|
34
|
+
export class EvidenceAccessError extends Error {
|
|
35
|
+
constructor() {
|
|
36
|
+
super("Evidence artifact is unavailable")
|
|
37
|
+
this.name = "EvidenceAccessError"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export class EvidenceCapacityError extends Error {
|
|
42
|
+
/** @param {string} message */
|
|
43
|
+
constructor(message = "Evidence capacity exceeded") {
|
|
44
|
+
super(message)
|
|
45
|
+
this.name = "EvidenceCapacityError"
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export class EvidenceStore {
|
|
50
|
+
/**
|
|
51
|
+
* @param {string} root
|
|
52
|
+
* @param {import("node:fs/promises").FileHandle} rootHandle
|
|
53
|
+
* @param {{dev: bigint, ino: bigint}} rootIdentity
|
|
54
|
+
* @param {import("node:fs/promises").FileHandle} lease
|
|
55
|
+
* @param {EvidenceLimits} limits
|
|
56
|
+
* @param {number} baseBytes
|
|
57
|
+
* @param {() => number} clock
|
|
58
|
+
* @param {string[]} runtimeRedactions
|
|
59
|
+
* @param {(point: string) => void | Promise<void>} durabilityObserver
|
|
60
|
+
*/
|
|
61
|
+
constructor(root, rootHandle, rootIdentity, lease, limits, baseBytes, clock, runtimeRedactions, durabilityObserver) {
|
|
62
|
+
this.root = root
|
|
63
|
+
this.rootHandle = rootHandle
|
|
64
|
+
this.rootIdentity = rootIdentity
|
|
65
|
+
this.lease = lease
|
|
66
|
+
this.closed = false
|
|
67
|
+
this.closing = false
|
|
68
|
+
/** @type {Promise<void> | undefined} */
|
|
69
|
+
this.closePromise = undefined
|
|
70
|
+
this.activeOperations = 0
|
|
71
|
+
/** @type {(() => void)[]} */
|
|
72
|
+
this.operationWaiters = []
|
|
73
|
+
this.limits = limits
|
|
74
|
+
this.clock = clock
|
|
75
|
+
this.runtimeRedactions = runtimeRedactions
|
|
76
|
+
this.durabilityObserver = durabilityObserver
|
|
77
|
+
/** @type {Map<string, EvidenceMetadata>} */
|
|
78
|
+
this.artifacts = new Map()
|
|
79
|
+
/** @type {Map<string, string>} */
|
|
80
|
+
this.capabilities = new Map()
|
|
81
|
+
/** @type {WeakSet<object>} */
|
|
82
|
+
this.scopes = new WeakSet()
|
|
83
|
+
/** @type {Map<string, Set<string>>} */
|
|
84
|
+
this.redactions = new Map()
|
|
85
|
+
/** @type {Map<string, number>} */
|
|
86
|
+
this.activeReaders = new Map()
|
|
87
|
+
this.storeBytes = baseBytes
|
|
88
|
+
this.storeEvents = 0
|
|
89
|
+
this.artifactSlots = 0
|
|
90
|
+
/** @type {Promise<unknown>} */
|
|
91
|
+
this.maintenance = Promise.resolve()
|
|
92
|
+
/** @type {Map<string, {bytes: number, events: number}>} */
|
|
93
|
+
this.runs = new Map()
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {{root: string, limits?: Partial<EvidenceLimits>, clock?: () => number, redactions?: string[], durabilityObserver?: (point: string) => void | Promise<void>}} options
|
|
98
|
+
*/
|
|
99
|
+
static async open(options) {
|
|
100
|
+
if (process.platform !== "linux" || !process.getuid) throw new Error("Secure evidence root is supported only on Linux")
|
|
101
|
+
const limits = validateLimits({...DEFAULT_LIMITS, ...options.limits})
|
|
102
|
+
const rootHandle = await openSecureRoot(options.root, options.durabilityObserver)
|
|
103
|
+
try {
|
|
104
|
+
const rootStats = await rootHandle.stat({bigint: true})
|
|
105
|
+
if (!rootStats.isDirectory() || rootStats.uid !== BigInt(process.getuid()) || (rootStats.mode & 0o777n) !== 0o700n) {
|
|
106
|
+
throw new Error("Secure evidence root must be an owned 0700 directory")
|
|
107
|
+
}
|
|
108
|
+
const pinnedRoot = `/proc/self/fd/${rootHandle.fd}`
|
|
109
|
+
await proveDedicatedRoot(pinnedRoot)
|
|
110
|
+
await options.durabilityObserver?.("root-pinned-before-lease")
|
|
111
|
+
const lease = await acquireLease(pinnedRoot)
|
|
112
|
+
const baseBytes = await storeBaseReservation(pinnedRoot)
|
|
113
|
+
if (baseBytes > limits.maxStoreBytes) {
|
|
114
|
+
await lease.close()
|
|
115
|
+
throw new EvidenceCapacityError()
|
|
116
|
+
}
|
|
117
|
+
const store = new EvidenceStore(
|
|
118
|
+
pinnedRoot,
|
|
119
|
+
rootHandle,
|
|
120
|
+
{dev: rootStats.dev, ino: rootStats.ino},
|
|
121
|
+
lease,
|
|
122
|
+
limits,
|
|
123
|
+
baseBytes,
|
|
124
|
+
options.clock ?? Date.now,
|
|
125
|
+
validateRedactions(options.redactions ?? []),
|
|
126
|
+
options.durabilityObserver ?? (() => {})
|
|
127
|
+
)
|
|
128
|
+
try {
|
|
129
|
+
await store.reconcile()
|
|
130
|
+
return store
|
|
131
|
+
} catch (error) {
|
|
132
|
+
await store.close().catch(() => {})
|
|
133
|
+
throw error
|
|
134
|
+
}
|
|
135
|
+
} catch (error) {
|
|
136
|
+
await rootHandle.close().catch(() => {})
|
|
137
|
+
throw error
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async close() {
|
|
142
|
+
if (this.closed) return
|
|
143
|
+
if (this.closePromise !== undefined) return this.closePromise
|
|
144
|
+
this.closing = true
|
|
145
|
+
this.closePromise = this.finishClose()
|
|
146
|
+
return this.closePromise
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async finishClose() {
|
|
150
|
+
await this.maintenance
|
|
151
|
+
if (this.activeOperations > 0) await new Promise((resolve) => this.operationWaiters.push(() => resolve(undefined)))
|
|
152
|
+
await this.assertPinnedRoot()
|
|
153
|
+
await this.lease.close()
|
|
154
|
+
this.closed = true
|
|
155
|
+
await this.rootHandle.close()
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async assertRootIdentity() {
|
|
159
|
+
if (this.closed || this.closing) throw new Error("Evidence store is closed")
|
|
160
|
+
await this.assertPinnedRoot()
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async assertPinnedRoot() {
|
|
164
|
+
const stats = await this.rootHandle.stat({bigint: true})
|
|
165
|
+
if (!stats.isDirectory() || stats.dev !== this.rootIdentity.dev || stats.ino !== this.rootIdentity.ino) {
|
|
166
|
+
throw new Error("Secure evidence root identity changed")
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** @param {{destinationId: string, runId: string}} identity */
|
|
171
|
+
createOwnerScope(identity) {
|
|
172
|
+
if (!validIdentity(identity.destinationId) || !validIdentity(identity.runId)) throw new Error("Invalid trusted evidence owner identity")
|
|
173
|
+
const scope = Object.freeze({destinationId: identity.destinationId, runId: identity.runId})
|
|
174
|
+
this.scopes.add(scope)
|
|
175
|
+
return scope
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* @param {{destinationId: string, runId: string}} owner
|
|
180
|
+
* @param {{contentType: string, redactions?: string[]}} options
|
|
181
|
+
*/
|
|
182
|
+
async createArtifact(owner, options) {
|
|
183
|
+
this.assertScope(owner)
|
|
184
|
+
const contentType = validateContentType(options.contentType)
|
|
185
|
+
const redactions = [...this.runtimeRedactions, ...validateRedactions(options.redactions ?? [])]
|
|
186
|
+
const id = randomBytes(32).toString("base64url")
|
|
187
|
+
const handle = `evidence_${randomBytes(32).toString("base64url")}`
|
|
188
|
+
const now = this.clock()
|
|
189
|
+
const metadata = /** @type {EvidenceMetadata} */ ({
|
|
190
|
+
version: 2, id, state: "pending", owner: {...owner}, contentType,
|
|
191
|
+
capabilityHash: capabilityHash(handle),
|
|
192
|
+
policyVersion: 1, framingVersion: 1,
|
|
193
|
+
redactionPolicy: {algorithm: "aes-256-gcm", nonce: "", ciphertext: "", tag: ""},
|
|
194
|
+
payloadCommitment: "A".repeat(43),
|
|
195
|
+
bytes: 0, storedBytes: 0, lines: 0, events: 0, createdAt: now, expiresAt: now + this.limits.retentionMs
|
|
196
|
+
})
|
|
197
|
+
metadata.redactionPolicy = encryptRedactions(handle, redactions, metadata)
|
|
198
|
+
const overheadBytes = durableArtifactReservation(metadata)
|
|
199
|
+
const finishOperation = this.beginOperation()
|
|
200
|
+
if (this.artifactSlots >= this.limits.maxArtifacts) {
|
|
201
|
+
finishOperation()
|
|
202
|
+
throw new EvidenceCapacityError("Evidence artifact count exceeded")
|
|
203
|
+
}
|
|
204
|
+
try {
|
|
205
|
+
this.reserve(metadata.owner.runId, overheadBytes, 0)
|
|
206
|
+
} catch (error) {
|
|
207
|
+
finishOperation()
|
|
208
|
+
throw error
|
|
209
|
+
}
|
|
210
|
+
metadata.reservedBytes = overheadBytes
|
|
211
|
+
// JavaScript runs through here without yielding: reserve the slot before
|
|
212
|
+
// the first await so concurrent creations observe this pending artifact.
|
|
213
|
+
this.artifactSlots += 1
|
|
214
|
+
/** @type {import("node:fs/promises").FileHandle | undefined} */
|
|
215
|
+
let file
|
|
216
|
+
try {
|
|
217
|
+
await atomicJson(this.root, `${id}${META_SUFFIX}`, metadata)
|
|
218
|
+
await this.durabilityObserver("pending-metadata-durable")
|
|
219
|
+
file = await open(this.path(id, PENDING_SUFFIX), "wx", 0o600)
|
|
220
|
+
await this.durabilityObserver("pending-payload-open")
|
|
221
|
+
} catch (error) {
|
|
222
|
+
await file?.close().catch(() => {})
|
|
223
|
+
await safeUnlink(this.path(id, PENDING_SUFFIX))
|
|
224
|
+
await safeUnlink(this.path(id, META_SUFFIX))
|
|
225
|
+
this.reserve(metadata.owner.runId, -overheadBytes, 0)
|
|
226
|
+
this.artifactSlots -= 1
|
|
227
|
+
finishOperation()
|
|
228
|
+
throw error
|
|
229
|
+
}
|
|
230
|
+
if (!file) throw new Error("Evidence pending payload is unavailable")
|
|
231
|
+
this.artifacts.set(id, metadata)
|
|
232
|
+
this.capabilities.set(metadata.capabilityHash, id)
|
|
233
|
+
this.redactions.set(id, new Set(redactions))
|
|
234
|
+
return new EvidenceWriter(this, metadata, file, handle, finishOperation, redactions, owner)
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** @param {object} owner @param {unknown} request */
|
|
238
|
+
async read(owner, request) {
|
|
239
|
+
this.assertScope(owner)
|
|
240
|
+
const validated = validateEvidenceReadRequest(request, this.limits)
|
|
241
|
+
const finishOperation = this.beginOperation()
|
|
242
|
+
const id = this.lookupCapability(validated.handle)
|
|
243
|
+
if (!id) {
|
|
244
|
+
finishOperation()
|
|
245
|
+
throw new EvidenceAccessError()
|
|
246
|
+
}
|
|
247
|
+
this.activeReaders.set(id, (this.activeReaders.get(id) ?? 0) + 1)
|
|
248
|
+
try {
|
|
249
|
+
const metadata = this.artifacts.get(id)
|
|
250
|
+
if (!metadata || metadata.state !== "ready" || metadata.expiresAt <= this.clock() || !sameOwner(metadata.owner, owner)) {
|
|
251
|
+
throw new EvidenceAccessError()
|
|
252
|
+
}
|
|
253
|
+
const payloadPath = this.path(id, PAYLOAD_SUFFIX)
|
|
254
|
+
const file = await open(payloadPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW).catch(() => { throw new EvidenceAccessError() })
|
|
255
|
+
let selected
|
|
256
|
+
let redacted = false
|
|
257
|
+
let truncated = false
|
|
258
|
+
try {
|
|
259
|
+
const payloadStats = await file.stat()
|
|
260
|
+
if (
|
|
261
|
+
!payloadStats.isFile() || payloadStats.nlink !== 1 || payloadStats.size !== metadata.storedBytes
|
|
262
|
+
|| process.getuid?.() !== payloadStats.uid || (payloadStats.mode & 0o777) !== 0o600
|
|
263
|
+
) throw new EvidenceAccessError()
|
|
264
|
+
const payloadMac = createHmac("sha256", payloadKey(validated.handle))
|
|
265
|
+
const events = framedChunks(file, payloadStats.size, this.limits.maxEventBytes, payloadMac)
|
|
266
|
+
const secrets = decryptRedactions(validated.handle, metadata.redactionPolicy, metadata)
|
|
267
|
+
const chunks = isText(metadata.contentType)
|
|
268
|
+
? redactedChunks(events, secrets)
|
|
269
|
+
: redactedBinaryChunks(events, secrets)
|
|
270
|
+
const selection = await selectChunks(chunks, validated, this.limits.maxReadBytes)
|
|
271
|
+
const expected = Buffer.from(metadata.payloadCommitment, "base64url")
|
|
272
|
+
const actual = payloadMac.digest()
|
|
273
|
+
if (expected.length !== actual.length || !timingSafeEqual(expected, actual)) throw new EvidenceAccessError()
|
|
274
|
+
selected = selection.data
|
|
275
|
+
redacted = selection.redacted
|
|
276
|
+
truncated = selection.truncated
|
|
277
|
+
} finally {
|
|
278
|
+
await file.close()
|
|
279
|
+
}
|
|
280
|
+
if (!isText(metadata.contentType)) {
|
|
281
|
+
const binary = selected.subarray(0, Math.min(selected.length, this.limits.maxReadBytes))
|
|
282
|
+
return fitSerializedResult({
|
|
283
|
+
contentType: metadata.contentType,
|
|
284
|
+
data: binary.toString("base64"),
|
|
285
|
+
encoding: "base64",
|
|
286
|
+
redacted,
|
|
287
|
+
truncated: truncated || binary.length < selected.length
|
|
288
|
+
}, this.limits.maxReadBytes)
|
|
289
|
+
}
|
|
290
|
+
return fitSerializedResult({
|
|
291
|
+
contentType: metadata.contentType,
|
|
292
|
+
data: decodeUtf8(selected),
|
|
293
|
+
redacted,
|
|
294
|
+
truncated
|
|
295
|
+
}, this.limits.maxReadBytes)
|
|
296
|
+
} finally {
|
|
297
|
+
const readers = (this.activeReaders.get(id) ?? 1) - 1
|
|
298
|
+
if (readers === 0) this.activeReaders.delete(id)
|
|
299
|
+
else this.activeReaders.set(id, readers)
|
|
300
|
+
finishOperation()
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** @param {unknown} request */
|
|
305
|
+
async readBearer(request) {
|
|
306
|
+
const validated = validateEvidenceReadRequest(request, this.limits)
|
|
307
|
+
const id = this.lookupCapability(validated.handle)
|
|
308
|
+
if (!id) throw new EvidenceAccessError()
|
|
309
|
+
const metadata = this.artifacts.get(id)
|
|
310
|
+
if (!metadata) throw new EvidenceAccessError()
|
|
311
|
+
const owner = Object.freeze({...metadata.owner})
|
|
312
|
+
this.scopes.add(owner)
|
|
313
|
+
try {
|
|
314
|
+
return await this.read(owner, validated)
|
|
315
|
+
} finally {
|
|
316
|
+
this.scopes.delete(owner)
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** @param {{chatId: string, threadId: number | null, senderId: string}} requester @param {unknown} request */
|
|
321
|
+
async readTelegram(requester, request) {
|
|
322
|
+
const validated = validateEvidenceReadRequest(request, this.limits)
|
|
323
|
+
const id = this.lookupCapability(validated.handle)
|
|
324
|
+
if (!id) throw new EvidenceAccessError()
|
|
325
|
+
const metadata = this.artifacts.get(id)
|
|
326
|
+
if (!metadata || metadata.owner.destinationId !== evidenceTelegramDestination(requester, requester.senderId)) throw new EvidenceAccessError()
|
|
327
|
+
const owner = Object.freeze({...metadata.owner})
|
|
328
|
+
this.scopes.add(owner)
|
|
329
|
+
try {
|
|
330
|
+
return await this.read(owner, validated)
|
|
331
|
+
} finally {
|
|
332
|
+
this.scopes.delete(owner)
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
cleanup() {
|
|
337
|
+
const finish = this.beginOperation()
|
|
338
|
+
return this.enqueueMaintenance(() => this.cleanupOnce()).finally(finish)
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
async cleanupOnce() {
|
|
342
|
+
await this.assertRootIdentity()
|
|
343
|
+
const now = this.clock()
|
|
344
|
+
for (const [id, metadata] of [...this.artifacts]) {
|
|
345
|
+
if (metadata.state !== "ready" || metadata.expiresAt > now) continue
|
|
346
|
+
if ((this.activeReaders.get(id) ?? 0) > 0) continue
|
|
347
|
+
await safeUnlink(this.path(id, PAYLOAD_SUFFIX))
|
|
348
|
+
await safeUnlink(this.path(id, META_SUFFIX))
|
|
349
|
+
await syncDirectory(this.root)
|
|
350
|
+
metadata.state = "expired"
|
|
351
|
+
this.artifacts.delete(id)
|
|
352
|
+
this.capabilities.delete(metadata.capabilityHash)
|
|
353
|
+
this.release(metadata)
|
|
354
|
+
this.artifactSlots -= 1
|
|
355
|
+
this.redactions.delete(id)
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** @param {object} owner @param {string} handle */
|
|
360
|
+
remove(owner, handle) {
|
|
361
|
+
const finish = this.beginOperation()
|
|
362
|
+
return this.enqueueMaintenance(() => this.removeOnce(owner, handle)).finally(finish)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** @param {object} owner @param {string} handle */
|
|
366
|
+
async removeOnce(owner, handle) {
|
|
367
|
+
await this.assertRootIdentity()
|
|
368
|
+
this.assertScope(owner)
|
|
369
|
+
const id = this.lookupCapability(handle)
|
|
370
|
+
if (!id) throw new EvidenceAccessError()
|
|
371
|
+
const metadata = this.artifacts.get(id)
|
|
372
|
+
if (!metadata || metadata.state !== "ready" || !sameOwner(metadata.owner, owner)) throw new EvidenceAccessError()
|
|
373
|
+
if ((this.activeReaders.get(id) ?? 0) > 0) throw new Error("Evidence artifact is busy")
|
|
374
|
+
await safeUnlink(this.path(id, PAYLOAD_SUFFIX))
|
|
375
|
+
await safeUnlink(this.path(id, META_SUFFIX))
|
|
376
|
+
await syncDirectory(this.root)
|
|
377
|
+
metadata.state = "removed"
|
|
378
|
+
this.artifacts.delete(id)
|
|
379
|
+
this.capabilities.delete(metadata.capabilityHash)
|
|
380
|
+
this.release(metadata)
|
|
381
|
+
this.artifactSlots -= 1
|
|
382
|
+
this.redactions.delete(id)
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/** @template T @param {() => Promise<T>} operation @returns {Promise<T>} */
|
|
386
|
+
enqueueMaintenance(operation) {
|
|
387
|
+
const result = this.maintenance.then(operation)
|
|
388
|
+
this.maintenance = result.catch(() => {})
|
|
389
|
+
return result
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** @param {string} handle */
|
|
393
|
+
async inspectFilesForTest(handle) {
|
|
394
|
+
const id = this.lookupCapability(handle)
|
|
395
|
+
if (!id) throw new EvidenceAccessError()
|
|
396
|
+
return {payload: this.path(id, PAYLOAD_SUFFIX), metadata: this.path(id, META_SUFFIX)}
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
async reconcile() {
|
|
400
|
+
await this.assertPinnedRoot()
|
|
401
|
+
const names = await readdir(this.root)
|
|
402
|
+
const candidates = new Map()
|
|
403
|
+
const abandonedTemps = []
|
|
404
|
+
for (const name of names) {
|
|
405
|
+
const match = /^([A-Za-z0-9_-]{43})(\.meta\.json|\.payload|\.pending)$/u.exec(name)
|
|
406
|
+
if (name === LOCK_FILE || name === ROOT_MARKER) continue
|
|
407
|
+
if (/^\.(?:[A-Za-z0-9_-]{43}\.meta\.json|\.threadwire-evidence-root\.json)\.[a-f0-9]{24}\.tmp$/u.test(name)) {
|
|
408
|
+
abandonedTemps.push(name)
|
|
409
|
+
continue
|
|
410
|
+
}
|
|
411
|
+
if (!match) {
|
|
412
|
+
throw new Error("Evidence store contains an unknown entry")
|
|
413
|
+
}
|
|
414
|
+
const id = match[1]
|
|
415
|
+
const suffix = match[2]
|
|
416
|
+
if (id === undefined || suffix === undefined) continue
|
|
417
|
+
const group = candidates.get(id) ?? new Set()
|
|
418
|
+
group.add(suffix)
|
|
419
|
+
candidates.set(id, group)
|
|
420
|
+
}
|
|
421
|
+
for (const name of abandonedTemps) await safeUnlink(join(this.root, basename(name)))
|
|
422
|
+
for (const [id, files] of candidates) {
|
|
423
|
+
let metadata
|
|
424
|
+
try {
|
|
425
|
+
const metaStats = await safeRegularFile(this.path(id, META_SUFFIX))
|
|
426
|
+
if (!metaStats || metaStats.nlink !== 1) throw new Error("invalid")
|
|
427
|
+
metadata = await readBoundedJson(this.path(id, META_SUFFIX), 512 * 1024)
|
|
428
|
+
if (!validMetadata(metadata, id, this.limits) || metadata.state !== "ready" || metadata.expiresAt <= this.clock()) throw new Error("invalid")
|
|
429
|
+
const payloadStats = await safeRegularFile(this.path(id, PAYLOAD_SUFFIX))
|
|
430
|
+
if (!payloadStats || payloadStats.nlink !== 1 || payloadStats.size !== metadata.storedBytes || files.has(PENDING_SUFFIX)) throw new Error("invalid")
|
|
431
|
+
const counters = await inspectFramedPayload(this.path(id, PAYLOAD_SUFFIX), payloadStats.size, this.limits.maxEventBytes)
|
|
432
|
+
if (counters.bytes !== metadata.bytes || counters.lines !== metadata.lines || counters.events !== metadata.events) throw new Error("invalid")
|
|
433
|
+
if (this.artifactSlots >= this.limits.maxArtifacts) throw new EvidenceCapacityError()
|
|
434
|
+
this.reserve(metadata.owner.runId, metadata.reservedBytes + metadata.storedBytes, metadata.events)
|
|
435
|
+
this.artifactSlots += 1
|
|
436
|
+
this.artifacts.set(id, metadata)
|
|
437
|
+
this.capabilities.set(metadata.capabilityHash, id)
|
|
438
|
+
this.redactions.set(id, new Set(this.runtimeRedactions))
|
|
439
|
+
} catch {
|
|
440
|
+
await Promise.all([...files].map((suffix) => safeUnlink(this.path(id, suffix))))
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** @param {string} runId @param {number} bytes @param {number} events */
|
|
446
|
+
reserve(runId, bytes, events) {
|
|
447
|
+
const run = this.runs.get(runId) ?? {bytes: 0, events: 0}
|
|
448
|
+
if (
|
|
449
|
+
this.storeBytes + bytes > this.limits.maxStoreBytes || this.storeEvents + events > this.limits.maxStoreEvents
|
|
450
|
+
|| run.bytes + bytes > this.limits.maxRunBytes || run.events + events > this.limits.maxRunEvents
|
|
451
|
+
) throw new EvidenceCapacityError()
|
|
452
|
+
this.storeBytes += bytes
|
|
453
|
+
this.storeEvents += events
|
|
454
|
+
run.bytes += bytes
|
|
455
|
+
run.events += events
|
|
456
|
+
this.runs.set(runId, run)
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** @param {EvidenceMetadata} metadata */
|
|
460
|
+
release(metadata) {
|
|
461
|
+
this.storeBytes -= metadata.storedBytes + metadata.reservedBytes
|
|
462
|
+
this.storeEvents -= metadata.events
|
|
463
|
+
const run = this.runs.get(metadata.owner.runId)
|
|
464
|
+
if (run) {
|
|
465
|
+
run.bytes -= metadata.storedBytes + metadata.reservedBytes
|
|
466
|
+
run.events -= metadata.events
|
|
467
|
+
if (run.bytes === 0 && run.events === 0) this.runs.delete(metadata.owner.runId)
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/** @param {object} owner */
|
|
472
|
+
assertScope(owner) {
|
|
473
|
+
if (!this.scopes.has(owner)) throw new EvidenceAccessError()
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
beginOperation() {
|
|
477
|
+
if (this.closed || this.closing) throw new Error("Evidence store is closed")
|
|
478
|
+
this.activeOperations += 1
|
|
479
|
+
let finished = false
|
|
480
|
+
return () => {
|
|
481
|
+
if (finished) return
|
|
482
|
+
finished = true
|
|
483
|
+
this.activeOperations -= 1
|
|
484
|
+
if (this.activeOperations === 0) {
|
|
485
|
+
for (const resolveWaiter of this.operationWaiters.splice(0)) resolveWaiter()
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/** @param {string} handle */
|
|
491
|
+
lookupCapability(handle) {
|
|
492
|
+
const candidate = Buffer.from(capabilityHash(handle), "base64url")
|
|
493
|
+
let found
|
|
494
|
+
for (const [hash, id] of this.capabilities) {
|
|
495
|
+
const stored = Buffer.from(hash, "base64url")
|
|
496
|
+
if (stored.length === candidate.length && timingSafeEqual(stored, candidate)) found = id
|
|
497
|
+
}
|
|
498
|
+
return found
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/** @param {string} id @param {string} suffix */
|
|
502
|
+
path(id, suffix) {
|
|
503
|
+
return join(this.root, `${id}${suffix}`)
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** @param {{chatId: string, threadId: number | null}} target @param {string} senderId */
|
|
508
|
+
export function evidenceTelegramDestination(target, senderId) {
|
|
509
|
+
if (!validIdentity(senderId)) throw new Error("Invalid trusted Telegram sender identity")
|
|
510
|
+
return `telegram:${target.chatId}:${target.threadId ?? "dm"}:sender:${senderId}`
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
class EvidenceWriter {
|
|
514
|
+
/** @type {object | undefined} */
|
|
515
|
+
#ownerScope
|
|
516
|
+
|
|
517
|
+
/** @param {EvidenceStore} store @param {EvidenceMetadata} metadata @param {import("node:fs/promises").FileHandle} file @param {string} handle @param {() => void} finishOperation @param {string[]} redactions @param {object} ownerScope */
|
|
518
|
+
constructor(store, metadata, file, handle, finishOperation, redactions, ownerScope) {
|
|
519
|
+
this.store = store
|
|
520
|
+
this.metadata = metadata
|
|
521
|
+
this.file = file
|
|
522
|
+
this.handle = handle
|
|
523
|
+
this.finishOperation = finishOperation
|
|
524
|
+
this.redactions = redactions
|
|
525
|
+
this.#ownerScope = ownerScope
|
|
526
|
+
this.payloadMac = createHmac("sha256", payloadKey(handle))
|
|
527
|
+
this.closed = false
|
|
528
|
+
this.decoder = new StringDecoder("utf8")
|
|
529
|
+
this.queue = Promise.resolve()
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** @param {string} kind @param {Buffer | string} chunk */
|
|
533
|
+
append(kind, chunk) {
|
|
534
|
+
if (!/^[a-z][a-z0-9_-]{0,31}$/u.test(kind)) return Promise.reject(new Error("Invalid evidence event kind"))
|
|
535
|
+
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
|
536
|
+
const kindBytes = Buffer.from(kind, "ascii")
|
|
537
|
+
const header = Buffer.allocUnsafe(1 + kindBytes.length + 4)
|
|
538
|
+
header[0] = kindBytes.length
|
|
539
|
+
kindBytes.copy(header, 1)
|
|
540
|
+
header.writeUInt32BE(bytes.length, 1 + kindBytes.length)
|
|
541
|
+
const storedBytes = header.length + bytes.length
|
|
542
|
+
this.queue = this.queue.then(async () => {
|
|
543
|
+
if (this.closed) throw new Error("Evidence writer is closed")
|
|
544
|
+
if (bytes.length > this.store.limits.maxEventBytes) throw new EvidenceCapacityError("Evidence event exceeds capacity")
|
|
545
|
+
if (
|
|
546
|
+
this.metadata.storedBytes + storedBytes > this.store.limits.maxArtifactBytes
|
|
547
|
+
|| this.metadata.events + 1 > this.store.limits.maxArtifactEvents
|
|
548
|
+
) throw new EvidenceCapacityError()
|
|
549
|
+
this.store.reserve(this.metadata.owner.runId, storedBytes, 1)
|
|
550
|
+
try {
|
|
551
|
+
await writeAll(this.file, header)
|
|
552
|
+
await writeAll(this.file, bytes)
|
|
553
|
+
} catch (error) {
|
|
554
|
+
this.store.reserve(this.metadata.owner.runId, -storedBytes, -1)
|
|
555
|
+
throw error
|
|
556
|
+
}
|
|
557
|
+
this.payloadMac.update(header)
|
|
558
|
+
this.payloadMac.update(bytes)
|
|
559
|
+
this.metadata.bytes += bytes.length
|
|
560
|
+
this.metadata.storedBytes += storedBytes
|
|
561
|
+
this.metadata.events += 1
|
|
562
|
+
for (const byte of bytes) if (byte === 0x0a) this.metadata.lines += 1
|
|
563
|
+
})
|
|
564
|
+
return this.queue
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
async finalize() {
|
|
568
|
+
await this.queue
|
|
569
|
+
if (this.closed) throw new Error("Evidence writer is closed")
|
|
570
|
+
this.closed = true
|
|
571
|
+
try {
|
|
572
|
+
return await this.store.enqueueMaintenance(() => this.finalizeOnce())
|
|
573
|
+
} finally {
|
|
574
|
+
this.#releaseOwnerScope()
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
async finalizeOnce() {
|
|
579
|
+
try {
|
|
580
|
+
await this.file.sync()
|
|
581
|
+
await this.store.durabilityObserver("payload-fsynced")
|
|
582
|
+
await this.file.close()
|
|
583
|
+
await rename(this.store.path(this.metadata.id, PENDING_SUFFIX), this.store.path(this.metadata.id, PAYLOAD_SUFFIX))
|
|
584
|
+
await this.store.durabilityObserver("payload-renamed")
|
|
585
|
+
this.metadata.state = "ready"
|
|
586
|
+
this.metadata.payloadCommitment = this.payloadMac.digest("base64url")
|
|
587
|
+
this.metadata.redactionPolicy = encryptRedactions(this.handle, this.redactions, this.metadata)
|
|
588
|
+
this.metadata.reservedBytes = durableArtifactReservation(this.metadata)
|
|
589
|
+
await atomicJson(this.store.root, `${this.metadata.id}${META_SUFFIX}`, this.metadata)
|
|
590
|
+
await this.store.durabilityObserver("ready-metadata-directory-fsynced")
|
|
591
|
+
this.finishOperation()
|
|
592
|
+
return {
|
|
593
|
+
contentType: this.metadata.contentType,
|
|
594
|
+
bytes: this.metadata.bytes,
|
|
595
|
+
lines: this.metadata.lines,
|
|
596
|
+
events: this.metadata.events
|
|
597
|
+
}
|
|
598
|
+
} catch (error) {
|
|
599
|
+
await this.file.close().catch(() => {})
|
|
600
|
+
this.store.release(this.metadata)
|
|
601
|
+
this.store.artifactSlots -= 1
|
|
602
|
+
this.store.artifacts.delete(this.metadata.id)
|
|
603
|
+
this.store.capabilities.delete(this.metadata.capabilityHash)
|
|
604
|
+
this.store.redactions.delete(this.metadata.id)
|
|
605
|
+
await safeUnlink(this.store.path(this.metadata.id, PENDING_SUFFIX))
|
|
606
|
+
await safeUnlink(this.store.path(this.metadata.id, PAYLOAD_SUFFIX))
|
|
607
|
+
await safeUnlink(this.store.path(this.metadata.id, META_SUFFIX))
|
|
608
|
+
await syncDirectory(this.store.root)
|
|
609
|
+
this.finishOperation()
|
|
610
|
+
throw error
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
async abort() {
|
|
615
|
+
await this.queue.catch(() => {})
|
|
616
|
+
if (this.closed) return
|
|
617
|
+
this.closed = true
|
|
618
|
+
try {
|
|
619
|
+
await this.store.enqueueMaintenance(() => this.abortOnce())
|
|
620
|
+
} finally {
|
|
621
|
+
this.#releaseOwnerScope()
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
#releaseOwnerScope() {
|
|
626
|
+
if (this.#ownerScope !== undefined) this.#ownerScope = undefined
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
async abortOnce() {
|
|
630
|
+
await this.file.close().catch(() => {})
|
|
631
|
+
this.store.release(this.metadata)
|
|
632
|
+
this.store.artifactSlots -= 1
|
|
633
|
+
this.store.artifacts.delete(this.metadata.id)
|
|
634
|
+
this.store.capabilities.delete(this.metadata.capabilityHash)
|
|
635
|
+
this.store.redactions.delete(this.metadata.id)
|
|
636
|
+
await safeUnlink(this.store.path(this.metadata.id, PENDING_SUFFIX))
|
|
637
|
+
await safeUnlink(this.store.path(this.metadata.id, META_SUFFIX))
|
|
638
|
+
await syncDirectory(this.store.root)
|
|
639
|
+
this.finishOperation()
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
/** @param {unknown} request @param {Partial<EvidenceLimits>} [limits] @returns {ValidatedRead} */
|
|
644
|
+
export function validateEvidenceReadRequest(request, limits = DEFAULT_LIMITS) {
|
|
645
|
+
const maxReadBytes = limits.maxReadBytes ?? DEFAULT_LIMITS.maxReadBytes
|
|
646
|
+
const maxReadLines = limits.maxReadLines ?? DEFAULT_LIMITS.maxReadLines
|
|
647
|
+
if (!plainObject(request)) throw new Error("Invalid evidence handle")
|
|
648
|
+
const handleDescriptor = Object.getOwnPropertyDescriptor(request, "handle")
|
|
649
|
+
if (!handleDescriptor || handleDescriptor.get !== undefined || handleDescriptor.set !== undefined || typeof handleDescriptor.value !== "string" || !HANDLE_PATTERN.test(handleDescriptor.value)) {
|
|
650
|
+
throw new Error("Invalid evidence handle")
|
|
651
|
+
}
|
|
652
|
+
const selectors = ["bytes", "lines", "query"].filter((field) => Object.prototype.hasOwnProperty.call(request, field))
|
|
653
|
+
if (selectors.length !== 1) throw new Error("Evidence retrieval requires exactly one bounded selector")
|
|
654
|
+
if (request.bytes !== undefined) {
|
|
655
|
+
if (!plainObject(request.bytes) || !exactDataKeys(request.bytes, ["offset", "limit"]) || !nonnegative(request.bytes.offset) || !positive(request.bytes.limit) || typeof request.bytes.limit !== "number" || request.bytes.limit > maxReadBytes) {
|
|
656
|
+
throw new Error("Invalid bounded evidence byte range")
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
if (request.lines !== undefined) {
|
|
660
|
+
if (!plainObject(request.lines) || !exactDataKeys(request.lines, ["start", "limit"]) || !positive(request.lines.start) || !positive(request.lines.limit) || typeof request.lines.limit !== "number" || request.lines.limit > maxReadLines) {
|
|
661
|
+
throw new Error("Invalid bounded evidence line range")
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
if (request.query !== undefined) {
|
|
665
|
+
if (
|
|
666
|
+
!plainObject(request.query) || typeof request.query.literal !== "string" || request.query.literal.length === 0
|
|
667
|
+
|| request.query.literal.length > 256 || request.query.limit !== 1
|
|
668
|
+
|| !nonnegative(request.query.contextBytes) || typeof request.query.contextBytes !== "number" || request.query.contextBytes > maxReadBytes
|
|
669
|
+
|| !exactDataKeys(request.query, ["literal", "limit", "contextBytes"])
|
|
670
|
+
) throw new Error("Invalid bounded literal evidence query")
|
|
671
|
+
}
|
|
672
|
+
if (!exactOptionalDataKeys(request, ["handle", selectors[0]])) throw new Error("Invalid evidence retrieval field")
|
|
673
|
+
return /** @type {ValidatedRead} */ (request)
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
/** @param {string} root @param {string} name @param {unknown} value */
|
|
677
|
+
async function atomicJson(root, name, value) {
|
|
678
|
+
const temporary = join(root, `.${name}.${randomBytes(12).toString("hex")}.tmp`)
|
|
679
|
+
const target = join(root, name)
|
|
680
|
+
const file = await open(temporary, "wx", 0o600)
|
|
681
|
+
try {
|
|
682
|
+
await file.writeFile(`${JSON.stringify(value)}\n`)
|
|
683
|
+
await file.sync()
|
|
684
|
+
await file.close()
|
|
685
|
+
await rename(temporary, target)
|
|
686
|
+
const directory = await open(root, "r")
|
|
687
|
+
await directory.sync()
|
|
688
|
+
await directory.close()
|
|
689
|
+
} catch (error) {
|
|
690
|
+
await file.close().catch(() => {})
|
|
691
|
+
await safeUnlink(temporary)
|
|
692
|
+
throw error
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** @param {string} root */
|
|
697
|
+
async function acquireLease(root) {
|
|
698
|
+
const executable = await open(FLOCK, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW)
|
|
699
|
+
const lock = await open(join(root, LOCK_FILE), fsConstants.O_CREAT | fsConstants.O_RDWR | fsConstants.O_NOFOLLOW, 0o600)
|
|
700
|
+
try {
|
|
701
|
+
const stats = await executable.stat()
|
|
702
|
+
if (!stats.isFile() || stats.uid !== 0 || (stats.mode & 0o022) !== 0) throw new Error("Trusted flock executable is unavailable")
|
|
703
|
+
const lockStats = await lock.stat()
|
|
704
|
+
if (
|
|
705
|
+
!lockStats.isFile() || lockStats.nlink !== 1 || lockStats.uid !== process.getuid?.()
|
|
706
|
+
|| (lockStats.mode & 0o777) !== 0o600
|
|
707
|
+
) throw new Error("Invalid evidence store lock file")
|
|
708
|
+
const result = spawnSync("/proc/self/fd/4", ["-n", "3"], {
|
|
709
|
+
shell: false,
|
|
710
|
+
stdio: ["ignore", "pipe", "pipe", lock.fd, executable.fd],
|
|
711
|
+
encoding: "utf8",
|
|
712
|
+
timeout: 2_000,
|
|
713
|
+
maxBuffer: 256
|
|
714
|
+
})
|
|
715
|
+
if (result.error || result.status !== 0 || result.stdout !== "" || result.stderr !== "") {
|
|
716
|
+
throw new Error("Evidence store is already open exclusively")
|
|
717
|
+
}
|
|
718
|
+
return lock
|
|
719
|
+
} catch (error) {
|
|
720
|
+
await lock.close().catch(() => {})
|
|
721
|
+
throw error
|
|
722
|
+
} finally {
|
|
723
|
+
await executable.close()
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/** @param {string} root */
|
|
728
|
+
async function proveDedicatedRoot(root) {
|
|
729
|
+
const markerPath = join(root, ROOT_MARKER)
|
|
730
|
+
const names = await readdir(root)
|
|
731
|
+
if (names.length === 0) {
|
|
732
|
+
await atomicJson(root, ROOT_MARKER, {version: 1, purpose: "threadwire-evidence"})
|
|
733
|
+
return
|
|
734
|
+
}
|
|
735
|
+
const marker = await readBoundedJson(markerPath, 256).catch(() => null)
|
|
736
|
+
if (
|
|
737
|
+
!plainObject(marker) || !exactKeys(marker, ["version", "purpose"])
|
|
738
|
+
|| marker.version !== 1 || marker.purpose !== "threadwire-evidence"
|
|
739
|
+
) throw new Error("Secure evidence root is not a proven dedicated store")
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
/** @param {string} requested @param {((point: string) => void | Promise<void>) | undefined} observer */
|
|
743
|
+
async function openSecureRoot(requested, observer) {
|
|
744
|
+
if (!isAbsolute(requested) || requested.includes("\0")) throw new Error("Secure evidence root must be an absolute path")
|
|
745
|
+
if (requested.split("/").slice(1).some((component) => component === "" || component === "." || component === "..")) {
|
|
746
|
+
throw new Error("Invalid secure evidence root path")
|
|
747
|
+
}
|
|
748
|
+
const components = relative(parse(requested).root, requested).split("/")
|
|
749
|
+
if (components.length === 0 || components.some((component) => component === "" || component === "." || component === "..")) {
|
|
750
|
+
throw new Error("Invalid secure evidence root path")
|
|
751
|
+
}
|
|
752
|
+
let directory = await open("/", fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW)
|
|
753
|
+
try {
|
|
754
|
+
for (let index = 0; index < components.length; index += 1) {
|
|
755
|
+
const stats = await directory.stat()
|
|
756
|
+
if (!trustedRootAncestor(stats)) throw new Error("Untrusted evidence root ancestor")
|
|
757
|
+
const component = /** @type {string} */ (components[index])
|
|
758
|
+
const childPath = `/proc/self/fd/${directory.fd}/${component}`
|
|
759
|
+
await observer?.(`root-traversal-before-open:${index}`)
|
|
760
|
+
let child
|
|
761
|
+
try {
|
|
762
|
+
child = await open(childPath, fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW)
|
|
763
|
+
} catch (error) {
|
|
764
|
+
if (/** @type {NodeJS.ErrnoException} */ (error).code !== "ENOENT" || index !== components.length - 1) throw error
|
|
765
|
+
await mkdir(childPath, {mode: 0o700})
|
|
766
|
+
child = await open(childPath, fsConstants.O_RDONLY | fsConstants.O_DIRECTORY | fsConstants.O_NOFOLLOW)
|
|
767
|
+
}
|
|
768
|
+
await directory.close()
|
|
769
|
+
directory = child
|
|
770
|
+
}
|
|
771
|
+
const finalStats = await directory.stat()
|
|
772
|
+
if (
|
|
773
|
+
!finalStats.isDirectory() || finalStats.uid !== process.getuid?.()
|
|
774
|
+
|| (finalStats.mode & 0o777) !== 0o700
|
|
775
|
+
) throw new Error("Secure evidence root must be an owned 0700 directory")
|
|
776
|
+
return directory
|
|
777
|
+
} catch (error) {
|
|
778
|
+
await directory.close().catch(() => {})
|
|
779
|
+
throw error
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/** @param {import("node:fs").Stats} stats */
|
|
784
|
+
function trustedRootAncestor(stats) {
|
|
785
|
+
const mode = stats.mode & 0o7777
|
|
786
|
+
if (!stats.isDirectory()) return false
|
|
787
|
+
if (stats.uid === process.getuid?.()) return (mode & 0o022) === 0
|
|
788
|
+
return stats.uid === 0 && ((mode & 0o022) === 0 || (mode & 0o1000) !== 0)
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
/** @param {string} root */
|
|
792
|
+
async function syncDirectory(root) {
|
|
793
|
+
const directory = await open(root, fsConstants.O_RDONLY | fsConstants.O_DIRECTORY)
|
|
794
|
+
try {
|
|
795
|
+
await directory.sync()
|
|
796
|
+
} finally {
|
|
797
|
+
await directory.close()
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/** @param {string} root */
|
|
802
|
+
async function storeBaseReservation(root) {
|
|
803
|
+
let bytes = 2 * ARTIFACT_DIRECTORY_OVERHEAD
|
|
804
|
+
for (const name of [ROOT_MARKER, LOCK_FILE]) {
|
|
805
|
+
const file = await open(join(root, name), fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW)
|
|
806
|
+
try {
|
|
807
|
+
const stats = await file.stat()
|
|
808
|
+
if (!stats.isFile() || stats.nlink !== 1) throw new Error("Invalid evidence store control file")
|
|
809
|
+
bytes += stats.size
|
|
810
|
+
} finally {
|
|
811
|
+
await file.close()
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
return bytes
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
/** @param {string} path @param {number} limit */
|
|
818
|
+
async function readBoundedJson(path, limit) {
|
|
819
|
+
const file = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW)
|
|
820
|
+
try {
|
|
821
|
+
const stats = await file.stat()
|
|
822
|
+
if (
|
|
823
|
+
!stats.isFile() || stats.nlink !== 1 || stats.size > limit
|
|
824
|
+
|| process.getuid?.() !== stats.uid || (stats.mode & 0o777) !== 0o600
|
|
825
|
+
) throw new Error("Invalid bounded JSON file")
|
|
826
|
+
const bytes = Buffer.alloc(stats.size)
|
|
827
|
+
const {bytesRead} = await file.read(bytes, 0, bytes.length, 0)
|
|
828
|
+
if (bytesRead !== bytes.length) throw new Error("Invalid bounded JSON file")
|
|
829
|
+
return JSON.parse(bytes.toString("utf8"))
|
|
830
|
+
} finally {
|
|
831
|
+
await file.close()
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
/** @param {string} path */
|
|
836
|
+
async function safeRegularFile(path) {
|
|
837
|
+
try {
|
|
838
|
+
const stats = await import("node:fs/promises").then(({lstat}) => lstat(path))
|
|
839
|
+
return stats.isFile() && !stats.isSymbolicLink() && stats.nlink === 1
|
|
840
|
+
&& process.getuid?.() === stats.uid && (stats.mode & 0o777) === 0o600
|
|
841
|
+
? stats
|
|
842
|
+
: null
|
|
843
|
+
} catch (error) {
|
|
844
|
+
if (/** @type {NodeJS.ErrnoException} */ (error).code === "ENOENT") return null
|
|
845
|
+
throw error
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
/** @param {string} path */
|
|
850
|
+
async function safeUnlink(path) {
|
|
851
|
+
try {
|
|
852
|
+
await unlink(path)
|
|
853
|
+
} catch (error) {
|
|
854
|
+
if (/** @type {NodeJS.ErrnoException} */ (error).code !== "ENOENT") throw error
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/** @param {Buffer} buffer */
|
|
859
|
+
function decodeUtf8(buffer) {
|
|
860
|
+
const decoded = buffer.toString("utf8")
|
|
861
|
+
if (Buffer.from(decoded).compare(buffer) !== 0) throw new Error("Evidence byte range must align to a UTF-8 boundary")
|
|
862
|
+
return decoded
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
/** @param {Buffer} buffer @param {number} limit */
|
|
866
|
+
function utf8Prefix(buffer, limit) {
|
|
867
|
+
if (buffer.length <= limit) return buffer
|
|
868
|
+
let end = limit
|
|
869
|
+
while (end > 0 && ((buffer[end] ?? 0) & 0xc0) === 0x80) end -= 1
|
|
870
|
+
const lead = buffer[end]
|
|
871
|
+
if (lead !== undefined) {
|
|
872
|
+
const width = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4
|
|
873
|
+
if (end + width > limit) return buffer.subarray(0, end)
|
|
874
|
+
}
|
|
875
|
+
return buffer.subarray(0, limit)
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/** @param {{contentType: string, data: string, encoding?: string, redacted: boolean, truncated: boolean}} result @param {number} limit */
|
|
879
|
+
function fitSerializedResult(result, limit) {
|
|
880
|
+
if (Buffer.byteLength(`${JSON.stringify(result)}\n`) <= limit) return result
|
|
881
|
+
let bytes = Buffer.from(result.data, "utf8")
|
|
882
|
+
while (bytes.length > 0) {
|
|
883
|
+
let nextLength = Math.max(0, bytes.length - Math.max(1, Math.ceil(bytes.length / 8)))
|
|
884
|
+
if (result.encoding === "base64") nextLength -= nextLength % 4
|
|
885
|
+
bytes = Buffer.from(utf8Prefix(bytes, nextLength))
|
|
886
|
+
result.data = bytes.toString("utf8")
|
|
887
|
+
result.truncated = true
|
|
888
|
+
if (Buffer.byteLength(`${JSON.stringify(result)}\n`) <= limit) return result
|
|
889
|
+
}
|
|
890
|
+
if (Buffer.byteLength(`${JSON.stringify(result)}\n`) > limit) throw new EvidenceCapacityError("Evidence response metadata exceeds read capacity")
|
|
891
|
+
return result
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
/** @param {Buffer} buffer @param {number} offset */
|
|
895
|
+
function utf8Start(buffer, offset) {
|
|
896
|
+
while (offset < buffer.length && ((buffer[offset] ?? 0) & 0xc0) === 0x80) offset += 1
|
|
897
|
+
return offset
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
/** @param {Buffer} buffer @param {number} offset */
|
|
901
|
+
function utf8End(buffer, offset) {
|
|
902
|
+
while (offset > 0 && offset < buffer.length && ((buffer[offset] ?? 0) & 0xc0) === 0x80) offset -= 1
|
|
903
|
+
return offset
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
/** @param {string} text @param {string[]} secrets */
|
|
907
|
+
function redactText(text, secrets) {
|
|
908
|
+
const merged = redactionRanges(text, secrets)
|
|
909
|
+
if (merged.length === 0) return {text, redacted: false}
|
|
910
|
+
return {text: applyRedactionRanges(text, merged, text.length), redacted: true}
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
/** @param {string} text @param {string[]} secrets */
|
|
914
|
+
function redactionRanges(text, secrets) {
|
|
915
|
+
/** @type {{start: number, end: number}[]} */
|
|
916
|
+
const ranges = []
|
|
917
|
+
for (const secret of secrets) {
|
|
918
|
+
let offset = 0
|
|
919
|
+
while (offset <= text.length - secret.length) {
|
|
920
|
+
const index = text.indexOf(secret, offset)
|
|
921
|
+
if (index < 0) break
|
|
922
|
+
ranges.push({start: index, end: index + secret.length})
|
|
923
|
+
offset = index + 1
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
ranges.sort((left, right) => left.start - right.start || left.end - right.end)
|
|
927
|
+
/** @type {{start: number, end: number}[]} */
|
|
928
|
+
const merged = []
|
|
929
|
+
for (const range of ranges) {
|
|
930
|
+
const previous = merged.at(-1)
|
|
931
|
+
if (previous && range.start <= previous.end) previous.end = Math.max(previous.end, range.end)
|
|
932
|
+
else merged.push({...range})
|
|
933
|
+
}
|
|
934
|
+
return merged
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
/** @param {string} text @param {{start: number, end: number}[]} ranges @param {number} end */
|
|
938
|
+
function applyRedactionRanges(text, ranges, end) {
|
|
939
|
+
let output = ""
|
|
940
|
+
let offset = 0
|
|
941
|
+
for (const range of ranges) {
|
|
942
|
+
if (range.start >= end) break
|
|
943
|
+
output += `${text.slice(offset, range.start)}[REDACTED]`
|
|
944
|
+
offset = range.end
|
|
945
|
+
}
|
|
946
|
+
return output + text.slice(offset, end)
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/** @param {string} text @param {string[]} secrets @param {number} safeEnd */
|
|
950
|
+
function redactStablePrefix(text, secrets, safeEnd) {
|
|
951
|
+
const ranges = redactionRanges(text, secrets)
|
|
952
|
+
let consumed = safeEnd
|
|
953
|
+
for (const range of ranges) if (range.start < safeEnd && range.end > consumed) consumed = range.end
|
|
954
|
+
return {
|
|
955
|
+
text: applyRedactionRanges(text, ranges, consumed),
|
|
956
|
+
consumed,
|
|
957
|
+
redacted: ranges.some((range) => range.start < consumed)
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
/**
|
|
962
|
+
* @param {import("node:fs/promises").FileHandle} file
|
|
963
|
+
* @param {number} size
|
|
964
|
+
* @param {number} maxEventBytes
|
|
965
|
+
* @param {import("node:crypto").Hmac} payloadMac
|
|
966
|
+
* @yields {{data: Buffer, redacted: boolean}}
|
|
967
|
+
*/
|
|
968
|
+
async function* framedChunks(file, size, maxEventBytes, payloadMac) {
|
|
969
|
+
let position = 0
|
|
970
|
+
while (position < size) {
|
|
971
|
+
const length = await readExact(file, position, 1)
|
|
972
|
+
payloadMac.update(length)
|
|
973
|
+
const kindLength = length[0] ?? 0
|
|
974
|
+
if (kindLength < 1 || kindLength > 32) throw new EvidenceAccessError()
|
|
975
|
+
const header = await readExact(file, position + 1, kindLength + 4)
|
|
976
|
+
payloadMac.update(header)
|
|
977
|
+
const kind = header.subarray(0, kindLength).toString("ascii")
|
|
978
|
+
if (!/^[a-z][a-z0-9_-]{0,31}$/u.test(kind)) throw new EvidenceAccessError()
|
|
979
|
+
const payloadLength = header.readUInt32BE(kindLength)
|
|
980
|
+
if (payloadLength > maxEventBytes || position + 1 + header.length + payloadLength > size) throw new EvidenceAccessError()
|
|
981
|
+
position += 1 + header.length
|
|
982
|
+
let remaining = payloadLength
|
|
983
|
+
while (remaining > 0) {
|
|
984
|
+
const data = await readExact(file, position, Math.min(16_384, remaining))
|
|
985
|
+
payloadMac.update(data)
|
|
986
|
+
position += data.length
|
|
987
|
+
remaining -= data.length
|
|
988
|
+
yield {data, redacted: false}
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
/** @param {string} path @param {number} size @param {number} maxEventBytes */
|
|
994
|
+
async function inspectFramedPayload(path, size, maxEventBytes) {
|
|
995
|
+
const file = await open(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW)
|
|
996
|
+
let position = 0
|
|
997
|
+
let bytes = 0
|
|
998
|
+
let lines = 0
|
|
999
|
+
let events = 0
|
|
1000
|
+
try {
|
|
1001
|
+
while (position < size) {
|
|
1002
|
+
const length = await readExact(file, position, 1)
|
|
1003
|
+
const kindLength = length[0] ?? 0
|
|
1004
|
+
if (kindLength < 1 || kindLength > 32) throw new Error("invalid frame")
|
|
1005
|
+
const header = await readExact(file, position + 1, kindLength + 4)
|
|
1006
|
+
const kind = header.subarray(0, kindLength).toString("ascii")
|
|
1007
|
+
const payloadLength = header.readUInt32BE(kindLength)
|
|
1008
|
+
if (
|
|
1009
|
+
!/^[a-z][a-z0-9_-]{0,31}$/u.test(kind) || payloadLength > maxEventBytes
|
|
1010
|
+
|| position + 1 + header.length + payloadLength > size
|
|
1011
|
+
) throw new Error("invalid frame")
|
|
1012
|
+
position += 1 + header.length
|
|
1013
|
+
let remaining = payloadLength
|
|
1014
|
+
while (remaining > 0) {
|
|
1015
|
+
const chunk = await readExact(file, position, Math.min(16_384, remaining))
|
|
1016
|
+
for (const byte of chunk) if (byte === 0x0a) lines += 1
|
|
1017
|
+
position += chunk.length
|
|
1018
|
+
remaining -= chunk.length
|
|
1019
|
+
}
|
|
1020
|
+
bytes += payloadLength
|
|
1021
|
+
events += 1
|
|
1022
|
+
}
|
|
1023
|
+
return {bytes, lines, events}
|
|
1024
|
+
} finally {
|
|
1025
|
+
await file.close()
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
/**
|
|
1030
|
+
* @param {AsyncGenerator<{data: Buffer, redacted: boolean}>} chunks
|
|
1031
|
+
* @param {string[]} secrets
|
|
1032
|
+
* @yields {{data: Buffer, redacted: boolean}}
|
|
1033
|
+
*/
|
|
1034
|
+
async function* redactedChunks(chunks, secrets) {
|
|
1035
|
+
const decoder = new StringDecoder("utf8")
|
|
1036
|
+
const overlap = Math.max(1, ...secrets.map((secret) => secret.length))
|
|
1037
|
+
let carry = ""
|
|
1038
|
+
let anyRedacted = false
|
|
1039
|
+
for await (const chunk of chunks) {
|
|
1040
|
+
carry += decoder.write(chunk.data)
|
|
1041
|
+
const safeEnd = safeStringEnd(carry, Math.max(0, carry.length - overlap))
|
|
1042
|
+
if (safeEnd === 0) continue
|
|
1043
|
+
const part = redactStablePrefix(carry, secrets, safeEnd)
|
|
1044
|
+
anyRedacted ||= part.redacted
|
|
1045
|
+
carry = carry.slice(part.consumed)
|
|
1046
|
+
const data = Buffer.from(part.text, "utf8")
|
|
1047
|
+
if (data.length > 0) yield {data, redacted: anyRedacted}
|
|
1048
|
+
}
|
|
1049
|
+
carry += decoder.end()
|
|
1050
|
+
const part = redactText(carry, secrets)
|
|
1051
|
+
anyRedacted ||= part.redacted
|
|
1052
|
+
const data = Buffer.from(part.text, "utf8")
|
|
1053
|
+
if (data.length > 0) yield {data, redacted: anyRedacted}
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* @param {AsyncGenerator<{data: Buffer, redacted: boolean}>} chunks
|
|
1058
|
+
* @param {string[]} secrets
|
|
1059
|
+
* @yields {{data: Buffer, redacted: boolean}}
|
|
1060
|
+
*/
|
|
1061
|
+
async function* redactedBinaryChunks(chunks, secrets) {
|
|
1062
|
+
const patterns = secrets.map((secret) => Buffer.from(secret, "utf8"))
|
|
1063
|
+
const overlap = Math.max(1, ...patterns.map((pattern) => pattern.length))
|
|
1064
|
+
let carry = Buffer.alloc(0)
|
|
1065
|
+
let anyRedacted = false
|
|
1066
|
+
for await (const chunk of chunks) {
|
|
1067
|
+
carry = Buffer.concat([carry, chunk.data])
|
|
1068
|
+
const safeEnd = Math.max(0, carry.length - overlap)
|
|
1069
|
+
if (safeEnd === 0) continue
|
|
1070
|
+
const part = redactStableBinaryPrefix(carry, patterns, safeEnd)
|
|
1071
|
+
anyRedacted ||= part.redacted
|
|
1072
|
+
carry = carry.subarray(part.consumed)
|
|
1073
|
+
if (part.data.length > 0) yield {data: part.data, redacted: anyRedacted}
|
|
1074
|
+
}
|
|
1075
|
+
const part = redactStableBinaryPrefix(carry, patterns, carry.length)
|
|
1076
|
+
anyRedacted ||= part.redacted
|
|
1077
|
+
if (part.data.length > 0) yield {data: part.data, redacted: anyRedacted}
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
/** @param {Buffer} value @param {Buffer[]} patterns @param {number} safeEnd */
|
|
1081
|
+
function redactStableBinaryPrefix(value, patterns, safeEnd) {
|
|
1082
|
+
/** @type {{start: number, end: number}[]} */
|
|
1083
|
+
const ranges = []
|
|
1084
|
+
for (const pattern of patterns) {
|
|
1085
|
+
let offset = 0
|
|
1086
|
+
while (offset <= value.length - pattern.length) {
|
|
1087
|
+
const index = value.indexOf(pattern, offset)
|
|
1088
|
+
if (index < 0) break
|
|
1089
|
+
ranges.push({start: index, end: index + pattern.length})
|
|
1090
|
+
offset = index + 1
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
ranges.sort((left, right) => left.start - right.start || left.end - right.end)
|
|
1094
|
+
/** @type {{start: number, end: number}[]} */
|
|
1095
|
+
const merged = []
|
|
1096
|
+
for (const range of ranges) {
|
|
1097
|
+
const previous = merged.at(-1)
|
|
1098
|
+
if (previous && range.start <= previous.end) previous.end = Math.max(previous.end, range.end)
|
|
1099
|
+
else merged.push({...range})
|
|
1100
|
+
}
|
|
1101
|
+
let consumed = safeEnd
|
|
1102
|
+
for (const range of merged) if (range.start < safeEnd && range.end > consumed) consumed = range.end
|
|
1103
|
+
const output = []
|
|
1104
|
+
let offset = 0
|
|
1105
|
+
for (const range of merged) {
|
|
1106
|
+
if (range.start >= consumed) break
|
|
1107
|
+
output.push(value.subarray(offset, range.start), Buffer.from("[REDACTED]"))
|
|
1108
|
+
offset = range.end
|
|
1109
|
+
}
|
|
1110
|
+
output.push(value.subarray(offset, consumed))
|
|
1111
|
+
return {data: Buffer.concat(output), consumed, redacted: merged.some((range) => range.start < consumed)}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
/** @param {import("node:fs/promises").FileHandle} file @param {number} position @param {number} length */
|
|
1115
|
+
async function readExact(file, position, length) {
|
|
1116
|
+
const buffer = Buffer.allocUnsafe(length)
|
|
1117
|
+
let offset = 0
|
|
1118
|
+
while (offset < length) {
|
|
1119
|
+
const {bytesRead} = await file.read(buffer, offset, length - offset, position + offset)
|
|
1120
|
+
if (bytesRead <= 0) throw new EvidenceAccessError()
|
|
1121
|
+
offset += bytesRead
|
|
1122
|
+
}
|
|
1123
|
+
return buffer
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
/** @param {import("node:fs/promises").FileHandle} file @param {Buffer} buffer */
|
|
1127
|
+
async function writeAll(file, buffer) {
|
|
1128
|
+
let offset = 0
|
|
1129
|
+
while (offset < buffer.length) {
|
|
1130
|
+
const {bytesWritten} = await file.write(buffer, offset, buffer.length - offset)
|
|
1131
|
+
if (bytesWritten <= 0) throw new Error("Evidence write made no progress")
|
|
1132
|
+
offset += bytesWritten
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
/** @param {string} value @param {number} end */
|
|
1137
|
+
function safeStringEnd(value, end) {
|
|
1138
|
+
if (end > 0 && end < value.length) {
|
|
1139
|
+
const code = value.charCodeAt(end - 1)
|
|
1140
|
+
if (code >= 0xd800 && code <= 0xdbff) return end - 1
|
|
1141
|
+
}
|
|
1142
|
+
return end
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
/** @param {AsyncGenerator<{data: Buffer, redacted: boolean}>} chunks @param {ValidatedRead} request @param {number} maxBytes */
|
|
1146
|
+
async function selectChunks(chunks, request, maxBytes) {
|
|
1147
|
+
if (request.bytes) return selectBytes(chunks, request.bytes, maxBytes)
|
|
1148
|
+
if (request.lines) return selectLines(chunks, request.lines, maxBytes)
|
|
1149
|
+
return selectQuery(chunks, /** @type {NonNullable<ValidatedRead["query"]>} */ (request.query), maxBytes)
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
/** @param {AsyncGenerator<{data: Buffer, redacted: boolean}>} chunks @param {{offset: number, limit: number}} range @param {number} maxBytes */
|
|
1153
|
+
async function selectBytes(chunks, range, maxBytes) {
|
|
1154
|
+
const limit = Math.min(range.limit, maxBytes)
|
|
1155
|
+
let skipped = 0
|
|
1156
|
+
let redacted = false
|
|
1157
|
+
/** @type {Buffer[]} */
|
|
1158
|
+
const output = []
|
|
1159
|
+
let length = 0
|
|
1160
|
+
let hasMore = false
|
|
1161
|
+
let complete = false
|
|
1162
|
+
for await (const chunk of chunks) {
|
|
1163
|
+
redacted ||= chunk.redacted
|
|
1164
|
+
if (complete) continue
|
|
1165
|
+
if (skipped + chunk.data.length <= range.offset) {
|
|
1166
|
+
skipped += chunk.data.length
|
|
1167
|
+
continue
|
|
1168
|
+
}
|
|
1169
|
+
const start = Math.max(0, range.offset - skipped)
|
|
1170
|
+
const available = chunk.data.subarray(start)
|
|
1171
|
+
const take = Math.min(available.length, limit - length)
|
|
1172
|
+
if (take > 0) {
|
|
1173
|
+
output.push(available.subarray(0, take))
|
|
1174
|
+
length += take
|
|
1175
|
+
}
|
|
1176
|
+
skipped += chunk.data.length
|
|
1177
|
+
if (take < available.length) {
|
|
1178
|
+
hasMore = true
|
|
1179
|
+
complete = true
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
return {data: Buffer.concat(output, length), redacted, truncated: range.offset > 0 || hasMore}
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
/** @param {AsyncGenerator<{data: Buffer, redacted: boolean}>} chunks @param {{start: number, limit: number}} range @param {number} maxBytes */
|
|
1186
|
+
async function selectLines(chunks, range, maxBytes) {
|
|
1187
|
+
let line = 1
|
|
1188
|
+
let redacted = false
|
|
1189
|
+
let length = 0
|
|
1190
|
+
let complete = 0
|
|
1191
|
+
let selectionDone = false
|
|
1192
|
+
let truncated = range.start > 1
|
|
1193
|
+
/** @type {Buffer[]} */
|
|
1194
|
+
const output = []
|
|
1195
|
+
for await (const chunk of chunks) {
|
|
1196
|
+
redacted ||= chunk.redacted
|
|
1197
|
+
if (selectionDone) {
|
|
1198
|
+
if (chunk.data.length > 0) truncated = true
|
|
1199
|
+
continue
|
|
1200
|
+
}
|
|
1201
|
+
for (let index = 0; index < chunk.data.length; index += 1) {
|
|
1202
|
+
const byte = /** @type {number} */ (chunk.data[index])
|
|
1203
|
+
if (line >= range.start && complete < range.limit && length < maxBytes) {
|
|
1204
|
+
output.push(Buffer.from([byte]))
|
|
1205
|
+
length += 1
|
|
1206
|
+
}
|
|
1207
|
+
if (byte === 0x0a) {
|
|
1208
|
+
if (line >= range.start) complete += 1
|
|
1209
|
+
line += 1
|
|
1210
|
+
if (complete >= range.limit) {
|
|
1211
|
+
truncated ||= index + 1 < chunk.data.length
|
|
1212
|
+
selectionDone = true
|
|
1213
|
+
break
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
if (length >= maxBytes) {
|
|
1217
|
+
truncated = true
|
|
1218
|
+
selectionDone = true
|
|
1219
|
+
break
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
return {data: utf8Prefix(Buffer.concat(output, length), maxBytes), redacted, truncated}
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
/** @param {AsyncGenerator<{data: Buffer, redacted: boolean}>} chunks @param {{literal: string, limit: number, contextBytes: number}} query @param {number} maxBytes */
|
|
1227
|
+
async function selectQuery(chunks, query, maxBytes) {
|
|
1228
|
+
const needle = Buffer.from(query.literal, "utf8")
|
|
1229
|
+
const keep = Math.min(maxBytes, query.contextBytes + needle.length - 1)
|
|
1230
|
+
let prefix = Buffer.alloc(0)
|
|
1231
|
+
let dropped = false
|
|
1232
|
+
let redacted = false
|
|
1233
|
+
let match = -1
|
|
1234
|
+
let matchStart = 0
|
|
1235
|
+
/** @type {{data: Buffer, redacted: boolean, truncated: boolean} | null} */
|
|
1236
|
+
let result = null
|
|
1237
|
+
for await (const chunk of chunks) {
|
|
1238
|
+
redacted ||= chunk.redacted
|
|
1239
|
+
if (result) {
|
|
1240
|
+
result.redacted ||= chunk.redacted
|
|
1241
|
+
if (chunk.data.length > 0) result.truncated = true
|
|
1242
|
+
continue
|
|
1243
|
+
}
|
|
1244
|
+
const combined = Buffer.concat([prefix, chunk.data])
|
|
1245
|
+
const index = match >= 0 ? match : combined.indexOf(needle)
|
|
1246
|
+
if (index >= 0) {
|
|
1247
|
+
match = index
|
|
1248
|
+
const start = utf8Start(combined, Math.max(0, index - query.contextBytes))
|
|
1249
|
+
const requestedEnd = index + needle.length + query.contextBytes
|
|
1250
|
+
if (combined.length < requestedEnd && combined.length < maxBytes) {
|
|
1251
|
+
prefix = combined
|
|
1252
|
+
matchStart = start
|
|
1253
|
+
continue
|
|
1254
|
+
}
|
|
1255
|
+
const end = utf8End(combined, Math.min(combined.length, requestedEnd))
|
|
1256
|
+
const data = utf8Prefix(combined.subarray(matchStart || start, end), maxBytes)
|
|
1257
|
+
result = {
|
|
1258
|
+
data,
|
|
1259
|
+
redacted,
|
|
1260
|
+
truncated: dropped || start > 0 || end < combined.length || data.length < end - start || requestedEnd > combined.length
|
|
1261
|
+
}
|
|
1262
|
+
continue
|
|
1263
|
+
}
|
|
1264
|
+
if (combined.length > keep) {
|
|
1265
|
+
prefix = combined.subarray(combined.length - keep)
|
|
1266
|
+
dropped = true
|
|
1267
|
+
} else prefix = combined
|
|
1268
|
+
}
|
|
1269
|
+
return result ?? {data: Buffer.alloc(0), redacted, truncated: dropped}
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
/** @param {string} value */
|
|
1273
|
+
function isText(value) {
|
|
1274
|
+
const base = value.split(";")[0]?.trim().toLowerCase() ?? ""
|
|
1275
|
+
return base.startsWith("text/") || TEXT_CONTENT_TYPES.has(base)
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
/** @param {string} handle */
|
|
1279
|
+
function capabilityHash(handle) {
|
|
1280
|
+
if (!HANDLE_PATTERN.test(handle)) throw new Error("Invalid evidence handle")
|
|
1281
|
+
return createHash("sha256").update(handle, "utf8").digest("base64url")
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
/** @param {string} handle @param {string[]} redactions @param {EvidenceMetadata} metadata @returns {EvidenceMetadata["redactionPolicy"]} */
|
|
1285
|
+
function encryptRedactions(handle, redactions, metadata) {
|
|
1286
|
+
const nonce = randomBytes(12)
|
|
1287
|
+
const cipher = createCipheriv("aes-256-gcm", redactionKey(handle), nonce)
|
|
1288
|
+
cipher.setAAD(redactionAad(metadata))
|
|
1289
|
+
const ciphertext = Buffer.concat([cipher.update(JSON.stringify(redactions), "utf8"), cipher.final()])
|
|
1290
|
+
return {
|
|
1291
|
+
algorithm: "aes-256-gcm",
|
|
1292
|
+
nonce: nonce.toString("base64url"),
|
|
1293
|
+
ciphertext: ciphertext.toString("base64url"),
|
|
1294
|
+
tag: cipher.getAuthTag().toString("base64url")
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1298
|
+
/** @param {string} handle @param {EvidenceMetadata["redactionPolicy"]} policy @param {EvidenceMetadata} metadata */
|
|
1299
|
+
function decryptRedactions(handle, policy, metadata) {
|
|
1300
|
+
try {
|
|
1301
|
+
const decipher = createDecipheriv("aes-256-gcm", redactionKey(handle), Buffer.from(policy.nonce, "base64url"))
|
|
1302
|
+
decipher.setAAD(redactionAad(metadata))
|
|
1303
|
+
decipher.setAuthTag(Buffer.from(policy.tag, "base64url"))
|
|
1304
|
+
const plaintext = Buffer.concat([
|
|
1305
|
+
decipher.update(Buffer.from(policy.ciphertext, "base64url")),
|
|
1306
|
+
decipher.final()
|
|
1307
|
+
]).toString("utf8")
|
|
1308
|
+
return validateRedactions(JSON.parse(plaintext))
|
|
1309
|
+
} catch {
|
|
1310
|
+
throw new EvidenceAccessError()
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
/** @param {EvidenceMetadata} metadata */
|
|
1315
|
+
function redactionAad(metadata) {
|
|
1316
|
+
return Buffer.from(JSON.stringify([
|
|
1317
|
+
"threadwire-evidence-policy", metadata.policyVersion, metadata.version, metadata.framingVersion, metadata.id,
|
|
1318
|
+
metadata.state,
|
|
1319
|
+
metadata.capabilityHash, metadata.owner.destinationId, metadata.owner.runId,
|
|
1320
|
+
metadata.contentType, metadata.createdAt, metadata.expiresAt,
|
|
1321
|
+
metadata.bytes, metadata.storedBytes, metadata.reservedBytes,
|
|
1322
|
+
metadata.lines, metadata.events, metadata.payloadCommitment
|
|
1323
|
+
]), "utf8")
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/** @param {EvidenceMetadata} metadata */
|
|
1327
|
+
function durableArtifactReservation(metadata) {
|
|
1328
|
+
return 2_048 + metadata.redactionPolicy.ciphertext.length * 2 + ARTIFACT_DIRECTORY_OVERHEAD
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/** @param {string} handle */
|
|
1332
|
+
function redactionKey(handle) {
|
|
1333
|
+
return createHash("sha256").update("threadwire-evidence-redaction-v1\0").update(handle, "utf8").digest()
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
/** @param {string} handle */
|
|
1337
|
+
function payloadKey(handle) {
|
|
1338
|
+
return createHash("sha256").update("threadwire-evidence-payload-v1\0").update(handle, "utf8").digest()
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
/** @param {unknown} value @returns {value is Record<string, unknown>} */
|
|
1342
|
+
function plainObject(value) {
|
|
1343
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) && (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null)
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
/** @param {unknown} value */
|
|
1347
|
+
function positive(value) {
|
|
1348
|
+
return Number.isSafeInteger(value) && /** @type {number} */ (value) > 0
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
/** @param {unknown} value */
|
|
1352
|
+
function nonnegative(value) {
|
|
1353
|
+
return Number.isSafeInteger(value) && /** @type {number} */ (value) >= 0
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
/** @param {string} value */
|
|
1357
|
+
function validIdentity(value) {
|
|
1358
|
+
return typeof value === "string" && value.length > 0 && value.length <= 256
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
/** @param {string} value */
|
|
1362
|
+
function validateContentType(value) {
|
|
1363
|
+
if (typeof value !== "string" || !/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+(?:;\s*charset=utf-8)?$/iu.test(value) || value.length > 96) {
|
|
1364
|
+
throw new Error("Invalid evidence content type")
|
|
1365
|
+
}
|
|
1366
|
+
return value.toLowerCase()
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
/** @param {string[]} values */
|
|
1370
|
+
function validateRedactions(values) {
|
|
1371
|
+
if (!Array.isArray(values) || values.length > 64 || values.some((value) => typeof value !== "string" || value.length < 4 || value.length > 4096)) {
|
|
1372
|
+
throw new Error("Invalid evidence redactions")
|
|
1373
|
+
}
|
|
1374
|
+
return values
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
/** @param {Record<string, unknown>} left @param {object} right */
|
|
1378
|
+
function sameOwner(left, right) {
|
|
1379
|
+
return left.destinationId === /** @type {{destinationId?: unknown}} */ (right).destinationId
|
|
1380
|
+
&& left.runId === /** @type {{runId?: unknown}} */ (right).runId
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
/** @param {unknown} value @param {string} id @param {EvidenceLimits} limits */
|
|
1384
|
+
function validMetadata(value, id, limits) {
|
|
1385
|
+
return plainObject(value) && value.version === 2 && value.id === id && value.state === "ready"
|
|
1386
|
+
&& exactKeys(value, [
|
|
1387
|
+
"version", "policyVersion", "framingVersion", "id", "state", "owner", "contentType", "capabilityHash",
|
|
1388
|
+
"redactionPolicy", "payloadCommitment", "bytes", "storedBytes", "reservedBytes", "lines", "events", "createdAt", "expiresAt"
|
|
1389
|
+
])
|
|
1390
|
+
&& value.policyVersion === 1 && value.framingVersion === 1
|
|
1391
|
+
&& plainObject(value.owner) && typeof value.owner.destinationId === "string" && validIdentity(value.owner.destinationId)
|
|
1392
|
+
&& typeof value.owner.runId === "string" && validIdentity(value.owner.runId)
|
|
1393
|
+
&& exactKeys(value.owner, ["destinationId", "runId"])
|
|
1394
|
+
&& typeof value.capabilityHash === "string" && /^[A-Za-z0-9_-]{43}$/u.test(value.capabilityHash)
|
|
1395
|
+
&& validRedactionPolicy(value.redactionPolicy)
|
|
1396
|
+
&& typeof value.contentType === "string" && validContentType(value.contentType)
|
|
1397
|
+
&& typeof value.bytes === "number" && typeof value.storedBytes === "number"
|
|
1398
|
+
&& typeof value.reservedBytes === "number" && positive(value.reservedBytes)
|
|
1399
|
+
&& nonnegative(value.bytes) && nonnegative(value.storedBytes) && value.bytes <= value.storedBytes
|
|
1400
|
+
&& value.bytes <= limits.maxArtifactBytes && value.storedBytes <= limits.maxArtifactBytes
|
|
1401
|
+
&& typeof value.payloadCommitment === "string" && /^[A-Za-z0-9_-]{43}$/u.test(value.payloadCommitment)
|
|
1402
|
+
&& value.reservedBytes === durableArtifactReservation(/** @type {EvidenceMetadata} */ (value))
|
|
1403
|
+
&& typeof value.lines === "number" && nonnegative(value.lines) && value.lines <= value.bytes
|
|
1404
|
+
&& typeof value.events === "number" && nonnegative(value.events) && value.events <= limits.maxArtifactEvents
|
|
1405
|
+
&& typeof value.createdAt === "number" && typeof value.expiresAt === "number"
|
|
1406
|
+
&& nonnegative(value.createdAt) && positive(value.expiresAt) && value.createdAt < value.expiresAt
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
/** @param {unknown} value */
|
|
1410
|
+
function validRedactionPolicy(value) {
|
|
1411
|
+
return plainObject(value) && value.algorithm === "aes-256-gcm"
|
|
1412
|
+
&& exactKeys(value, ["algorithm", "nonce", "ciphertext", "tag"])
|
|
1413
|
+
&& typeof value.nonce === "string" && typeof value.ciphertext === "string" && typeof value.tag === "string"
|
|
1414
|
+
&& value.nonce.length === 16 && value.tag.length === 22 && value.ciphertext.length <= 350_000
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
/** @param {Record<string, unknown>} value @param {string[]} keys */
|
|
1418
|
+
function exactKeys(value, keys) {
|
|
1419
|
+
const own = Reflect.ownKeys(value)
|
|
1420
|
+
return own.length === keys.length && own.every((key) => typeof key === "string" && keys.includes(key))
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
/** @param {Record<string, unknown>} value @param {string[]} keys */
|
|
1424
|
+
function exactDataKeys(value, keys) {
|
|
1425
|
+
return exactKeys(value, keys) && keys.every((key) => {
|
|
1426
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key)
|
|
1427
|
+
return descriptor?.get === undefined && descriptor?.set === undefined && descriptor?.enumerable === true
|
|
1428
|
+
})
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
/** @param {Record<string, unknown>} value @param {(string | undefined)[]} keys */
|
|
1432
|
+
function exactOptionalDataKeys(value, keys) {
|
|
1433
|
+
const expected = keys.filter((key) => key !== undefined)
|
|
1434
|
+
return exactDataKeys(value, /** @type {string[]} */ (expected))
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
/** @param {string} value */
|
|
1438
|
+
function validContentType(value) {
|
|
1439
|
+
try {
|
|
1440
|
+
return validateContentType(value) === value
|
|
1441
|
+
} catch {
|
|
1442
|
+
return false
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
/** @param {EvidenceLimits} limits */
|
|
1447
|
+
function validateLimits(limits) {
|
|
1448
|
+
for (const [name, value] of Object.entries(limits)) if (!positive(value)) throw new Error(`Invalid evidence limit: ${name}`)
|
|
1449
|
+
return Object.freeze(limits)
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
/**
|
|
1453
|
+
* @typedef {{
|
|
1454
|
+
* version: 2, policyVersion: 1, framingVersion: 1, id: string, state: "pending" | "ready" | "expired" | "removed",
|
|
1455
|
+
* owner: {destinationId: string, runId: string}, contentType: string,
|
|
1456
|
+
* capabilityHash: string,
|
|
1457
|
+
* redactionPolicy: {algorithm: "aes-256-gcm", nonce: string, ciphertext: string, tag: string}, payloadCommitment: string,
|
|
1458
|
+
* bytes: number, storedBytes: number, reservedBytes: number, lines: number, events: number, createdAt: number, expiresAt: number
|
|
1459
|
+
* }} EvidenceMetadata
|
|
1460
|
+
* @typedef {{
|
|
1461
|
+
* handle: string,
|
|
1462
|
+
* bytes?: {offset: number, limit: number},
|
|
1463
|
+
* lines?: {start: number, limit: number},
|
|
1464
|
+
* query?: {literal: string, limit: number, contextBytes: number}
|
|
1465
|
+
* }} ValidatedRead
|
|
1466
|
+
* @typedef {{
|
|
1467
|
+
* maxEventBytes: number, maxArtifactBytes: number, maxArtifactEvents: number,
|
|
1468
|
+
* maxRunBytes: number, maxRunEvents: number, maxStoreBytes: number,
|
|
1469
|
+
* maxStoreEvents: number, maxArtifacts: number, maxReadBytes: number, maxReadLines: number,
|
|
1470
|
+
* retentionMs: number
|
|
1471
|
+
* }} EvidenceLimits
|
|
1472
|
+
*/
|