typeclaw 0.46.1 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +2 -2
- package/src/agent/tool-file-safety.ts +18 -9
- package/src/agent/tools/webfetch/strategies/jq.ts +1 -2
- package/src/agent/tools/webfetch/strategies/readability.ts +39 -16
- package/src/agent/tools/webfetch/strategies/selector.ts +4 -2
- package/src/agent/tools/webfetch/strategies/snapshot.ts +3 -2
- package/src/agent/tools/webfetch/tool.ts +3 -3
- package/src/bundled-plugins/backup/index.ts +21 -4
- package/src/bundled-plugins/backup/runner.ts +19 -5
- package/src/bundled-plugins/memory/dreaming.ts +2 -1
- package/src/bundled-plugins/memory/load-shards.ts +31 -3
- package/src/bundled-plugins/memory/memory-logger.ts +2 -1
- package/src/bundled-plugins/memory/memory-snapshot.ts +8 -0
- package/src/bundled-plugins/memory/vector/embedder.ts +28 -9
- package/src/bundled-plugins/memory/vector/hybrid.ts +10 -2
- package/src/bundled-plugins/memory/vector/store.ts +21 -9
- package/src/bundled-plugins/security/policies/git-exfil.ts +45 -27
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +8 -3
- package/src/bundled-plugins/security/policies/private-surface-read.ts +6 -0
- package/src/channels/adapters/telegram-bot-classify.ts +29 -0
- package/src/channels/router.ts +98 -0
- package/src/channels/types.ts +7 -0
- package/src/cli/command-meta.ts +2 -0
- package/src/cli/compose-stats.ts +103 -0
- package/src/cli/compose.ts +13 -0
- package/src/cli/index.ts +1 -0
- package/src/cli/inspect.ts +16 -0
- package/src/cli/stats.ts +134 -0
- package/src/compose/index.ts +1 -0
- package/src/compose/stats.ts +51 -0
- package/src/config/config.ts +14 -0
- package/src/config/index.ts +4 -0
- package/src/container/controller.ts +6 -0
- package/src/container/index.ts +1 -0
- package/src/container/log-archive.ts +448 -0
- package/src/container/shared.ts +49 -19
- package/src/container/start.ts +78 -63
- package/src/container/stats.ts +86 -0
- package/src/container/stop.ts +63 -36
- package/src/init/checkpoint.ts +4 -4
- package/src/init/dockerfile.ts +87 -88
- package/src/init/gitignore.ts +5 -6
- package/src/init/restart-deps-preflight.ts +2 -2
- package/src/init/run-bun-install.ts +16 -2
- package/src/init/validate-api-key.ts +1 -0
- package/src/inspect/item-list.ts +12 -6
- package/src/permissions/permissions.ts +93 -2
- package/src/role-claim/code.ts +9 -5
- package/src/run/index.ts +50 -36
- package/src/run/resource-report.ts +226 -0
- package/src/sandbox/canonical-secrets.ts +7 -0
- package/src/sandbox/index.ts +1 -0
- package/src/sandbox/symlinks.ts +1 -1
- package/src/secrets/export-claude-credentials-file.ts +13 -22
- package/src/secrets/export-codex-auth-file.ts +12 -28
- package/typeclaw.schema.json +14 -0
package/src/config/config.ts
CHANGED
|
@@ -287,6 +287,17 @@ export const gitSchema = z
|
|
|
287
287
|
|
|
288
288
|
export type GitConfig = z.infer<typeof gitSchema>
|
|
289
289
|
|
|
290
|
+
export const DEFAULT_LOG_RETENTION_DAYS = 14
|
|
291
|
+
export const MAX_LOG_RETENTION_DAYS = 3650
|
|
292
|
+
|
|
293
|
+
const logsObjectSchema = z.object({
|
|
294
|
+
retentionDays: z.number().int().min(1).max(MAX_LOG_RETENTION_DAYS).default(DEFAULT_LOG_RETENTION_DAYS),
|
|
295
|
+
})
|
|
296
|
+
|
|
297
|
+
export const logsSchema = logsObjectSchema.default(() => logsObjectSchema.parse({}))
|
|
298
|
+
|
|
299
|
+
export type LogsConfig = z.infer<typeof logsSchema>
|
|
300
|
+
|
|
290
301
|
const IPV4_CIDR_PATTERN = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})(?:\/(\d{1,2}))?$/
|
|
291
302
|
|
|
292
303
|
const ipv4CidrSchema = z.string().refine(
|
|
@@ -784,6 +795,7 @@ export const configSchema = z
|
|
|
784
795
|
sandbox: sandboxSchema,
|
|
785
796
|
docker: dockerSchema,
|
|
786
797
|
git: gitSchema,
|
|
798
|
+
logs: logsSchema,
|
|
787
799
|
roles: rolesConfigSchema.optional(),
|
|
788
800
|
tunnels: tunnelsArraySchema,
|
|
789
801
|
// When `true` (default), the system prompt tells the agent it runs on
|
|
@@ -1010,6 +1022,7 @@ export const FIELD_EFFECTS: Record<string, FieldEffect> = {
|
|
|
1010
1022
|
tunnels: 'restart-required',
|
|
1011
1023
|
'docker.file': 'restart-required',
|
|
1012
1024
|
'git.ignore': 'restart-required',
|
|
1025
|
+
logs: 'ignored',
|
|
1013
1026
|
// Split: `match` lists are reload-safe (typeclaw role claim, hand-edits
|
|
1014
1027
|
// adding/removing match rules apply without a container restart);
|
|
1015
1028
|
// `permissions` lists are restart-required (changing what a role can DO
|
|
@@ -1104,6 +1117,7 @@ export function extractPluginConfigs(raw: unknown): Record<string, unknown> {
|
|
|
1104
1117
|
'network',
|
|
1105
1118
|
'docker',
|
|
1106
1119
|
'git',
|
|
1120
|
+
'logs',
|
|
1107
1121
|
'roles',
|
|
1108
1122
|
'permissions',
|
|
1109
1123
|
'tunnels',
|
package/src/config/index.ts
CHANGED
|
@@ -4,6 +4,7 @@ export {
|
|
|
4
4
|
configSchema,
|
|
5
5
|
customModelMetaSchema,
|
|
6
6
|
customModelsSchema,
|
|
7
|
+
DEFAULT_LOG_RETENTION_DAYS,
|
|
7
8
|
DEFAULT_PLUGINS,
|
|
8
9
|
dockerSchema,
|
|
9
10
|
dockerfileSchema,
|
|
@@ -19,6 +20,8 @@ export {
|
|
|
19
20
|
loadConfigSync,
|
|
20
21
|
loadConfigSyncOrDefaults,
|
|
21
22
|
loadPluginConfigsSync,
|
|
23
|
+
logsSchema,
|
|
24
|
+
MAX_LOG_RETENTION_DAYS,
|
|
22
25
|
migrateLegacyConfigShape,
|
|
23
26
|
modelsSchema,
|
|
24
27
|
mountSchema,
|
|
@@ -39,6 +42,7 @@ export {
|
|
|
39
42
|
type DockerfileConfig,
|
|
40
43
|
type GitConfig,
|
|
41
44
|
type GitignoreConfig,
|
|
45
|
+
type LogsConfig,
|
|
42
46
|
type MigrationResult,
|
|
43
47
|
type MigrationStep,
|
|
44
48
|
type Models,
|
|
@@ -2,6 +2,7 @@ import { logs, type LogsOptions, type LogsResult } from './logs'
|
|
|
2
2
|
import { containerNameFromCwd, imageTagFromCwd } from './shared'
|
|
3
3
|
import { shell, type ShellResult } from './shell'
|
|
4
4
|
import { start, type StartOptions, type StartResult } from './start'
|
|
5
|
+
import { stats, type ContainerStats, type StatsOptions } from './stats'
|
|
5
6
|
import { status, type ContainerStatus, type StatusOptions } from './status'
|
|
6
7
|
import { stop, type StopOptions, type StopResult } from './stop'
|
|
7
8
|
|
|
@@ -17,6 +18,7 @@ export interface Controller {
|
|
|
17
18
|
start(options: StartOptions): Promise<StartResult>
|
|
18
19
|
stop(options: StopOptions): Promise<StopResult>
|
|
19
20
|
status(options: StatusOptions): Promise<ContainerStatus>
|
|
21
|
+
stats(options: StatsOptions): Promise<ContainerStats>
|
|
20
22
|
logs(options: LogsOptions): Promise<LogsResult>
|
|
21
23
|
shell(options: { cwd: string; shell?: string }): Promise<ShellResult>
|
|
22
24
|
}
|
|
@@ -30,6 +32,7 @@ export function createLocalDockerController(): Controller {
|
|
|
30
32
|
start: (options) => start(options),
|
|
31
33
|
stop: (options) => stop(options),
|
|
32
34
|
status: (options) => status(options),
|
|
35
|
+
stats: (options) => stats(options),
|
|
33
36
|
logs: (options) => logs(options),
|
|
34
37
|
shell: (options) => shell(options),
|
|
35
38
|
}
|
|
@@ -53,6 +56,9 @@ export function createNoopController(): Controller {
|
|
|
53
56
|
async status({ cwd }) {
|
|
54
57
|
return { kind: 'missing', containerName: containerNameFromCwd(cwd), imageTag: imageTagFromCwd(cwd) }
|
|
55
58
|
},
|
|
59
|
+
async stats({ cwd }) {
|
|
60
|
+
return { kind: 'missing', containerName: containerNameFromCwd(cwd), imageTag: imageTagFromCwd(cwd) }
|
|
61
|
+
},
|
|
56
62
|
async logs({ cwd }) {
|
|
57
63
|
return { ok: false, reason: unsupported('logs', cwd) }
|
|
58
64
|
},
|
package/src/container/index.ts
CHANGED
|
@@ -24,6 +24,7 @@ export {
|
|
|
24
24
|
type RequireContainerRunningResult,
|
|
25
25
|
} from './require-running'
|
|
26
26
|
export { planShell, shell, type ShellPlan, type ShellResult } from './shell'
|
|
27
|
+
export { parseStatsLine, stats, type ContainerStats, type StatsOptions, type StatsSnapshot } from './stats'
|
|
27
28
|
export { status, type ContainerStatus, type StatusOptions } from './status'
|
|
28
29
|
export {
|
|
29
30
|
checkDockerAvailable,
|
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto'
|
|
2
|
+
import { chmod, lstat, mkdir, open, opendir, rename, rm, type FileHandle } from 'node:fs/promises'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
|
|
5
|
+
import { DEFAULT_LOG_RETENTION_DAYS, MAX_LOG_RETENTION_DAYS } from '@/config'
|
|
6
|
+
|
|
7
|
+
import { resolveDockerBinary, sanitizeDockerStderr } from './shared'
|
|
8
|
+
|
|
9
|
+
export type DockerLogArchiveResult = { ok: true; status: 'archived'; path: string } | { ok: false; reason: string }
|
|
10
|
+
|
|
11
|
+
export type DockerLogArchiver = (input: {
|
|
12
|
+
agentDir: string
|
|
13
|
+
containerId: string
|
|
14
|
+
retentionDays?: number
|
|
15
|
+
}) => Promise<DockerLogArchiveResult>
|
|
16
|
+
|
|
17
|
+
export type StreamCaptureInput = {
|
|
18
|
+
args: string[]
|
|
19
|
+
binary: string
|
|
20
|
+
cwd: string
|
|
21
|
+
finalSettlementMs?: number
|
|
22
|
+
killFallbackMs?: number
|
|
23
|
+
maxBytes: number
|
|
24
|
+
output: FileHandle
|
|
25
|
+
spawn?: (input: { args: string[]; binary: string; cwd: string }) => StreamingProcess
|
|
26
|
+
timeoutMs: number
|
|
27
|
+
writeChunk?: (output: FileHandle, chunk: Uint8Array, offset: number) => Promise<number>
|
|
28
|
+
writeSettlementMs?: number
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export type StreamingProcess = {
|
|
32
|
+
exited: Promise<number>
|
|
33
|
+
kill(signal: NodeJS.Signals): void
|
|
34
|
+
stderr: ReadableStream<Uint8Array>
|
|
35
|
+
stdout: ReadableStream<Uint8Array>
|
|
36
|
+
unref(): void
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type StreamCaptureResult = {
|
|
40
|
+
exitCode: number
|
|
41
|
+
overflowed: boolean
|
|
42
|
+
stderrExcerpt: string
|
|
43
|
+
terminationBoundExceeded?: boolean
|
|
44
|
+
timedOut: boolean
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type DockerLogArchiveRuntime = {
|
|
48
|
+
capture?: (input: StreamCaptureInput) => Promise<StreamCaptureResult>
|
|
49
|
+
nonce?: () => string
|
|
50
|
+
now?: () => Date
|
|
51
|
+
resolveDockerBinary?: () => string | null
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export const DOCKER_LOG_ARCHIVE_TIMEOUT_MS = 60_000
|
|
55
|
+
export const MAX_DOCKER_LOG_ARCHIVE_BYTES = 64 * 1024 * 1024
|
|
56
|
+
export const MAX_DOCKER_LOG_ARCHIVE_SNAPSHOTS = 512
|
|
57
|
+
export const MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES = 512 * 1024 * 1024
|
|
58
|
+
|
|
59
|
+
const MAX_ARCHIVE_DIRECTORY_ENTRIES = 2048
|
|
60
|
+
const STDERR_EXCERPT_BYTES = 8 * 1024
|
|
61
|
+
const KILL_FALLBACK_MS = 1_000
|
|
62
|
+
const FINAL_SETTLEMENT_MS = 1_000
|
|
63
|
+
const CLEANUP_SETTLEMENT_MS = 1_000
|
|
64
|
+
const WRITE_SETTLEMENT_MS = 60_000
|
|
65
|
+
const STALE_PARTIAL_GRACE_MS = 5 * 60 * 1_000
|
|
66
|
+
const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1_000
|
|
67
|
+
const MAX_FILENAME_ATTEMPTS = 16
|
|
68
|
+
const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/
|
|
69
|
+
const NONCE_PATTERN = /^[a-f0-9]{32}$/
|
|
70
|
+
const ARCHIVE_FILENAME_PATTERN = /^[a-f0-9]{64}-(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z-[a-f0-9]{32}\.log$/
|
|
71
|
+
const PARTIAL_FILENAME_PATTERN = /^\.[a-f0-9]{64}-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z-[a-f0-9]{32}\.partial$/
|
|
72
|
+
|
|
73
|
+
export async function archiveContainerLogs(
|
|
74
|
+
{ agentDir, containerId, retentionDays = DEFAULT_LOG_RETENTION_DAYS }: Parameters<DockerLogArchiver>[0],
|
|
75
|
+
runtime: DockerLogArchiveRuntime = {},
|
|
76
|
+
): Promise<DockerLogArchiveResult> {
|
|
77
|
+
if (!CONTAINER_ID_PATTERN.test(containerId)) {
|
|
78
|
+
return { ok: false, reason: `Invalid full Docker container ID: ${containerId}` }
|
|
79
|
+
}
|
|
80
|
+
if (!Number.isInteger(retentionDays) || retentionDays < 1 || retentionDays > MAX_LOG_RETENTION_DAYS) {
|
|
81
|
+
return { ok: false, reason: `Invalid Docker log archive retention period: ${retentionDays}` }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const logsDir = join(agentDir, '.typeclaw', 'logs')
|
|
85
|
+
let temporaryPath: string | null = null
|
|
86
|
+
let output: FileHandle | null = null
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
await ensurePrivateDirectory(join(agentDir, '.typeclaw'))
|
|
90
|
+
await ensurePrivateDirectory(logsDir)
|
|
91
|
+
const now = runtime.now?.() ?? new Date()
|
|
92
|
+
const ageCutoff = now.getTime() - retentionDays * MILLISECONDS_PER_DAY
|
|
93
|
+
await pruneByAgeAndRemoveStalePartials(logsDir, ageCutoff, now.getTime())
|
|
94
|
+
|
|
95
|
+
const allocation = await allocateArchive(logsDir, containerId, archiveTimestampText(now), runtime)
|
|
96
|
+
temporaryPath = allocation.temporaryPath
|
|
97
|
+
output = allocation.output
|
|
98
|
+
|
|
99
|
+
const resolver = runtime.resolveDockerBinary ?? resolveDockerBinary
|
|
100
|
+
const binary = resolver()
|
|
101
|
+
if (binary === null) throw new Error('Docker binary could not be resolved for log archival')
|
|
102
|
+
|
|
103
|
+
const capture = runtime.capture ?? streamProcessOutput
|
|
104
|
+
const captured = await capture({
|
|
105
|
+
args: ['logs', '--timestamps', containerId],
|
|
106
|
+
binary,
|
|
107
|
+
cwd: agentDir,
|
|
108
|
+
maxBytes: MAX_DOCKER_LOG_ARCHIVE_BYTES,
|
|
109
|
+
output,
|
|
110
|
+
timeoutMs: DOCKER_LOG_ARCHIVE_TIMEOUT_MS,
|
|
111
|
+
})
|
|
112
|
+
if (captured.terminationBoundExceeded) throw new Error('docker logs did not settle after SIGKILL')
|
|
113
|
+
if (captured.timedOut) throw new Error(`docker logs timed out after ${DOCKER_LOG_ARCHIVE_TIMEOUT_MS}ms`)
|
|
114
|
+
if (captured.overflowed) {
|
|
115
|
+
throw new Error(`docker logs exceeded the ${MAX_DOCKER_LOG_ARCHIVE_BYTES}-byte archive limit`)
|
|
116
|
+
}
|
|
117
|
+
if (captured.exitCode !== 0) {
|
|
118
|
+
const detail = sanitizeDockerStderr(captured.stderrExcerpt)
|
|
119
|
+
throw new Error(`docker logs exited with code ${captured.exitCode}${detail ? `: ${detail}` : ''}`)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
await output.chmod(0o600)
|
|
123
|
+
const incomingSize = (await output.stat()).size
|
|
124
|
+
await pruneToCapacity(logsDir, incomingSize)
|
|
125
|
+
await output.close()
|
|
126
|
+
output = null
|
|
127
|
+
|
|
128
|
+
await rename(temporaryPath, allocation.finalPath)
|
|
129
|
+
temporaryPath = null
|
|
130
|
+
await chmod(allocation.finalPath, 0o600)
|
|
131
|
+
return { ok: true, status: 'archived', path: allocation.finalPath }
|
|
132
|
+
} catch (error) {
|
|
133
|
+
const cleanupErrors = await cleanupTemp(output, temporaryPath)
|
|
134
|
+
const suffix = cleanupErrors.length === 0 ? '' : ` (${cleanupErrors.join('; ')})`
|
|
135
|
+
return { ok: false, reason: `${errorMessage(error)}${suffix}` }
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export async function streamProcessOutput(input: StreamCaptureInput): Promise<StreamCaptureResult> {
|
|
140
|
+
const { args, binary, cwd, maxBytes, output, timeoutMs } = input
|
|
141
|
+
const killFallbackMs = input.killFallbackMs ?? KILL_FALLBACK_MS
|
|
142
|
+
const finalSettlementMs = input.finalSettlementMs ?? FINAL_SETTLEMENT_MS
|
|
143
|
+
const writeChunk = input.writeChunk ?? writeFileChunk
|
|
144
|
+
const writeSettlementMs = input.writeSettlementMs ?? WRITE_SETTLEMENT_MS
|
|
145
|
+
const proc = input.spawn?.({ args, binary, cwd }) ?? spawnStreamingProcess(input)
|
|
146
|
+
const abortPumps = new AbortController()
|
|
147
|
+
const stderrChunks: Uint8Array[] = []
|
|
148
|
+
let stderrBytes = 0
|
|
149
|
+
let acceptedBytes = 0
|
|
150
|
+
let overflowed = false
|
|
151
|
+
let timedOut = false
|
|
152
|
+
let terminating = false
|
|
153
|
+
let processSettled = false
|
|
154
|
+
let terminationBoundExceeded = false
|
|
155
|
+
let pumpError: unknown
|
|
156
|
+
let writeError: unknown
|
|
157
|
+
let writeQueue = Promise.resolve()
|
|
158
|
+
let sigkillTimer: ReturnType<typeof setTimeout> | undefined
|
|
159
|
+
let finalTimer: ReturnType<typeof setTimeout> | undefined
|
|
160
|
+
let forceSettle: ((exitCode: number) => void) | undefined
|
|
161
|
+
const forcedExit = new Promise<number>((resolve) => {
|
|
162
|
+
forceSettle = resolve
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
const terminate = (): void => {
|
|
166
|
+
if (terminating || processSettled) return
|
|
167
|
+
terminating = true
|
|
168
|
+
proc.kill('SIGTERM')
|
|
169
|
+
sigkillTimer = setTimeout(() => {
|
|
170
|
+
proc.kill('SIGKILL')
|
|
171
|
+
finalTimer = setTimeout(() => {
|
|
172
|
+
terminationBoundExceeded = true
|
|
173
|
+
abortPumps.abort()
|
|
174
|
+
proc.unref()
|
|
175
|
+
forceSettle?.(-1)
|
|
176
|
+
}, finalSettlementMs)
|
|
177
|
+
}, killFallbackMs)
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const enqueue = (chunk: Uint8Array, stderr: boolean): void => {
|
|
181
|
+
if (stderr && stderrBytes < STDERR_EXCERPT_BYTES) {
|
|
182
|
+
const excerpt = chunk.subarray(0, STDERR_EXCERPT_BYTES - stderrBytes)
|
|
183
|
+
stderrChunks.push(excerpt)
|
|
184
|
+
stderrBytes += excerpt.byteLength
|
|
185
|
+
}
|
|
186
|
+
if (terminating) return
|
|
187
|
+
if (acceptedBytes + chunk.byteLength > maxBytes) {
|
|
188
|
+
overflowed = true
|
|
189
|
+
terminate()
|
|
190
|
+
return
|
|
191
|
+
}
|
|
192
|
+
acceptedBytes += chunk.byteLength
|
|
193
|
+
writeQueue = writeQueue
|
|
194
|
+
.then(async () => {
|
|
195
|
+
await writeAll(output, chunk, writeChunk)
|
|
196
|
+
})
|
|
197
|
+
.catch((error: unknown) => {
|
|
198
|
+
writeError = error
|
|
199
|
+
terminate()
|
|
200
|
+
})
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const timeout = setTimeout(() => {
|
|
204
|
+
timedOut = true
|
|
205
|
+
terminate()
|
|
206
|
+
}, timeoutMs)
|
|
207
|
+
const stdoutPump = pump(proc.stdout, (chunk) => enqueue(chunk, false), abortPumps.signal).catch((error: unknown) => {
|
|
208
|
+
pumpError = error
|
|
209
|
+
terminate()
|
|
210
|
+
})
|
|
211
|
+
const stderrPump = pump(proc.stderr, (chunk) => enqueue(chunk, true), abortPumps.signal).catch((error: unknown) => {
|
|
212
|
+
pumpError = error
|
|
213
|
+
terminate()
|
|
214
|
+
})
|
|
215
|
+
const completion = Promise.all([proc.exited, stdoutPump, stderrPump]).then(([exitCode]) => {
|
|
216
|
+
processSettled = true
|
|
217
|
+
return exitCode
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
const exitCode = await Promise.race([completion, forcedExit])
|
|
221
|
+
clearTimeout(timeout)
|
|
222
|
+
if (sigkillTimer !== undefined) clearTimeout(sigkillTimer)
|
|
223
|
+
if (finalTimer !== undefined) clearTimeout(finalTimer)
|
|
224
|
+
if (terminationBoundExceeded) await Promise.allSettled([stdoutPump, stderrPump])
|
|
225
|
+
await withDeadline(writeQueue, writeSettlementMs, 'Docker log archive write')
|
|
226
|
+
if (writeError !== undefined) throw writeError
|
|
227
|
+
if (pumpError !== undefined) throw pumpError
|
|
228
|
+
return {
|
|
229
|
+
exitCode,
|
|
230
|
+
overflowed,
|
|
231
|
+
stderrExcerpt: Buffer.concat(stderrChunks).toString('utf8'),
|
|
232
|
+
terminationBoundExceeded,
|
|
233
|
+
timedOut,
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async function writeAll(
|
|
238
|
+
output: FileHandle,
|
|
239
|
+
chunk: Uint8Array,
|
|
240
|
+
writeChunk: NonNullable<StreamCaptureInput['writeChunk']>,
|
|
241
|
+
): Promise<void> {
|
|
242
|
+
let offset = 0
|
|
243
|
+
while (offset < chunk.byteLength) {
|
|
244
|
+
const bytesWritten = await writeChunk(output, chunk, offset)
|
|
245
|
+
if (bytesWritten < 1) throw new Error('Docker log archive write made no progress')
|
|
246
|
+
offset += bytesWritten
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async function writeFileChunk(output: FileHandle, chunk: Uint8Array, offset: number): Promise<number> {
|
|
251
|
+
return (await output.write(chunk, offset, chunk.byteLength - offset)).bytesWritten
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function spawnStreamingProcess({ args, binary, cwd }: StreamCaptureInput): StreamingProcess {
|
|
255
|
+
return Bun.spawn({ cmd: [binary, ...args], cwd, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' })
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function pump(
|
|
259
|
+
stream: ReadableStream<Uint8Array>,
|
|
260
|
+
onChunk: (chunk: Uint8Array) => void,
|
|
261
|
+
signal: AbortSignal,
|
|
262
|
+
): Promise<void> {
|
|
263
|
+
const reader = stream.getReader()
|
|
264
|
+
const cancel = (): void => {
|
|
265
|
+
void reader.cancel()
|
|
266
|
+
}
|
|
267
|
+
signal.addEventListener('abort', cancel, { once: true })
|
|
268
|
+
try {
|
|
269
|
+
for (;;) {
|
|
270
|
+
const { done, value } = await reader.read()
|
|
271
|
+
if (done) return
|
|
272
|
+
onChunk(value)
|
|
273
|
+
}
|
|
274
|
+
} catch (error) {
|
|
275
|
+
if (!signal.aborted) throw error
|
|
276
|
+
} finally {
|
|
277
|
+
signal.removeEventListener('abort', cancel)
|
|
278
|
+
reader.releaseLock()
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
async function ensurePrivateDirectory(path: string): Promise<void> {
|
|
283
|
+
try {
|
|
284
|
+
await mkdir(path, { mode: 0o700 })
|
|
285
|
+
} catch (error) {
|
|
286
|
+
if (errorCode(error) !== 'EEXIST') throw error
|
|
287
|
+
}
|
|
288
|
+
const stats = await lstat(path)
|
|
289
|
+
if (stats.isSymbolicLink() || !stats.isDirectory())
|
|
290
|
+
throw new Error(`Docker log archive path is not a real directory: ${path}`)
|
|
291
|
+
await chmod(path, 0o700)
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
async function pruneByAgeAndRemoveStalePartials(logsDir: string, ageCutoff: number, now: number): Promise<void> {
|
|
295
|
+
const stalePartialCutoff = now - DOCKER_LOG_ARCHIVE_TIMEOUT_MS - STALE_PARTIAL_GRACE_MS
|
|
296
|
+
for (const entry of await collectDirectoryEntries(logsDir)) {
|
|
297
|
+
if (!entry.isFile()) continue
|
|
298
|
+
const path = join(logsDir, entry.name)
|
|
299
|
+
const timestamp = archiveTimestamp(entry.name)
|
|
300
|
+
if (timestamp !== null && timestamp < ageCutoff) {
|
|
301
|
+
await removeRaceSafe(path)
|
|
302
|
+
} else if (PARTIAL_FILENAME_PATTERN.test(entry.name)) {
|
|
303
|
+
const stats = await lstatIfPresent(path)
|
|
304
|
+
if (stats !== null && stats.mtimeMs < stalePartialCutoff) await removeRaceSafe(path)
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
async function pruneToCapacity(logsDir: string, incomingSize: number): Promise<void> {
|
|
310
|
+
if (incomingSize > MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES) {
|
|
311
|
+
throw new Error('Docker log archive exceeds total archive capacity')
|
|
312
|
+
}
|
|
313
|
+
const snapshots: Array<{ name: string; path: string; size: number; timestamp: number }> = []
|
|
314
|
+
for (const entry of await collectDirectoryEntries(logsDir)) {
|
|
315
|
+
if (!entry.isFile()) continue
|
|
316
|
+
const timestamp = archiveTimestamp(entry.name)
|
|
317
|
+
if (timestamp === null) continue
|
|
318
|
+
const path = join(logsDir, entry.name)
|
|
319
|
+
const stats = await lstatIfPresent(path)
|
|
320
|
+
if (stats !== null) snapshots.push({ name: entry.name, path, size: stats.size, timestamp })
|
|
321
|
+
}
|
|
322
|
+
snapshots.sort((left, right) => left.timestamp - right.timestamp || left.name.localeCompare(right.name))
|
|
323
|
+
let totalBytes = snapshots.reduce((total, snapshot) => total + snapshot.size, 0)
|
|
324
|
+
let count = snapshots.length
|
|
325
|
+
for (const snapshot of snapshots) {
|
|
326
|
+
if (
|
|
327
|
+
count + 1 <= MAX_DOCKER_LOG_ARCHIVE_SNAPSHOTS &&
|
|
328
|
+
totalBytes + incomingSize <= MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES
|
|
329
|
+
)
|
|
330
|
+
break
|
|
331
|
+
await removeRaceSafe(snapshot.path)
|
|
332
|
+
count -= 1
|
|
333
|
+
totalBytes -= snapshot.size
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
async function collectDirectoryEntries(logsDir: string) {
|
|
338
|
+
const entries = []
|
|
339
|
+
const directory = await opendir(logsDir)
|
|
340
|
+
for await (const entry of directory) {
|
|
341
|
+
entries.push(entry)
|
|
342
|
+
if (entries.length > MAX_ARCHIVE_DIRECTORY_ENTRIES)
|
|
343
|
+
throw new Error(`Docker log archive directory exceeds ${MAX_ARCHIVE_DIRECTORY_ENTRIES} entries`)
|
|
344
|
+
}
|
|
345
|
+
return entries
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
async function allocateArchive(
|
|
349
|
+
logsDir: string,
|
|
350
|
+
containerId: string,
|
|
351
|
+
timestamp: string,
|
|
352
|
+
runtime: DockerLogArchiveRuntime,
|
|
353
|
+
): Promise<{ finalPath: string; output: FileHandle; temporaryPath: string }> {
|
|
354
|
+
for (let attempt = 0; attempt < MAX_FILENAME_ATTEMPTS; attempt++) {
|
|
355
|
+
const nonce = runtime.nonce?.() ?? randomBytes(16).toString('hex')
|
|
356
|
+
if (!NONCE_PATTERN.test(nonce)) throw new Error(`Invalid Docker log archive nonce: ${nonce}`)
|
|
357
|
+
const finalPath = join(logsDir, `${containerId}-${timestamp}-${nonce}.log`)
|
|
358
|
+
const temporaryPath = join(logsDir, `.${containerId}-${timestamp}-${nonce}.partial`)
|
|
359
|
+
let output: FileHandle
|
|
360
|
+
try {
|
|
361
|
+
output = await open(temporaryPath, 'wx', 0o600)
|
|
362
|
+
} catch (error) {
|
|
363
|
+
if (errorCode(error) === 'EEXIST') continue
|
|
364
|
+
throw error
|
|
365
|
+
}
|
|
366
|
+
try {
|
|
367
|
+
await lstat(finalPath)
|
|
368
|
+
} catch (error) {
|
|
369
|
+
if (errorCode(error) === 'ENOENT') return { finalPath, output, temporaryPath }
|
|
370
|
+
const cleanupErrors = await cleanupTemp(output, temporaryPath)
|
|
371
|
+
throw new Error([errorMessage(error), ...cleanupErrors].join('; '))
|
|
372
|
+
}
|
|
373
|
+
const cleanupErrors = await cleanupTemp(output, temporaryPath)
|
|
374
|
+
if (cleanupErrors.length > 0)
|
|
375
|
+
throw new Error(['Archive filename collision cleanup failed', ...cleanupErrors].join('; '))
|
|
376
|
+
}
|
|
377
|
+
throw new Error('Could not allocate a unique Docker log archive filename')
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
async function cleanupTemp(output: FileHandle | null, path: string | null): Promise<string[]> {
|
|
381
|
+
const errors: string[] = []
|
|
382
|
+
if (output !== null) {
|
|
383
|
+
try {
|
|
384
|
+
await withDeadline(output.close(), CLEANUP_SETTLEMENT_MS, 'Docker log archive temp close')
|
|
385
|
+
} catch (error) {
|
|
386
|
+
errors.push(`temp close failed: ${errorMessage(error)}`)
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
if (path !== null) {
|
|
390
|
+
try {
|
|
391
|
+
await withDeadline(rm(path, { force: true }), CLEANUP_SETTLEMENT_MS, 'Docker log archive temp cleanup')
|
|
392
|
+
} catch (error) {
|
|
393
|
+
errors.push(`temp cleanup failed: ${errorMessage(error)}`)
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
return errors
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
async function removeRaceSafe(path: string): Promise<void> {
|
|
400
|
+
try {
|
|
401
|
+
await rm(path)
|
|
402
|
+
} catch (error) {
|
|
403
|
+
if (errorCode(error) !== 'ENOENT') throw error
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
async function lstatIfPresent(path: string) {
|
|
408
|
+
try {
|
|
409
|
+
return await lstat(path)
|
|
410
|
+
} catch (error) {
|
|
411
|
+
if (errorCode(error) === 'ENOENT') return null
|
|
412
|
+
throw error
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
async function withDeadline<T>(promise: Promise<T>, timeoutMs: number, operation: string): Promise<T> {
|
|
417
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
418
|
+
try {
|
|
419
|
+
return await Promise.race([
|
|
420
|
+
promise,
|
|
421
|
+
new Promise<never>((_, reject) => {
|
|
422
|
+
timer = setTimeout(() => reject(new Error(`${operation} timed out after ${timeoutMs}ms`)), timeoutMs)
|
|
423
|
+
}),
|
|
424
|
+
])
|
|
425
|
+
} finally {
|
|
426
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
function archiveTimestamp(filename: string): number | null {
|
|
431
|
+
const match = ARCHIVE_FILENAME_PATTERN.exec(filename)
|
|
432
|
+
if (match === null) return null
|
|
433
|
+
const iso = `${match[1]}T${match[2]}:${match[3]}:${match[4]}.${match[5]}Z`
|
|
434
|
+
const timestamp = Date.parse(iso)
|
|
435
|
+
return Number.isFinite(timestamp) && new Date(timestamp).toISOString() === iso ? timestamp : null
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function archiveTimestampText(date: Date): string {
|
|
439
|
+
return date.toISOString().replace(/[:.]/g, '-')
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function errorCode(error: unknown): string | undefined {
|
|
443
|
+
return (error as NodeJS.ErrnoException).code
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function errorMessage(error: unknown): string {
|
|
447
|
+
return error instanceof Error ? error.message : String(error)
|
|
448
|
+
}
|
package/src/container/shared.ts
CHANGED
|
@@ -24,6 +24,16 @@ export type DockerExecOptions = {
|
|
|
24
24
|
|
|
25
25
|
export type DockerExec = (args: string[], options?: DockerExecOptions) => Promise<DockerExecResult>
|
|
26
26
|
|
|
27
|
+
export type ParsedContainerInspect = { ok: true; containerId: string; running: boolean } | { ok: false; reason: string }
|
|
28
|
+
|
|
29
|
+
export function parseContainerInspectOutput(output: string): ParsedContainerInspect {
|
|
30
|
+
const match = /^([0-9a-f]{64})\|(true|false)$/.exec(output.trim())
|
|
31
|
+
if (match === null) {
|
|
32
|
+
return { ok: false, reason: 'expected <64 lowercase hex container ID>|<true|false>' }
|
|
33
|
+
}
|
|
34
|
+
return { ok: true, containerId: match[1]!, running: match[2] === 'true' }
|
|
35
|
+
}
|
|
36
|
+
|
|
27
37
|
export const defaultDockerExec: DockerExec = async (args, options) => {
|
|
28
38
|
const bun = getBun()
|
|
29
39
|
if (!bun) return { exitCode: -1, stdout: '', stderr: 'bun runtime not available' }
|
|
@@ -92,7 +102,7 @@ export const defaultDockerExec: DockerExec = async (args, options) => {
|
|
|
92
102
|
// needs both: the user watches the build, and start() inspects the captured
|
|
93
103
|
// stderr to detect a missing credential helper and retry under a sanitized
|
|
94
104
|
// DOCKER_CONFIG.
|
|
95
|
-
async function spawnInheritTeeStderr(
|
|
105
|
+
export async function spawnInheritTeeStderr(
|
|
96
106
|
bun: NonNullable<ReturnType<typeof getBun>>,
|
|
97
107
|
cmd: string[],
|
|
98
108
|
cwd: string | undefined,
|
|
@@ -102,11 +112,24 @@ async function spawnInheritTeeStderr(
|
|
|
102
112
|
const proc = bun.spawn({ cmd, cwd, stdout: 'inherit', stderr: 'pipe', signal, env })
|
|
103
113
|
const chunks: Uint8Array[] = []
|
|
104
114
|
const reader = proc.stderr.getReader()
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
115
|
+
try {
|
|
116
|
+
for (;;) {
|
|
117
|
+
const { done, value } = await reader.read()
|
|
118
|
+
if (done) break
|
|
119
|
+
process.stderr.write(value)
|
|
120
|
+
chunks.push(value)
|
|
121
|
+
}
|
|
122
|
+
} catch (err) {
|
|
123
|
+
// A read/write throw skips the normal drain: cancel (not just releaseLock,
|
|
124
|
+
// which leaves the pipe live per the Streams contract), kill the child, and
|
|
125
|
+
// await its exit so neither the stderr pipe nor the docker subprocess is
|
|
126
|
+
// orphaned. releaseLock is handled in `finally` for both paths.
|
|
127
|
+
await reader.cancel().catch(() => {})
|
|
128
|
+
proc.kill()
|
|
129
|
+
await proc.exited.catch(() => {})
|
|
130
|
+
throw err
|
|
131
|
+
} finally {
|
|
132
|
+
reader.releaseLock()
|
|
110
133
|
}
|
|
111
134
|
const exitCode = await proc.exited
|
|
112
135
|
return { exitCode, stdout: '', stderr: Buffer.concat(chunks).toString('utf8') }
|
|
@@ -375,9 +398,8 @@ export function classifyRmStderr(stderr: string): BenignRmKind {
|
|
|
375
398
|
// sleep-only retries cannot make it go away.
|
|
376
399
|
//
|
|
377
400
|
// The fix is destructive: when this error fires for a non-running same-name
|
|
378
|
-
// container,
|
|
379
|
-
//
|
|
380
|
-
// concurrent legitimate start of the same name is never killed).
|
|
401
|
+
// container, remove it before retrying. cleanupRunCorpse uses non-force rm so
|
|
402
|
+
// Docker still refuses if that same ID starts after the inspect probe.
|
|
381
403
|
//
|
|
382
404
|
// Matches case-insensitively on the canonical phrasing across Docker
|
|
383
405
|
// Engine, Docker Desktop, and OrbStack. The (or rename) clause is the
|
|
@@ -390,7 +412,7 @@ export function isContainerNameConflict(stderr: string): boolean {
|
|
|
390
412
|
// Result of probing whether a previous `docker run --name <X>` left a corpse
|
|
391
413
|
// blocking the next run:
|
|
392
414
|
// - 'gone' — no container with that name. Safe to `docker run --name`.
|
|
393
|
-
// - 'removed' — corpse existed and was
|
|
415
|
+
// - 'removed' — corpse existed and was removed (and waitForRemoval
|
|
394
416
|
// confirmed it disappeared). Safe to `docker run --name`.
|
|
395
417
|
// - 'running' — a container with that name is currently RUNNING. We did
|
|
396
418
|
// NOT remove it. Caller must NOT proceed with `docker run
|
|
@@ -399,11 +421,13 @@ export function isContainerNameConflict(stderr: string): boolean {
|
|
|
399
421
|
// - 'stuck' — corpse existed but did not disappear within waitForRemoval
|
|
400
422
|
// budget. Caller should surface a clear error rather than
|
|
401
423
|
// loop forever.
|
|
402
|
-
export type
|
|
424
|
+
export type BeforeContainerRemove = (containerId: string) => Promise<{ ok: true } | { ok: false; reason: string }>
|
|
425
|
+
|
|
426
|
+
export type CorpseCleanupOutcome = 'gone' | 'removed' | 'running' | 'stuck' | { archiveFailed: string }
|
|
403
427
|
|
|
404
|
-
// Inspects the named container; if a non-running corpse is holding the
|
|
405
|
-
//
|
|
406
|
-
//
|
|
428
|
+
// Inspects the named container; if a non-running corpse is holding the name,
|
|
429
|
+
// runs the caller's required archival hook, removes it without force, and waits for the
|
|
430
|
+
// removal to drain. An archival failure leaves the corpse untouched.
|
|
407
431
|
// Explicitly refuses to touch a RUNNING container so that a concurrent
|
|
408
432
|
// legitimate start of the same name (or a foreign-but-named container the
|
|
409
433
|
// user wants kept alive) is never killed by this cleanup path. Errors from
|
|
@@ -420,13 +444,19 @@ export type CorpseCleanupOutcome = 'gone' | 'removed' | 'running' | 'stuck'
|
|
|
420
444
|
// container" which classifyRmStderr folds into 'gone'. waitForRemoval
|
|
421
445
|
// is still keyed on name because that's what the caller's next
|
|
422
446
|
// `docker run --name <name>` will actually collide on.
|
|
423
|
-
export async function cleanupRunCorpse(
|
|
447
|
+
export async function cleanupRunCorpse(
|
|
448
|
+
exec: DockerExec,
|
|
449
|
+
name: string,
|
|
450
|
+
beforeRemove: BeforeContainerRemove,
|
|
451
|
+
): Promise<CorpseCleanupOutcome> {
|
|
424
452
|
const probe = await exec(['inspect', '--format', '{{.Id}}|{{.State.Running}}', name])
|
|
425
453
|
if (probe.exitCode !== 0) return 'gone'
|
|
426
|
-
const
|
|
427
|
-
if (
|
|
428
|
-
if (
|
|
429
|
-
const
|
|
454
|
+
const parsed = parseContainerInspectOutput(probe.stdout)
|
|
455
|
+
if (!parsed.ok) return 'stuck'
|
|
456
|
+
if (parsed.running) return 'running'
|
|
457
|
+
const prepared = await beforeRemove(parsed.containerId)
|
|
458
|
+
if (!prepared.ok) return { archiveFailed: prepared.reason }
|
|
459
|
+
const rm = await exec(['rm', parsed.containerId])
|
|
430
460
|
if (rm.exitCode !== 0) {
|
|
431
461
|
const kind = classifyRmStderr(rm.stderr)
|
|
432
462
|
if (kind === 'gone') return 'gone'
|