typeclaw 0.48.10 → 0.48.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/scripts/apparmor/typeclaw-bwrap +43 -0
- package/scripts/verify-apparmor-signals.sh +161 -0
- package/scripts/verify-mask-fd-sandbox.sh +86 -16
- package/src/agent/index.ts +11 -1
- package/src/agent/live-subagents.ts +98 -0
- package/src/agent/session-origin.ts +5 -1
- package/src/agent/tools/channel-reply.ts +30 -2
- package/src/agent/tools/channel-send.ts +19 -2
- package/src/agent/tools/spawn-subagent.ts +37 -1
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +7 -1
- package/src/bundled-plugins/github-cli-auth/index.ts +2 -1
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +9 -1
- package/src/bundled-plugins/memory/dreaming.ts +21 -6
- package/src/channels/adapters/github/inbound.ts +195 -25
- package/src/channels/adapters/github/index.ts +16 -18
- package/src/channels/adapters/github/reconcile-cooldown-store.ts +15 -1
- package/src/channels/adapters/github/recover-failed-deliveries.ts +7 -2
- package/src/channels/adapters/github/review-submitter.ts +28 -0
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +86 -16
- package/src/channels/adapters/kakaotalk.ts +7 -8
- package/src/channels/adapters/teams-classify.ts +21 -0
- package/src/channels/adapters/teams-edit.ts +2 -1
- package/src/channels/adapters/teams-format.ts +16 -0
- package/src/channels/adapters/teams.ts +12 -5
- package/src/channels/github-repo.ts +11 -0
- package/src/channels/github-review-verdict-coordinator.ts +179 -29
- package/src/channels/github-verdict-activity.ts +10 -6
- package/src/channels/manager.ts +9 -0
- package/src/channels/persistence.ts +28 -2
- package/src/channels/router.ts +284 -61
- package/src/channels/schema.ts +18 -0
- package/src/channels/types.ts +1 -0
- package/src/cli/compose.ts +3 -3
- package/src/cli/docker-preflight.ts +8 -4
- package/src/cli/restart.ts +89 -23
- package/src/cli/start.ts +68 -16
- package/src/cli/status.ts +38 -6
- package/src/cli/stop.ts +5 -2
- package/src/compose/stop.ts +5 -4
- package/src/config/config.ts +9 -1
- package/src/container/build-cache.ts +807 -0
- package/src/container/compose-project.ts +4 -0
- package/src/container/docker-app.ts +74 -0
- package/src/container/log-archive.ts +33 -5
- package/src/container/memory-limit.ts +80 -0
- package/src/container/memory-oversubscription.ts +125 -0
- package/src/container/restart.ts +6 -1
- package/src/container/shared.ts +43 -16
- package/src/container/start.ts +313 -77
- package/src/container/stop.ts +19 -13
- package/src/container/verify-running.ts +12 -0
- package/src/cron/scheduler.ts +12 -4
- package/src/doctor/apparmor.ts +101 -0
- package/src/doctor/bounded-exec.ts +14 -0
- package/src/doctor/checks.ts +42 -15
- package/src/doctor/index.ts +14 -2
- package/src/doctor/plugin-bridge.ts +15 -3
- package/src/hostd/paths.ts +4 -0
- package/src/init/dockerfile.ts +7 -6
- package/src/init/windows-dev-link.ts +34 -2
- package/src/run/index.ts +17 -0
- package/src/run/resource-sample.ts +171 -0
- package/src/sandbox/availability.ts +7 -5
- package/src/sandbox/index.ts +8 -1
- package/src/sandbox/session-tmp.ts +26 -1
- package/src/skills/typeclaw-channel-github/SKILL.md +4 -2
- package/typeclaw.schema.json +7 -0
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
// The `com.docker.compose.project` value every typeclaw container is labelled
|
|
2
|
+
// with. Lives in its own module so the label can be read back (to enumerate
|
|
3
|
+
// running agents) without importing the start pipeline that writes it.
|
|
4
|
+
export const COMPOSE_PROJECT = 'typeclaw'
|
|
@@ -191,6 +191,41 @@ function startInstructions(app: DockerApp, platform: NodeJS.Platform): string[]
|
|
|
191
191
|
}
|
|
192
192
|
}
|
|
193
193
|
|
|
194
|
+
// Restart (not start) guidance for a daemon that is up but wedged. Kept
|
|
195
|
+
// separate from startInstructions because the two are not interchangeable:
|
|
196
|
+
// telling someone to open an app that is already open reads as a no-op and
|
|
197
|
+
// costs them the diagnosis. A livelocked VM often ignores the graceful stop,
|
|
198
|
+
// so each runtime's entry names the forceful fallback too.
|
|
199
|
+
function restartInstructions(app: DockerApp | null, platform: NodeJS.Platform): string[] {
|
|
200
|
+
if (app === null) {
|
|
201
|
+
return platform === 'linux'
|
|
202
|
+
? ['Restart the Docker daemon, then retry.', 'On most distros: `sudo systemctl restart docker`']
|
|
203
|
+
: ['Restart your Docker runtime, then retry.']
|
|
204
|
+
}
|
|
205
|
+
switch (app) {
|
|
206
|
+
case 'orbstack':
|
|
207
|
+
return [
|
|
208
|
+
'Restart OrbStack, then retry: `orb stop`, then reopen OrbStack.',
|
|
209
|
+
'A livelocked VM may ignore `orb stop`. If it hangs, quit OrbStack from the',
|
|
210
|
+
'menu bar (or kill its helper process), then reopen it.',
|
|
211
|
+
]
|
|
212
|
+
case 'docker-desktop':
|
|
213
|
+
return [
|
|
214
|
+
'Restart Docker Desktop, then retry.',
|
|
215
|
+
platform === 'darwin'
|
|
216
|
+
? 'Quit it from the menu bar (force quit if it will not exit), then reopen it.'
|
|
217
|
+
: 'Quit it from the tray (force quit if it will not exit), then reopen it.',
|
|
218
|
+
]
|
|
219
|
+
case 'colima':
|
|
220
|
+
return ['Restart Colima, then retry: `colima stop && colima start`', 'If the stop hangs: `colima stop --force`']
|
|
221
|
+
case 'podman':
|
|
222
|
+
return [
|
|
223
|
+
'Restart the Podman machine, then retry: `podman machine stop && podman machine start`',
|
|
224
|
+
'If the stop hangs: `podman machine stop --force`',
|
|
225
|
+
]
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
194
229
|
// Generic "start your daemon" guidance when we can't name the runtime. On Linux
|
|
195
230
|
// the daemon is usually systemd-managed; elsewhere it's a GUI app.
|
|
196
231
|
function genericStartInstructions(platform: NodeJS.Platform): string[] {
|
|
@@ -228,6 +263,25 @@ export function renderDockerUnavailableGuidance(
|
|
|
228
263
|
const { platform, nudge, installed, retryHint } = options
|
|
229
264
|
|
|
230
265
|
if (availability.reason === 'binary-missing') {
|
|
266
|
+
// Only claim "not installed" when nothing was actually detected. We probe
|
|
267
|
+
// the install locations separately, so a non-empty `installed` here means
|
|
268
|
+
// the runtime IS present and only the CLI is unreachable on this PATH —
|
|
269
|
+
// routine over SSH and in non-interactive shells, where the rc file that
|
|
270
|
+
// adds the runtime's bin dir never runs. Telling that operator to install
|
|
271
|
+
// Docker sends them down the wrong path entirely.
|
|
272
|
+
if (installed.length > 0) {
|
|
273
|
+
const names = installed.map(dockerAppLabel).join(', ')
|
|
274
|
+
return {
|
|
275
|
+
summary: `docker CLI not found on PATH (detected ${names}).`,
|
|
276
|
+
lines: [
|
|
277
|
+
`${names} appears to be installed, but no \`docker\` binary is on this shell's PATH.`,
|
|
278
|
+
'This is common over SSH and in non-interactive shells, where the shell',
|
|
279
|
+
"profile that adds the runtime's bin directory never runs.",
|
|
280
|
+
'Check `echo $PATH`, then start a login shell or add the runtime bin directory.',
|
|
281
|
+
...(retryHint ? ['', retryHint] : []),
|
|
282
|
+
],
|
|
283
|
+
}
|
|
284
|
+
}
|
|
231
285
|
const lines = [...INSTALL_LINES]
|
|
232
286
|
if (retryHint) {
|
|
233
287
|
lines.push('', retryHint)
|
|
@@ -235,6 +289,26 @@ export function renderDockerUnavailableGuidance(
|
|
|
235
289
|
return { summary: 'Docker is not installed.', lines }
|
|
236
290
|
}
|
|
237
291
|
|
|
292
|
+
// The daemon accepted our request and never answered. This is NOT "start it"
|
|
293
|
+
// territory — it is already running, so start instructions actively mislead.
|
|
294
|
+
// The observed cause is host/VM memory exhaustion driving the guest kernel
|
|
295
|
+
// into an unrecoverable reclaim livelock, which only a runtime restart clears
|
|
296
|
+
// and which recurs until the memory pressure itself is found.
|
|
297
|
+
if (availability.reason === 'unresponsive') {
|
|
298
|
+
const label = nudge !== null ? dockerAppLabel(nudge) : 'Your Docker runtime'
|
|
299
|
+
const lines = [
|
|
300
|
+
...restartInstructions(nudge, platform),
|
|
301
|
+
'',
|
|
302
|
+
'An unresponsive daemon is usually host or VM memory exhaustion, so it will',
|
|
303
|
+
'recur until the cause is found. After the restart, check what is consuming',
|
|
304
|
+
'memory before resuming work.',
|
|
305
|
+
]
|
|
306
|
+
if (retryHint) {
|
|
307
|
+
lines.push('', retryHint)
|
|
308
|
+
}
|
|
309
|
+
return { summary: `${label} is running but not responding. It needs a restart.`, lines }
|
|
310
|
+
}
|
|
311
|
+
|
|
238
312
|
// daemon-down: the configured Docker endpoint refused the connection. We can
|
|
239
313
|
// nudge the likely runtime to start, but must NOT claim it's stopped — the
|
|
240
314
|
// same error fires when the runtime is up and the CLI is on the wrong context.
|
|
@@ -6,7 +6,10 @@ import { DEFAULT_LOG_RETENTION_DAYS, MAX_LOG_RETENTION_DAYS } from '@/config'
|
|
|
6
6
|
|
|
7
7
|
import { resolveDockerBinary, sanitizeDockerStderr } from './shared'
|
|
8
8
|
|
|
9
|
-
export type DockerLogArchiveResult =
|
|
9
|
+
export type DockerLogArchiveResult =
|
|
10
|
+
| { ok: true; status: 'archived'; path: string }
|
|
11
|
+
| { ok: false; kind: 'unavailable'; reason: string }
|
|
12
|
+
| { ok: false; kind: 'failed'; reason: string }
|
|
10
13
|
|
|
11
14
|
export type DockerLogArchiver = (input: {
|
|
12
15
|
agentDir: string
|
|
@@ -65,6 +68,8 @@ const WRITE_SETTLEMENT_MS = 60_000
|
|
|
65
68
|
const STALE_PARTIAL_GRACE_MS = 5 * 60 * 1_000
|
|
66
69
|
const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1_000
|
|
67
70
|
const MAX_FILENAME_ATTEMPTS = 16
|
|
71
|
+
// Fixed daemon protocol text from moby's daemon/logs.go, not human-language input.
|
|
72
|
+
const DOCKER_LOGS_TERMINAL_UNAVAILABLE = 'can not get logs from container which is dead or marked for removal'
|
|
68
73
|
const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/
|
|
69
74
|
const NONCE_PATTERN = /^[a-f0-9]{32}$/
|
|
70
75
|
const ARCHIVE_FILENAME_PATTERN = /^[a-f0-9]{64}-(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z-[a-f0-9]{32}\.log$/
|
|
@@ -75,10 +80,10 @@ export async function archiveContainerLogs(
|
|
|
75
80
|
runtime: DockerLogArchiveRuntime = {},
|
|
76
81
|
): Promise<DockerLogArchiveResult> {
|
|
77
82
|
if (!CONTAINER_ID_PATTERN.test(containerId)) {
|
|
78
|
-
return { ok: false, reason: `Invalid full Docker container ID: ${containerId}` }
|
|
83
|
+
return { ok: false, kind: 'failed', reason: `Invalid full Docker container ID: ${containerId}` }
|
|
79
84
|
}
|
|
80
85
|
if (!Number.isInteger(retentionDays) || retentionDays < 1 || retentionDays > MAX_LOG_RETENTION_DAYS) {
|
|
81
|
-
return { ok: false, reason: `Invalid Docker log archive retention period: ${retentionDays}` }
|
|
86
|
+
return { ok: false, kind: 'failed', reason: `Invalid Docker log archive retention period: ${retentionDays}` }
|
|
82
87
|
}
|
|
83
88
|
|
|
84
89
|
const logsDir = join(agentDir, '.typeclaw', 'logs')
|
|
@@ -116,7 +121,9 @@ export async function archiveContainerLogs(
|
|
|
116
121
|
}
|
|
117
122
|
if (captured.exitCode !== 0) {
|
|
118
123
|
const detail = sanitizeDockerStderr(captured.stderrExcerpt)
|
|
119
|
-
|
|
124
|
+
const message = `docker logs exited with code ${captured.exitCode}${detail ? `: ${detail}` : ''}`
|
|
125
|
+
if (isDockerLogsPermanentlyUnavailable(captured.stderrExcerpt)) throw new LogArchiveError('unavailable', message)
|
|
126
|
+
throw new Error(message)
|
|
120
127
|
}
|
|
121
128
|
|
|
122
129
|
await output.chmod(0o600)
|
|
@@ -132,10 +139,31 @@ export async function archiveContainerLogs(
|
|
|
132
139
|
} catch (error) {
|
|
133
140
|
const cleanupErrors = await cleanupTemp(output, temporaryPath)
|
|
134
141
|
const suffix = cleanupErrors.length === 0 ? '' : ` (${cleanupErrors.join('; ')})`
|
|
135
|
-
return {
|
|
142
|
+
return {
|
|
143
|
+
ok: false,
|
|
144
|
+
kind: error instanceof LogArchiveError ? error.kind : 'failed',
|
|
145
|
+
reason: `${errorMessage(error)}${suffix}`,
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function dockerLogsUnavailableWarning(containerId: string): string {
|
|
151
|
+
return `Docker reported logs unavailable for container ${containerId} because it is dead or already being removed; no new snapshot was captured. Continuing cleanup so the agent can restart. Existing snapshots in .typeclaw/logs/ are preserved.`
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
class LogArchiveError extends Error {
|
|
155
|
+
constructor(
|
|
156
|
+
readonly kind: 'unavailable',
|
|
157
|
+
message: string,
|
|
158
|
+
) {
|
|
159
|
+
super(message)
|
|
136
160
|
}
|
|
137
161
|
}
|
|
138
162
|
|
|
163
|
+
function isDockerLogsPermanentlyUnavailable(stderr: string): boolean {
|
|
164
|
+
return stderr.toLowerCase().includes(DOCKER_LOGS_TERMINAL_UNAVAILABLE)
|
|
165
|
+
}
|
|
166
|
+
|
|
139
167
|
export async function streamProcessOutput(input: StreamCaptureInput): Promise<StreamCaptureResult> {
|
|
140
168
|
const { args, binary, cwd, maxBytes, output, timeoutMs } = input
|
|
141
169
|
const killFallbackMs = input.killFallbackMs ?? KILL_FALLBACK_MS
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { DockerExec } from './shared'
|
|
2
|
+
|
|
3
|
+
const BYTES_PER_MIB = 1024 * 1024
|
|
4
|
+
const BYTES_PER_GIB = 1024 * BYTES_PER_MIB
|
|
5
|
+
|
|
6
|
+
// A container has to hold the Bun runtime, the onnxruntime embedder during an
|
|
7
|
+
// index build, a headless Chrome on a heavy page, AND whatever of the 2 GiB
|
|
8
|
+
// /dev/shm allowance Chrome actually touches — shm pages are charged to the
|
|
9
|
+
// container's cgroup once one exists. 6 GiB clears that comfortably. A cap that
|
|
10
|
+
// strangles a legitimate browser or embedding workload is a defect, not
|
|
11
|
+
// hardening, so this is deliberately generous rather than minimal.
|
|
12
|
+
export const DEFAULT_CONTAINER_MEMORY_BYTES = 6 * BYTES_PER_GIB
|
|
13
|
+
|
|
14
|
+
// Memory left to the Docker VM itself (dockerd, containerd, the guest kernel,
|
|
15
|
+
// page cache) when the default does not fit. Without it a single agent sized to
|
|
16
|
+
// the whole machine reproduces the exhaustion this limit exists to prevent.
|
|
17
|
+
const HOST_HEADROOM_BYTES = 2 * BYTES_PER_GIB
|
|
18
|
+
|
|
19
|
+
// Below this a container cannot boot the agent at all, so clamping further down
|
|
20
|
+
// would trade a slow host for an agent that never starts.
|
|
21
|
+
const MINIMUM_CONTAINER_MEMORY_BYTES = 1 * BYTES_PER_GIB
|
|
22
|
+
|
|
23
|
+
// Docker accepts a plain byte count, which avoids re-introducing rounding error
|
|
24
|
+
// on the way back out of a value we already normalized.
|
|
25
|
+
export function formatMemorySize(bytes: number): string {
|
|
26
|
+
return String(Math.floor(bytes))
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export type MemoryLimitSource = 'default' | 'clamped'
|
|
30
|
+
|
|
31
|
+
export type ResolvedMemoryLimit = {
|
|
32
|
+
bytes: number
|
|
33
|
+
source: MemoryLimitSource
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// The limit is a fixed default clamped DOWNWARD to fit the machine, never a
|
|
37
|
+
// fraction of it.
|
|
38
|
+
//
|
|
39
|
+
// A percentage rule breaks precisely where this matters most: on a shared host
|
|
40
|
+
// running several agents, "50% of RAM" each oversubscribes by the agent count
|
|
41
|
+
// and reproduces the exhaustion with caps that made everyone feel safe. A
|
|
42
|
+
// budget that also moves with ambient conditions makes incidents
|
|
43
|
+
// unreproducible — the same agent behaves differently depending on what else
|
|
44
|
+
// happened to be running when it started.
|
|
45
|
+
//
|
|
46
|
+
// There is deliberately no operator override. The incident this limit exists
|
|
47
|
+
// for needed a ceiling, not a knob, and a config field nobody has asked for is
|
|
48
|
+
// surface that has to be honored, documented, and migrated forever. If a real
|
|
49
|
+
// workload ever proves 6 GiB too tight, that is the moment to add one — with a
|
|
50
|
+
// use case attached.
|
|
51
|
+
export function resolveMemoryLimit(options: { totalMemoryBytes?: number | undefined }): ResolvedMemoryLimit {
|
|
52
|
+
const total = options.totalMemoryBytes
|
|
53
|
+
if (total === undefined || !Number.isFinite(total) || total <= 0) {
|
|
54
|
+
return { bytes: DEFAULT_CONTAINER_MEMORY_BYTES, source: 'default' }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const affordable = total - HOST_HEADROOM_BYTES
|
|
58
|
+
if (affordable >= DEFAULT_CONTAINER_MEMORY_BYTES) {
|
|
59
|
+
return { bytes: DEFAULT_CONTAINER_MEMORY_BYTES, source: 'default' }
|
|
60
|
+
}
|
|
61
|
+
return { bytes: Math.max(affordable, MINIMUM_CONTAINER_MEMORY_BYTES), source: 'clamped' }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// The clamp has to measure the machine the CONTAINER runs on, which on macOS,
|
|
65
|
+
// Windows, and any remote-daemon setup is the Docker VM and not the workstation.
|
|
66
|
+
// During the incident that motivated this limit the workstation reported tens of
|
|
67
|
+
// gigabytes free while the VM died at 14.4 GiB.
|
|
68
|
+
//
|
|
69
|
+
// So there is deliberately NO os.totalmem() fallback: substituting workstation
|
|
70
|
+
// RAM for daemon capacity would reintroduce exactly the mismatch this function
|
|
71
|
+
// exists to avoid, and would do it silently. An unreadable daemon total is
|
|
72
|
+
// `undefined` — genuinely unknown — and every consumer treats unknown as "do not
|
|
73
|
+
// clamp, do not claim oversubscription" rather than inventing a number.
|
|
74
|
+
export async function readDockerTotalMemory(exec: DockerExec): Promise<number | undefined> {
|
|
75
|
+
const result = await exec(['info', '--format', '{{.MemTotal}}'])
|
|
76
|
+
if (result.exitCode !== 0) return undefined
|
|
77
|
+
const parsed = Number(result.stdout.trim())
|
|
78
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return undefined
|
|
79
|
+
return parsed
|
|
80
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { COMPOSE_PROJECT } from './compose-project'
|
|
2
|
+
import type { DockerExec } from './shared'
|
|
3
|
+
|
|
4
|
+
// `bytes: null` means the container runs with NO memory limit. That is a
|
|
5
|
+
// distinct, worse state than a large limit, not a missing reading: a pre-upgrade
|
|
6
|
+
// agent still running unbounded is exactly the thing that can exhaust the host,
|
|
7
|
+
// so it must stay on the roster rather than being dropped from the sum.
|
|
8
|
+
export type AgentMemoryClaim = {
|
|
9
|
+
containerName: string
|
|
10
|
+
bytes: number | null
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export type OversubscriptionWarning = {
|
|
14
|
+
claims: AgentMemoryClaim[]
|
|
15
|
+
claimedBytes: number
|
|
16
|
+
// null when the daemon total could not be read. An unbounded agent is still
|
|
17
|
+
// worth reporting without it, so this is not a reason to stay silent.
|
|
18
|
+
totalMemoryBytes: number | null
|
|
19
|
+
unbounded: AgentMemoryClaim[]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Warn, never block.
|
|
23
|
+
//
|
|
24
|
+
// The sum of every agent's cap is what the machine is on the hook for if they
|
|
25
|
+
// all peak together, so a total above the machine's memory means the caps do
|
|
26
|
+
// not actually bound the host — they only bound each agent individually, and
|
|
27
|
+
// the host can still be driven into the swap-backed reclaim livelock the caps
|
|
28
|
+
// exist to prevent.
|
|
29
|
+
//
|
|
30
|
+
// This deliberately does not stop the start. A fleet that has run fine at a
|
|
31
|
+
// nominal oversubscription for months would otherwise be bricked by an upgrade,
|
|
32
|
+
// and refusing to start an agent is a far worse failure than running one on a
|
|
33
|
+
// crowded host. The operator gets the arithmetic and decides.
|
|
34
|
+
export function decideMemoryOversubscription(options: {
|
|
35
|
+
running: AgentMemoryClaim[]
|
|
36
|
+
incoming: AgentMemoryClaim
|
|
37
|
+
totalMemoryBytes: number | undefined
|
|
38
|
+
}): OversubscriptionWarning | null {
|
|
39
|
+
const { running, incoming, totalMemoryBytes } = options
|
|
40
|
+
|
|
41
|
+
// A restart re-runs start for a container that is already listed, so drop any
|
|
42
|
+
// running claim under the incoming name before summing. Counting it twice
|
|
43
|
+
// would fire the warning on a plain `typeclaw restart` of a single agent.
|
|
44
|
+
const others = running.filter((claim) => claim.containerName !== incoming.containerName)
|
|
45
|
+
const claims = [...others, incoming]
|
|
46
|
+
const unbounded = claims.filter((claim) => claim.bytes === null)
|
|
47
|
+
const claimedBytes = claims.reduce((sum, claim) => sum + (claim.bytes ?? 0), 0)
|
|
48
|
+
|
|
49
|
+
const total =
|
|
50
|
+
totalMemoryBytes !== undefined && Number.isFinite(totalMemoryBytes) && totalMemoryBytes > 0
|
|
51
|
+
? totalMemoryBytes
|
|
52
|
+
: null
|
|
53
|
+
|
|
54
|
+
// Unbounded detection deliberately precedes the unknown-total guard.
|
|
55
|
+
// Identifying an agent running with no ceiling needs no capacity arithmetic,
|
|
56
|
+
// so an unreadable daemon total must not suppress it — and since the total is
|
|
57
|
+
// now genuinely unknown rather than silently backfilled from the workstation,
|
|
58
|
+
// that case is reachable in normal operation.
|
|
59
|
+
if (unbounded.length === 0 && (total === null || claimedBytes <= total)) return null
|
|
60
|
+
|
|
61
|
+
return { claims, claimedBytes, totalMemoryBytes: total, unbounded }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function formatOversubscriptionWarning(warning: OversubscriptionWarning): string[] {
|
|
65
|
+
const gib = (bytes: number): string => `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GiB`
|
|
66
|
+
const roster = warning.claims
|
|
67
|
+
.map((claim) => ` ${claim.containerName} ${claim.bytes === null ? 'unlimited' : gib(claim.bytes)}`)
|
|
68
|
+
.sort((a, b) => a.localeCompare(b))
|
|
69
|
+
|
|
70
|
+
const capacity = warning.totalMemoryBytes === null ? null : gib(warning.totalMemoryBytes)
|
|
71
|
+
const names = warning.unbounded.map((claim) => claim.containerName).join(', ')
|
|
72
|
+
const headline =
|
|
73
|
+
warning.unbounded.length > 0
|
|
74
|
+
? `${names} ${warning.unbounded.length === 1 ? 'is' : 'are'} running with no memory limit` +
|
|
75
|
+
(capacity === null ? ', and Docker capacity could not be read.' : `, against ${capacity} of Docker memory.`)
|
|
76
|
+
: `Agent memory limits total ${gib(warning.claimedBytes)} against ${capacity ?? 'an unknown amount'} of Docker memory.`
|
|
77
|
+
|
|
78
|
+
const remedy =
|
|
79
|
+
warning.unbounded.length > 0
|
|
80
|
+
? ['An unbounded agent can exhaust the host on its own. Restart it to apply', 'the current limit.']
|
|
81
|
+
: ['If they peak together the host can still exhaust. Stop an agent, or', 'give Docker more memory.']
|
|
82
|
+
|
|
83
|
+
return [headline, ...roster, ...remedy]
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Reads the memory cap Docker actually applied to every running agent. Docker
|
|
87
|
+
// reports 0 for a container created with no limit, which is UNLIMITED, not
|
|
88
|
+
// missing — an agent started before this field existed is precisely the case
|
|
89
|
+
// worth reporting, so it is preserved as a null claim rather than dropped.
|
|
90
|
+
export async function readRunningAgentMemoryClaims(exec: DockerExec): Promise<AgentMemoryClaim[]> {
|
|
91
|
+
const listed = await exec([
|
|
92
|
+
'ps',
|
|
93
|
+
'--filter',
|
|
94
|
+
`label=com.docker.compose.project=${COMPOSE_PROJECT}`,
|
|
95
|
+
'--format',
|
|
96
|
+
'{{.Names}}',
|
|
97
|
+
])
|
|
98
|
+
if (listed.exitCode !== 0) return []
|
|
99
|
+
const names = listed.stdout
|
|
100
|
+
.split('\n')
|
|
101
|
+
.map((line) => line.trim())
|
|
102
|
+
.filter((line) => line.length > 0)
|
|
103
|
+
if (names.length === 0) return []
|
|
104
|
+
|
|
105
|
+
const inspected = await exec(['inspect', '--format', '{{.Name}} {{.HostConfig.Memory}}', ...names])
|
|
106
|
+
if (inspected.exitCode !== 0) return []
|
|
107
|
+
return parseAgentMemoryClaims(inspected.stdout)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function parseAgentMemoryClaims(output: string): AgentMemoryClaim[] {
|
|
111
|
+
const claims: AgentMemoryClaim[] = []
|
|
112
|
+
for (const line of output.split('\n')) {
|
|
113
|
+
const trimmed = line.trim()
|
|
114
|
+
if (trimmed.length === 0) continue
|
|
115
|
+
const separator = trimmed.lastIndexOf(' ')
|
|
116
|
+
if (separator < 0) continue
|
|
117
|
+
const raw = Number(trimmed.slice(separator + 1))
|
|
118
|
+
if (!Number.isFinite(raw) || raw < 0) continue
|
|
119
|
+
// `docker inspect` renders a container name with a leading slash.
|
|
120
|
+
const containerName = trimmed.slice(0, separator).replace(/^\//, '')
|
|
121
|
+
if (containerName.length === 0) continue
|
|
122
|
+
claims.push({ containerName, bytes: raw === 0 ? null : raw })
|
|
123
|
+
}
|
|
124
|
+
return claims
|
|
125
|
+
}
|
package/src/container/restart.ts
CHANGED
|
@@ -38,7 +38,12 @@ async function restartWithLease(
|
|
|
38
38
|
lease: AgentOperationLease,
|
|
39
39
|
operationLock: WithAgentOperationLock,
|
|
40
40
|
): Promise<RestartResult> {
|
|
41
|
-
const stopped = await stop({
|
|
41
|
+
const stopped = await stop({
|
|
42
|
+
cwd: options.cwd,
|
|
43
|
+
onWarning: options.onWarning,
|
|
44
|
+
operationLock,
|
|
45
|
+
operationLease: lease,
|
|
46
|
+
})
|
|
42
47
|
if (!stopped.ok) return { ok: false, reason: `stop failed: ${stopped.reason}` }
|
|
43
48
|
options.onStopped?.(stopped)
|
|
44
49
|
|
package/src/container/shared.ts
CHANGED
|
@@ -271,15 +271,40 @@ export function sanitizeDockerStderr(stderr: string): string {
|
|
|
271
271
|
return lines.join('; ')
|
|
272
272
|
}
|
|
273
273
|
|
|
274
|
-
export type DockerAvailability =
|
|
274
|
+
export type DockerAvailability =
|
|
275
|
+
| { ok: true }
|
|
276
|
+
| { ok: false; reason: 'binary-missing' | 'daemon-down' | 'unresponsive'; detail: string }
|
|
277
|
+
|
|
278
|
+
// A healthy `docker info` answers in well under a second; a refused connection
|
|
279
|
+
// fails immediately. The only thing this budget can cut short is a daemon that
|
|
280
|
+
// accepted the request and never answered — which is exactly the case we want
|
|
281
|
+
// to report rather than wait out. Deliberately generous so a cold Docker
|
|
282
|
+
// Desktop / OrbStack still lands inside it.
|
|
283
|
+
export const DOCKER_AVAILABILITY_TIMEOUT_MS = 5_000
|
|
275
284
|
|
|
276
285
|
// `docker info --format {{.ServerVersion}}` is the probe of choice because it
|
|
277
286
|
// requires both the client AND a reachable daemon. `docker --version` would
|
|
278
287
|
// miss the "Docker Desktop installed but not running" case, which is the
|
|
279
288
|
// common failure mode on macOS.
|
|
280
|
-
|
|
281
|
-
|
|
289
|
+
//
|
|
290
|
+
// The probe is time-bounded because an unresponsive daemon otherwise hangs it
|
|
291
|
+
// forever: a host VM in an OOM livelock accepts the connection and never
|
|
292
|
+
// replies, and `docker info` has no client-side deadline of its own. Every
|
|
293
|
+
// Docker-dependent CLI command gates on this function, so an unbounded probe
|
|
294
|
+
// means `status`, `start`, `stop`, `logs`, `tui` and `doctor` all wedge with no
|
|
295
|
+
// output at all — the tool you reach for when the host is sick is the one that
|
|
296
|
+
// hangs. Mirrors captureCrashLogs in ./verify-running: AbortSignal.timeout
|
|
297
|
+
// plus an explicit `signal.aborted` check to tell a timeout from a failure.
|
|
298
|
+
export async function checkDockerAvailable(
|
|
299
|
+
exec: DockerExec = defaultDockerExec,
|
|
300
|
+
timeoutMs: number = DOCKER_AVAILABILITY_TIMEOUT_MS,
|
|
301
|
+
): Promise<DockerAvailability> {
|
|
302
|
+
const signal = AbortSignal.timeout(timeoutMs)
|
|
303
|
+
const result = await exec(['info', '--format', '{{.ServerVersion}}'], { signal })
|
|
282
304
|
if (result.exitCode === 0) return { ok: true }
|
|
305
|
+
if (signal.aborted) {
|
|
306
|
+
return { ok: false, reason: 'unresponsive', detail: `docker info did not respond within ${timeoutMs}ms` }
|
|
307
|
+
}
|
|
283
308
|
if (result.stderr === DOCKER_NOT_FOUND_STDERR) {
|
|
284
309
|
return { ok: false, reason: 'binary-missing', detail: result.stderr }
|
|
285
310
|
}
|
|
@@ -584,19 +609,21 @@ export type ContainerState = { exists: false } | { exists: true; running: boolea
|
|
|
584
609
|
// State.Running boolean — not mere presence in `ps -a` — is what callers need
|
|
585
610
|
// to distinguish a live agent from a corpse left over after a crash (which,
|
|
586
611
|
// since we run without `--rm`, sticks around in `ps -a` until the next start).
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
612
|
+
// Goes through DockerExec rather than spawning directly so callers can bound it
|
|
613
|
+
// and tests can inject it. Both matter for the diagnostic paths: an unresponsive
|
|
614
|
+
// daemon accepts `docker inspect` and never answers, which would hang `doctor` —
|
|
615
|
+
// the one command whose whole job is to report on a sick host.
|
|
616
|
+
export async function inspectContainer(
|
|
617
|
+
name: string,
|
|
618
|
+
options: { signal?: AbortSignal; exec?: DockerExec } = {},
|
|
619
|
+
): Promise<ContainerState> {
|
|
620
|
+
const exec = options.exec ?? defaultDockerExec
|
|
621
|
+
const result = await exec(
|
|
622
|
+
['inspect', '--format', '{{.State.Running}}', name],
|
|
623
|
+
options.signal === undefined ? undefined : { signal: options.signal },
|
|
624
|
+
)
|
|
625
|
+
if (result.exitCode !== 0) return { exists: false }
|
|
626
|
+
return { exists: true, running: result.stdout.trim() === 'true' }
|
|
600
627
|
}
|
|
601
628
|
|
|
602
629
|
export function getBun(): { spawn: typeof Bun.spawn; which: typeof Bun.which } | undefined {
|