typeclaw 0.48.13 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/package.json +5 -5
  2. package/scripts/dump-system-prompt.ts +52 -110
  3. package/src/agent/index.ts +144 -72
  4. package/src/agent/live-sessions.ts +2 -2
  5. package/src/agent/live-subagents.ts +1 -0
  6. package/src/agent/plugin-tools.ts +53 -11
  7. package/src/agent/prompt-cache-key.ts +42 -0
  8. package/src/agent/provider-error.ts +179 -10
  9. package/src/agent/restart-handoff/index.ts +7 -14
  10. package/src/agent/reviewer-activity.ts +68 -0
  11. package/src/agent/session-origin.ts +33 -0
  12. package/src/agent/subagents.ts +13 -1
  13. package/src/agent/system-prompt.ts +68 -202
  14. package/src/agent/todo/continuation-policy.ts +12 -4
  15. package/src/agent/tool-file-safety.ts +113 -14
  16. package/src/agent/tool-not-found-nudge.ts +1 -1
  17. package/src/agent/tools/channel-history-render.ts +2 -1
  18. package/src/agent/tools/channel-reply.ts +23 -5
  19. package/src/agent/tools/channel-send.ts +49 -7
  20. package/src/agent/tools/curl-impersonate.ts +11 -2
  21. package/src/agent/tools/post-github-review.ts +77 -15
  22. package/src/agent/tools/skip-response.ts +2 -2
  23. package/src/agent/tools/spawn-subagent.ts +163 -12
  24. package/src/agent/tools/subagent-output.ts +8 -0
  25. package/src/bundled-plugins/backup/README.md +10 -9
  26. package/src/bundled-plugins/backup/runner.ts +140 -58
  27. package/src/bundled-plugins/backup/subagents.ts +58 -26
  28. package/src/bundled-plugins/github-cli-auth/gh-command.ts +34 -2
  29. package/src/bundled-plugins/github-cli-auth/granted-push.ts +135 -0
  30. package/src/bundled-plugins/github-cli-auth/index.ts +83 -11
  31. package/src/bundled-plugins/github-cli-auth/review-account.ts +23 -0
  32. package/src/bundled-plugins/github-cli-auth/review-checkout.ts +9 -2
  33. package/src/bundled-plugins/github-cli-auth/review-recorder.ts +45 -7
  34. package/src/bundled-plugins/operator/operator.ts +15 -4
  35. package/src/bundled-plugins/reviewer/skills/code-review.ts +1 -1
  36. package/src/bundled-plugins/security/index.ts +50 -2
  37. package/src/bundled-plugins/security/policies/git-exfil.ts +24 -12
  38. package/src/channels/adapters/discord-bot-reactions.ts +18 -0
  39. package/src/channels/adapters/discord-bot.ts +122 -67
  40. package/src/channels/adapters/discord-reactions.ts +18 -0
  41. package/src/channels/adapters/discord.ts +114 -8
  42. package/src/channels/adapters/github/inbound.ts +103 -8
  43. package/src/channels/adapters/github/index.ts +251 -19
  44. package/src/channels/adapters/github/last-alive.ts +27 -0
  45. package/src/channels/adapters/github/outbound.ts +54 -18
  46. package/src/channels/adapters/github/pending-review-store.ts +141 -0
  47. package/src/channels/adapters/github/permission-guidance.ts +1 -0
  48. package/src/channels/adapters/github/reactions.ts +100 -0
  49. package/src/channels/adapters/github/reconcile-cooldown-store.ts +197 -67
  50. package/src/channels/adapters/github/reconcile-open-prs.ts +418 -33
  51. package/src/channels/adapters/github/recovery.ts +106 -0
  52. package/src/channels/adapters/github/review-check.ts +510 -0
  53. package/src/channels/adapters/github/review-submitter.ts +34 -2
  54. package/src/channels/adapters/instagram.ts +32 -14
  55. package/src/channels/adapters/kakaotalk.ts +22 -7
  56. package/src/channels/adapters/line.ts +18 -3
  57. package/src/channels/adapters/outbound-account.ts +20 -0
  58. package/src/channels/adapters/own-reactions.ts +66 -0
  59. package/src/channels/adapters/recovery-correlation.ts +214 -0
  60. package/src/channels/adapters/slack-bot-classify.ts +18 -11
  61. package/src/channels/adapters/slack-bot-dedupe.ts +111 -28
  62. package/src/channels/adapters/slack-bot-reactions.ts +18 -0
  63. package/src/channels/adapters/slack-bot.ts +125 -57
  64. package/src/channels/adapters/slack-classify.ts +25 -4
  65. package/src/channels/adapters/slack-inbound-revision.ts +96 -0
  66. package/src/channels/adapters/slack-reactions.ts +18 -0
  67. package/src/channels/adapters/slack-thread-ts.ts +6 -0
  68. package/src/channels/adapters/slack.ts +48 -3
  69. package/src/channels/adapters/teams.ts +14 -4
  70. package/src/channels/adapters/telegram-bot.ts +56 -10
  71. package/src/channels/adapters/webex-bot.ts +12 -2
  72. package/src/channels/adapters/webex.ts +12 -2
  73. package/src/channels/background-handoff.ts +434 -0
  74. package/src/channels/background-obligations.ts +893 -0
  75. package/src/channels/continuity-types.ts +271 -0
  76. package/src/channels/github-review-check.ts +598 -0
  77. package/src/channels/github-review-turn-ledger.ts +102 -28
  78. package/src/channels/github-review-verdict-coordinator.ts +72 -12
  79. package/src/channels/github-token-bridge.ts +11 -5
  80. package/src/channels/inbound-journal.ts +1738 -0
  81. package/src/channels/index.ts +6 -1
  82. package/src/channels/manager.ts +17 -7
  83. package/src/channels/recovery-dispatcher.ts +264 -0
  84. package/src/channels/recovery-notice.ts +94 -0
  85. package/src/channels/recovery-outbox.ts +459 -0
  86. package/src/channels/router.ts +2539 -615
  87. package/src/channels/schema.ts +8 -1
  88. package/src/channels/subagent-completion-bridge.ts +17 -14
  89. package/src/channels/types.ts +121 -16
  90. package/src/channels/waiting-reactions.ts +664 -0
  91. package/src/cli/compose.ts +7 -4
  92. package/src/cli/hostd.ts +49 -3
  93. package/src/cli/inspect.ts +11 -14
  94. package/src/coding-agent/broker-body.ts +38 -0
  95. package/src/coding-agent/broker.ts +290 -0
  96. package/src/coding-agent/claude.ts +95 -0
  97. package/src/coding-agent/codex.ts +104 -0
  98. package/src/coding-agent/index.ts +506 -0
  99. package/src/coding-agent/omo-state.ts +69 -0
  100. package/src/coding-agent/omo-stream.ts +98 -0
  101. package/src/coding-agent/omo.ts +192 -0
  102. package/src/coding-agent/runner.ts +263 -0
  103. package/src/coding-agent/tool.ts +62 -0
  104. package/src/coding-agent/worktree.ts +355 -0
  105. package/src/compose/restart.ts +10 -3
  106. package/src/compose/start.ts +20 -3
  107. package/src/config/config.ts +10 -3
  108. package/src/config/providers.ts +12 -9
  109. package/src/container/exit-reason.ts +44 -0
  110. package/src/container/index.ts +5 -0
  111. package/src/container/log-archive.ts +51 -13
  112. package/src/container/memory-limit.ts +1 -1
  113. package/src/container/memory-oversubscription.ts +36 -6
  114. package/src/container/oom-events.ts +86 -0
  115. package/src/container/restart.ts +2 -0
  116. package/src/container/shared.ts +8 -1
  117. package/src/container/start.ts +25 -13
  118. package/src/container/stop.ts +6 -2
  119. package/src/container/tool-oom-priority.ts +32 -0
  120. package/src/cron/consumer.ts +10 -4
  121. package/src/cron/reloadable.ts +23 -2
  122. package/src/doctor/checks.ts +14 -2
  123. package/src/hostd/daemon.ts +20 -10
  124. package/src/hostd/spawn.ts +106 -5
  125. package/src/hostd/version.ts +20 -0
  126. package/src/hostd/watchdog.ts +241 -0
  127. package/src/init/dockerfile.ts +57 -592
  128. package/src/init/index.ts +2 -3
  129. package/src/mcp/client.ts +3 -1
  130. package/src/operations/classify-tool-outcome.ts +45 -1
  131. package/src/operations/incidents.ts +10 -5
  132. package/src/plugin/core-tool-names.ts +1 -1
  133. package/src/plugin/define.ts +1 -0
  134. package/src/plugin/index.ts +1 -0
  135. package/src/plugin/types.ts +6 -1
  136. package/src/run/background-handoff-boot.ts +86 -0
  137. package/src/run/index.ts +83 -83
  138. package/src/run/resource-sample.ts +19 -0
  139. package/src/sandbox/build.ts +4 -0
  140. package/src/sandbox/coding-agent.ts +214 -0
  141. package/src/sandbox/policy.ts +1 -0
  142. package/src/sandbox/runtime-private.ts +4 -1
  143. package/src/secrets/coding-agent-auth.ts +97 -0
  144. package/src/secrets/index.ts +0 -16
  145. package/src/server/command-runner.ts +6 -2
  146. package/src/server/index.ts +101 -40
  147. package/src/skills/typeclaw-channel-github/SKILL.md +7 -5
  148. package/src/skills/typeclaw-claude-code/SKILL.md +25 -260
  149. package/src/skills/typeclaw-claude-code/references/auth-flow.md +7 -23
  150. package/src/skills/typeclaw-claude-code/references/headless-sessions.md +11 -0
  151. package/src/skills/typeclaw-claude-code/references/results-and-worktrees.md +13 -0
  152. package/src/skills/typeclaw-codex-cli/SKILL.md +35 -270
  153. package/src/skills/typeclaw-codex-cli/references/auth-flow.md +13 -20
  154. package/src/skills/typeclaw-codex-cli/references/headless-sessions.md +26 -0
  155. package/src/skills/typeclaw-codex-cli/references/results-and-worktrees.md +28 -0
  156. package/src/skills/typeclaw-config/SKILL.md +21 -21
  157. package/src/skills/typeclaw-config/references/dockerfile.md +14 -13
  158. package/src/skills/typeclaw-cron/SKILL.md +1 -1
  159. package/src/skills/typeclaw-github-contributing/SKILL.md +2 -2
  160. package/src/skills/typeclaw-memory/SKILL.md +3 -3
  161. package/src/skills/typeclaw-omo-cli/SKILL.md +52 -0
  162. package/src/skills/typeclaw-omo-cli/references/auth-flow.md +11 -0
  163. package/src/skills/typeclaw-omo-cli/references/headless-sessions.md +11 -0
  164. package/src/skills/typeclaw-omo-cli/references/results-and-worktrees.md +23 -0
  165. package/src/skills/typeclaw-plugins/SKILL.md +15 -15
  166. package/src/skills/typeclaw-skills/SKILL.md +19 -2
  167. package/src/skills/typeclaw-troubleshooting/SKILL.md +15 -2
  168. package/typeclaw.schema.json +12 -1
  169. package/src/secrets/claude-credentials-json.ts +0 -129
  170. package/src/secrets/codex-auth-json.ts +0 -102
  171. package/src/secrets/export-claude-credentials-file.ts +0 -270
  172. package/src/secrets/export-codex-auth-file.ts +0 -250
  173. package/src/skills/typeclaw-claude-code/references/stop-hook.md +0 -101
  174. package/src/skills/typeclaw-claude-code/references/tmux-driving.md +0 -243
  175. package/src/skills/typeclaw-codex-cli/references/stop-hook.md +0 -92
  176. package/src/skills/typeclaw-codex-cli/references/tmux-driving.md +0 -239
@@ -4,7 +4,7 @@ import { join } from 'node:path'
4
4
 
5
5
  import { DEFAULT_LOG_RETENTION_DAYS, MAX_LOG_RETENTION_DAYS } from '@/config'
6
6
 
7
- import { resolveDockerBinary, sanitizeDockerStderr } from './shared'
7
+ import { defaultDockerExec, resolveDockerBinary, sanitizeDockerStderr, type DockerExecResult } from './shared'
8
8
 
9
9
  export type DockerLogArchiveResult =
10
10
  | { ok: true; status: 'archived'; path: string }
@@ -49,6 +49,7 @@ export type StreamCaptureResult = {
49
49
 
50
50
  export type DockerLogArchiveRuntime = {
51
51
  capture?: (input: StreamCaptureInput) => Promise<StreamCaptureResult>
52
+ inspectLogConfig?: (args: string[], options: { cwd: string; signal: AbortSignal }) => Promise<DockerExecResult>
52
53
  nonce?: () => string
53
54
  now?: () => Date
54
55
  resolveDockerBinary?: () => string | null
@@ -74,6 +75,43 @@ const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/
74
75
  const NONCE_PATTERN = /^[a-f0-9]{32}$/
75
76
  const ARCHIVE_FILENAME_PATTERN = /^[a-f0-9]{64}-(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z-[a-f0-9]{32}\.log$/
76
77
  const PARTIAL_FILENAME_PATTERN = /^\.[a-f0-9]{64}-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z-[a-f0-9]{32}\.partial$/
78
+ const DOCKER_LOG_INSPECT_TIMEOUT_MS = 10_000
79
+
80
+ async function archiveByteLimit(
81
+ agentDir: string,
82
+ containerId: string,
83
+ inspect: NonNullable<DockerLogArchiveRuntime['inspectLogConfig']>,
84
+ ): Promise<number> {
85
+ try {
86
+ const result = await inspect(['inspect', '--format', '{{json .HostConfig.LogConfig}}', containerId], {
87
+ cwd: agentDir,
88
+ signal: AbortSignal.timeout(DOCKER_LOG_INSPECT_TIMEOUT_MS),
89
+ })
90
+ if (result.exitCode !== 0) return MAX_DOCKER_LOG_ARCHIVE_BYTES
91
+ const config: unknown = JSON.parse(result.stdout)
92
+ if (typeof config !== 'object' || config === null || !('Type' in config) || config.Type !== 'json-file')
93
+ return MAX_DOCKER_LOG_ARCHIVE_BYTES
94
+ if (!('Config' in config) || typeof config.Config !== 'object' || config.Config === null)
95
+ return MAX_DOCKER_LOG_ARCHIVE_BYTES
96
+ const opts = config.Config
97
+ if (!('max-size' in opts) || typeof opts['max-size'] !== 'string') return MAX_DOCKER_LOG_ARCHIVE_BYTES
98
+ const maxFiles = 'max-file' in opts ? opts['max-file'] : '1'
99
+ // Docker parses max-file with strconv.Atoi, so "05" is 5; only positive counts are bounded.
100
+ if (typeof maxFiles !== 'string' || !/^\+?\d+$/.test(maxFiles) || Number(maxFiles) < 1)
101
+ return MAX_DOCKER_LOG_ARCHIVE_BYTES
102
+ // Docker json-file uses go-units FromHumanSize: decimal powers even for "MiB".
103
+ const sizeMatch = /^(\+?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)(?: ?(b|[kmgtp](?:i?b)?))?$/i.exec(opts['max-size'])
104
+ if (!sizeMatch) return MAX_DOCKER_LOG_ARCHIVE_BYTES
105
+ const prefix = sizeMatch[2]?.toLowerCase()[0]
106
+ const multiplier = prefix && prefix !== 'b' ? 1000 ** ('kmgtp'.indexOf(prefix) + 1) : 1
107
+ const size = Math.trunc(Number(sizeMatch[1]) * multiplier)
108
+ const capacity = size * Number(maxFiles)
109
+ if (size <= 0 || !Number.isSafeInteger(capacity)) return MAX_DOCKER_LOG_ARCHIVE_BYTES
110
+ return Math.min(MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES, Math.max(MAX_DOCKER_LOG_ARCHIVE_BYTES, capacity))
111
+ } catch {
112
+ return MAX_DOCKER_LOG_ARCHIVE_BYTES
113
+ }
114
+ }
77
115
 
78
116
  export async function archiveContainerLogs(
79
117
  { agentDir, containerId, retentionDays = DEFAULT_LOG_RETENTION_DAYS }: Parameters<DockerLogArchiver>[0],
@@ -106,18 +144,21 @@ export async function archiveContainerLogs(
106
144
  if (binary === null) throw new Error('Docker binary could not be resolved for log archival')
107
145
 
108
146
  const capture = runtime.capture ?? streamProcessOutput
147
+ const maxBytes = await archiveByteLimit(agentDir, containerId, runtime.inspectLogConfig ?? defaultDockerExec)
109
148
  const captured = await capture({
110
149
  args: ['logs', '--timestamps', containerId],
111
150
  binary,
112
151
  cwd: agentDir,
113
- maxBytes: MAX_DOCKER_LOG_ARCHIVE_BYTES,
152
+ maxBytes,
114
153
  output,
115
154
  timeoutMs: DOCKER_LOG_ARCHIVE_TIMEOUT_MS,
116
155
  })
117
156
  if (captured.terminationBoundExceeded) throw new Error('docker logs did not settle after SIGKILL')
118
157
  if (captured.timedOut) throw new Error(`docker logs timed out after ${DOCKER_LOG_ARCHIVE_TIMEOUT_MS}ms`)
119
158
  if (captured.overflowed) {
120
- throw new Error(`docker logs exceeded the ${MAX_DOCKER_LOG_ARCHIVE_BYTES}-byte archive limit`)
159
+ throw new Error(
160
+ `docker logs exceeded the ${maxBytes}-byte archive limit. Container ${containerId} was preserved; export both log streams with docker logs --timestamps ${containerId} > private-log-file 2>&1 (choose a private destination); confirm the export succeeded before removing it with docker rm ${containerId} and retrying.`,
161
+ )
121
162
  }
122
163
  if (captured.exitCode !== 0) {
123
164
  const detail = sanitizeDockerStderr(captured.stderrExcerpt)
@@ -128,13 +169,16 @@ export async function archiveContainerLogs(
128
169
 
129
170
  await output.chmod(0o600)
130
171
  const incomingSize = (await output.stat()).size
131
- await pruneToCapacity(logsDir, incomingSize)
172
+ if (incomingSize > MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES) {
173
+ throw new Error('Docker log archive exceeds total archive capacity')
174
+ }
132
175
  await output.close()
133
176
  output = null
134
177
 
135
178
  await rename(temporaryPath, allocation.finalPath)
136
179
  temporaryPath = null
137
180
  await chmod(allocation.finalPath, 0o600)
181
+ await pruneToCapacity(logsDir, allocation.finalPath)
138
182
  return { ok: true, status: 'archived', path: allocation.finalPath }
139
183
  } catch (error) {
140
184
  const cleanupErrors = await cleanupTemp(output, temporaryPath)
@@ -334,10 +378,7 @@ async function pruneByAgeAndRemoveStalePartials(logsDir: string, ageCutoff: numb
334
378
  }
335
379
  }
336
380
 
337
- async function pruneToCapacity(logsDir: string, incomingSize: number): Promise<void> {
338
- if (incomingSize > MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES) {
339
- throw new Error('Docker log archive exceeds total archive capacity')
340
- }
381
+ async function pruneToCapacity(logsDir: string, publishedPath: string): Promise<void> {
341
382
  const snapshots: Array<{ name: string; path: string; size: number; timestamp: number }> = []
342
383
  for (const entry of await collectDirectoryEntries(logsDir)) {
343
384
  if (!entry.isFile()) continue
@@ -351,11 +392,8 @@ async function pruneToCapacity(logsDir: string, incomingSize: number): Promise<v
351
392
  let totalBytes = snapshots.reduce((total, snapshot) => total + snapshot.size, 0)
352
393
  let count = snapshots.length
353
394
  for (const snapshot of snapshots) {
354
- if (
355
- count + 1 <= MAX_DOCKER_LOG_ARCHIVE_SNAPSHOTS &&
356
- totalBytes + incomingSize <= MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES
357
- )
358
- break
395
+ if (count <= MAX_DOCKER_LOG_ARCHIVE_SNAPSHOTS && totalBytes <= MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES) break
396
+ if (snapshot.path === publishedPath) continue
359
397
  await removeRaceSafe(snapshot.path)
360
398
  count -= 1
361
399
  totalBytes -= snapshot.size
@@ -14,7 +14,7 @@ export const DEFAULT_CONTAINER_MEMORY_BYTES = 6 * BYTES_PER_GIB
14
14
  // Memory left to the Docker VM itself (dockerd, containerd, the guest kernel,
15
15
  // page cache) when the default does not fit. Without it a single agent sized to
16
16
  // the whole machine reproduces the exhaustion this limit exists to prevent.
17
- const HOST_HEADROOM_BYTES = 2 * BYTES_PER_GIB
17
+ export const HOST_HEADROOM_BYTES = 2 * BYTES_PER_GIB
18
18
 
19
19
  // Below this a container cannot boot the agent at all, so clamping further down
20
20
  // would trade a slow host for an agent that never starts.
@@ -1,4 +1,5 @@
1
1
  import { COMPOSE_PROJECT } from './compose-project'
2
+ import { HOST_HEADROOM_BYTES, readDockerTotalMemory } from './memory-limit'
2
3
  import type { DockerExec } from './shared'
3
4
 
4
5
  // `bytes: null` means the container runs with NO memory limit. That is a
@@ -33,7 +34,8 @@ export type OversubscriptionWarning = {
33
34
  // crowded host. The operator gets the arithmetic and decides.
34
35
  export function decideMemoryOversubscription(options: {
35
36
  running: AgentMemoryClaim[]
36
- incoming: AgentMemoryClaim
37
+ // Omitted for a fleet-wide check run after every agent is already up.
38
+ incoming?: AgentMemoryClaim
37
39
  totalMemoryBytes: number | undefined
38
40
  }): OversubscriptionWarning | null {
39
41
  const { running, incoming, totalMemoryBytes } = options
@@ -41,8 +43,10 @@ export function decideMemoryOversubscription(options: {
41
43
  // A restart re-runs start for a container that is already listed, so drop any
42
44
  // running claim under the incoming name before summing. Counting it twice
43
45
  // would fire the warning on a plain `typeclaw restart` of a single agent.
44
- const others = running.filter((claim) => claim.containerName !== incoming.containerName)
45
- const claims = [...others, incoming]
46
+ const claims =
47
+ incoming === undefined
48
+ ? running
49
+ : [...running.filter((claim) => claim.containerName !== incoming.containerName), incoming]
46
50
  const unbounded = claims.filter((claim) => claim.bytes === null)
47
51
  const claimedBytes = claims.reduce((sum, claim) => sum + (claim.bytes ?? 0), 0)
48
52
 
@@ -63,9 +67,10 @@ export function decideMemoryOversubscription(options: {
63
67
 
64
68
  export function formatOversubscriptionWarning(warning: OversubscriptionWarning): string[] {
65
69
  const gib = (bytes: number): string => `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GiB`
70
+ const width = warning.claims.reduce((w, claim) => Math.max(w, claim.containerName.length), 0)
66
71
  const roster = warning.claims
67
- .map((claim) => ` ${claim.containerName} ${claim.bytes === null ? 'unlimited' : gib(claim.bytes)}`)
68
- .sort((a, b) => a.localeCompare(b))
72
+ .toSorted((a, b) => a.containerName.localeCompare(b.containerName))
73
+ .map((claim) => ` ${claim.containerName.padEnd(width)} ${claim.bytes === null ? 'unlimited' : gib(claim.bytes)}`)
69
74
 
70
75
  const capacity = warning.totalMemoryBytes === null ? null : gib(warning.totalMemoryBytes)
71
76
  const names = warning.unbounded.map((claim) => claim.containerName).join(', ')
@@ -78,11 +83,36 @@ export function formatOversubscriptionWarning(warning: OversubscriptionWarning):
78
83
  const remedy =
79
84
  warning.unbounded.length > 0
80
85
  ? ['An unbounded agent can exhaust the host on its own. Restart it to apply', 'the current limit.']
81
- : ['If they peak together the host can still exhaust. Stop an agent, or', 'give Docker more memory.']
86
+ : oversubscriptionRemedy(warning.claimedBytes)
82
87
 
83
88
  return [headline, ...roster, ...remedy]
84
89
  }
85
90
 
91
+ function oversubscriptionRemedy(claimedBytes: number): string[] {
92
+ const targetGib = Math.ceil((claimedBytes + HOST_HEADROOM_BYTES) / (1024 * 1024 * 1024))
93
+ return [
94
+ 'If they peak together the host can still exhaust. Either:',
95
+ ' • Stop an agent: run `typeclaw stop` in its folder.',
96
+ ` • Give Docker at least ${targetGib}GiB of memory, then restart the runtime:`,
97
+ ' Docker Desktop: Settings → Resources → Memory limit → Apply & restart',
98
+ ` OrbStack: orb config set memory_mib ${targetGib * 1024}`,
99
+ ` Colima: colima stop && colima start --memory ${targetGib}`,
100
+ ' Linux Engine: Docker uses host RAM directly; stop an agent instead.',
101
+ ]
102
+ }
103
+
104
+ // Fleet-wide variant for `typeclaw compose`: agents start concurrently, so a
105
+ // per-agent check sees whichever siblings happened to be up and repeats the same
106
+ // advice once per agent. Checking once after the whole fleet settled reports the
107
+ // real total exactly once.
108
+ export async function readFleetMemoryOversubscription(exec: DockerExec): Promise<OversubscriptionWarning | null> {
109
+ const [running, totalMemoryBytes] = await Promise.all([
110
+ readRunningAgentMemoryClaims(exec),
111
+ readDockerTotalMemory(exec),
112
+ ])
113
+ return decideMemoryOversubscription({ running, totalMemoryBytes })
114
+ }
115
+
86
116
  // Reads the memory cap Docker actually applied to every running agent. Docker
87
117
  // reports 0 for a container created with no limit, which is UNLIMITED, not
88
118
  // missing — an agent started before this field existed is precisely the case
@@ -0,0 +1,86 @@
1
+ import { readFileSync } from 'node:fs'
2
+
3
+ export type OomEventCounts = { oom: number; oomKill: number }
4
+ export type OomEventsReader = () => OomEventCounts | null
5
+
6
+ // Docker's private cgroup namespace exposes its own v2 root here. Cgroup v1,
7
+ // dev hosts and unreadable/malformed files provide no reliable pair: fail open.
8
+ export function readOomEvents(path = '/sys/fs/cgroup/memory.events'): OomEventCounts | null {
9
+ try {
10
+ const events = readFileSync(path, 'utf8')
11
+ const oom = /^oom (\d+)$/m.exec(events)
12
+ const oomKill = /^oom_kill (\d+)$/m.exec(events)
13
+ if (oom === null || oomKill === null) return null
14
+ const counts = { oom: Number(oom[1]), oomKill: Number(oomKill[1]) }
15
+ return Number.isSafeInteger(counts.oom) && Number.isSafeInteger(counts.oomKill) ? counts : null
16
+ } catch {
17
+ return null
18
+ }
19
+ }
20
+
21
+ export function readMemoryLimitMiB(path = '/sys/fs/cgroup/memory.max'): number | null {
22
+ try {
23
+ const bytes = readFileSync(path, 'utf8').trim()
24
+ if (!/^\d+$/.test(bytes)) return null
25
+ const value = Number(bytes)
26
+ return Number.isSafeInteger(value) ? Math.round(value / 1048576) : null
27
+ } catch {
28
+ return null
29
+ }
30
+ }
31
+
32
+ const claimedCounts = new Map<number, string>()
33
+ const reportedCounts = new Set<number>()
34
+
35
+ // memory.events is CONTAINER-wide: a concurrent unrelated OOM may increase
36
+ // either counter during this command. A limit event plus kill is evidence of
37
+ // container pressure, not proof that the observed child was the victim.
38
+ export function toolOomNote(
39
+ before: OomEventCounts | null,
40
+ exitCode: number,
41
+ tool: string,
42
+ options: { read?: OomEventsReader; limitMiB?: () => number | null; log?: (line: string) => void } = {},
43
+ ): string | null {
44
+ if (exitCode !== 137 || before === null) return null
45
+ const after = (options.read ?? readOomEvents)()
46
+ if (after === null || after.oomKill <= before.oomKill) return null
47
+ // Only a single new event can be claimed unambiguously. A sampler may
48
+ // already have logged it before this command's stdout finished draining.
49
+ if (after.oomKill === before.oomKill + 1) claimedCounts.set(after.oomKill, tool)
50
+ const limit = (options.limitMiB ?? readMemoryLimitMiB)()
51
+ const reachedLimit = after.oom > before.oom
52
+ const detail = reachedLimit
53
+ ? `the container reached its memory limit${limit === null ? '' : ` (${limit} MiB)`}`
54
+ : `without a container memory-limit event${limit === null ? '' : ` (configured limit ${limit} MiB)`}`
55
+ if (!reportedCounts.has(after.oomKill)) {
56
+ options.log?.(`[tool] ${tool} subprocess SIGKILL; OOM kill observed while command ran; ${detail}`)
57
+ }
58
+ return `An OOM kill was observed while this command ran (exit 137); ${detail}. Reduce parallelism or memory use before retrying.`
59
+ }
60
+
61
+ export function unclaimedOomEvents(
62
+ previous: OomEventCounts,
63
+ current: OomEventCounts,
64
+ ): { from: number; to: number; claimed: string[]; reachedLimit: boolean } | null {
65
+ if (current.oomKill <= previous.oomKill) return null
66
+ const claimed: string[] = []
67
+ let unclaimed = false
68
+ for (let count = previous.oomKill + 1; count <= current.oomKill; count++) {
69
+ const tool = claimedCounts.get(count)
70
+ if (tool === undefined) {
71
+ unclaimed = true
72
+ reportedCounts.add(count)
73
+ } else {
74
+ claimed.push(tool)
75
+ }
76
+ claimedCounts.delete(count)
77
+ }
78
+ // Retain a bounded window for a late tool completion after the sampler's
79
+ // tick. Its model note still arrives, but the operator need not see two lines.
80
+ for (const count of reportedCounts) {
81
+ if (count < current.oomKill - 64) reportedCounts.delete(count)
82
+ }
83
+ return unclaimed
84
+ ? { from: previous.oomKill, to: current.oomKill, claimed, reachedLimit: current.oom > previous.oom }
85
+ : null
86
+ }
@@ -8,6 +8,7 @@ export type RestartOptions = {
8
8
  forceBuild?: boolean
9
9
  streamOutput?: boolean
10
10
  onWarning?: (warning: string) => void
11
+ skipMemoryOversubscriptionCheck?: boolean
11
12
  cliEntry?: string
12
13
  reuseCurrentHostDaemon?: boolean
13
14
  currentHostDaemon?: CurrentHostDaemon
@@ -53,6 +54,7 @@ async function restartWithLease(
53
54
  forceBuild: options.forceBuild,
54
55
  streamOutput: options.streamOutput,
55
56
  onWarning: options.onWarning,
57
+ skipMemoryOversubscriptionCheck: options.skipMemoryOversubscriptionCheck,
56
58
  cliEntry: options.cliEntry,
57
59
  reuseCurrentHostDaemon: options.reuseCurrentHostDaemon,
58
60
  currentHostDaemon: options.currentHostDaemon,
@@ -536,12 +536,19 @@ export async function cleanupRunCorpse(
536
536
  // `docker run --name <same>` to succeed — Docker's name-reservation table
537
537
  // drains independently. waitForRemoval is the fast path; the retry on
538
538
  // `docker run` (see isContainerNameConflict) is the safety net.
539
+ //
540
+ // The budget is generous because a draining removal is normal, not stuck:
541
+ // OrbStack has been observed taking ~40s to destroy agents with large
542
+ // writable layers. Timing out early makes `compose restart` report a healthy
543
+ // drain as stuck and tell the operator to restart Docker for nothing.
544
+ export const CONTAINER_REMOVAL_TIMEOUT_MS = 120_000
545
+
539
546
  export async function waitForRemoval(
540
547
  exec: DockerExec,
541
548
  name: string,
542
549
  options: { timeoutMs?: number; intervalMs?: number } = {},
543
550
  ): Promise<boolean> {
544
- const timeoutMs = options.timeoutMs ?? 10_000
551
+ const timeoutMs = options.timeoutMs ?? CONTAINER_REMOVAL_TIMEOUT_MS
545
552
  const intervalMs = options.intervalMs ?? 100
546
553
  const deadline = Date.now() + timeoutMs
547
554
  while (Date.now() < deadline) {
@@ -43,6 +43,7 @@ import { hostLocaleIsCjk } from '@/shared/host-locale'
43
43
  import { type AgentOperationLease, type WithAgentOperationLock, withAgentOperationLock } from './agent-operation-lock'
44
44
  import { acquireManagedBuildCache, type ManagedBuildCacheLease } from './build-cache'
45
45
  import { COMPOSE_PROJECT } from './compose-project'
46
+ import { inspectContainerExit } from './exit-reason'
46
47
  import { archiveContainerLogs, dockerLogsUnavailableWarning, type DockerLogArchiver } from './log-archive'
47
48
  import { formatMemorySize, readDockerTotalMemory, resolveMemoryLimit } from './memory-limit'
48
49
  import {
@@ -157,6 +158,8 @@ export type StartOptions = {
157
158
  // disables process output so child progress and warnings cannot corrupt it.
158
159
  streamOutput?: boolean
159
160
  onWarning?: (warning: string) => void
161
+ // Compose runs one fleet-wide check after every agent settled instead.
162
+ skipMemoryOversubscriptionCheck?: boolean
160
163
  exec?: DockerExec
161
164
  // Test seam: allows tests to inject a deterministic port allocator. In
162
165
  // production we go through the real kernel via `findFreePort`.
@@ -272,6 +275,7 @@ async function runStart({
272
275
  forceBuild = false,
273
276
  streamOutput = true,
274
277
  onWarning,
278
+ skipMemoryOversubscriptionCheck = false,
275
279
  exec = defaultDockerExec,
276
280
  allocatePort = findFreePort,
277
281
  cliEntry,
@@ -296,21 +300,27 @@ async function runStart({
296
300
  const containerName = containerNameFromCwd(cwd)
297
301
  const imageTagValue = imageTagFromCwd(cwd)
298
302
  const archiveBeforeRemove = async (containerId: string) => {
303
+ const exit = await inspectContainerExit(exec, containerId)
299
304
  const archive = await archiveLogs({
300
305
  agentDir: cwd,
301
306
  containerId,
302
307
  retentionDays: (await loadTypeclawConfig(cwd)).logs.retentionDays,
303
308
  })
304
- if (archive.ok) return { ok: true as const }
305
- if (archive.kind === 'failed') return { ok: false as const, reason: archive.reason }
309
+ if (!archive.ok && archive.kind === 'failed') return { ok: false as const, reason: archive.reason }
306
310
  // Dispatch on the callback, NOT on streamOutput. `streamOutput` defaults
307
311
  // to true, and standalone start/restart both stream Docker build output
308
312
  // AND pass a collector so they can render warnings after their spinner
309
313
  // settles; keying on streamOutput writes under the live spinner and
310
314
  // leaves that collector empty. stderr stays the no-callback fallback.
311
- const warning = dockerLogsUnavailableWarning(containerId)
312
- if (onWarning !== undefined) onWarning(warning)
313
- else if (streamOutput) process.stderr.write(`${warning}\n`)
315
+ if (!archive.ok) {
316
+ const warning = dockerLogsUnavailableWarning(containerId)
317
+ if (onWarning !== undefined) onWarning(warning)
318
+ else if (streamOutput) process.stderr.write(`${warning}\n`)
319
+ }
320
+ if (exit?.kind === 'stopped' && exit.reason !== null) {
321
+ if (onWarning !== undefined) onWarning(exit.reason)
322
+ else if (streamOutput) process.stderr.write(`typeclaw: ${exit.reason}\n`)
323
+ }
314
324
  return { ok: true as const }
315
325
  }
316
326
 
@@ -601,14 +611,16 @@ async function runStart({
601
611
  totalMemoryBytes,
602
612
  })
603
613
 
604
- await warnOnMemoryOversubscription({
605
- exec,
606
- containerName,
607
- memoryLimitBytes: plan.memoryLimitBytes,
608
- totalMemoryBytes,
609
- onWarning,
610
- streamOutput,
611
- })
614
+ if (!skipMemoryOversubscriptionCheck) {
615
+ await warnOnMemoryOversubscription({
616
+ exec,
617
+ containerName,
618
+ memoryLimitBytes: plan.memoryLimitBytes,
619
+ totalMemoryBytes,
620
+ onWarning,
621
+ streamOutput,
622
+ })
623
+ }
612
624
 
613
625
  let built = false
614
626
  if (plan.needsBuild) {
@@ -2,9 +2,11 @@ import { DEFAULT_LOG_RETENTION_DAYS, loadConfigSync } from '@/config'
2
2
  import { isDaemonReachable, send as sendToDaemon } from '@/hostd/client'
3
3
 
4
4
  import { type AgentOperationLease, type WithAgentOperationLock, withAgentOperationLock } from './agent-operation-lock'
5
+ import { inspectContainerExit } from './exit-reason'
5
6
  import { archiveContainerLogs, dockerLogsUnavailableWarning, type DockerLogArchiver } from './log-archive'
6
7
  import {
7
8
  classifyRmStderr,
9
+ CONTAINER_REMOVAL_TIMEOUT_MS,
8
10
  containerNameFromCwd,
9
11
  defaultDockerExec,
10
12
  type DockerExec,
@@ -78,6 +80,7 @@ async function runStop({
78
80
  }
79
81
  }
80
82
  const { containerId, running } = parsed
83
+ const staleExit = running ? null : await inspectContainerExit(exec, containerId)
81
84
 
82
85
  // Only call `docker stop` when the container is actually running. A stopped
83
86
  // corpse from a prior crash is left around by design (no `--rm`), and
@@ -100,6 +103,7 @@ async function runStop({
100
103
  }
101
104
  onWarning?.(dockerLogsUnavailableWarning(containerId))
102
105
  }
106
+ if (staleExit?.kind === 'stopped' && staleExit.reason !== null) onWarning?.(staleExit.reason)
103
107
 
104
108
  // Containers run without `--rm`, so `docker stop` only stops them. Remove
105
109
  // the inspected ID without force after archival. If that same ID resumed,
@@ -134,12 +138,12 @@ async function runStop({
134
138
  return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
135
139
  }
136
140
  if (kind === 'in-progress' && !(await waitForRemoval(exec, containerName))) {
137
- const reason = `Container ${containerName} is still being removed by docker after 10s.`
141
+ const reason = `Container ${containerName} is still being removed by docker after ${CONTAINER_REMOVAL_TIMEOUT_MS / 1000}s.`
138
142
  const diagnosis = await describeUnremovableContainer(exec, containerId)
139
143
  return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
140
144
  }
141
145
  } else if (!(await waitForRemoval(exec, containerName))) {
142
- const reason = `Container ${containerName} is still being removed by docker after 10s.`
146
+ const reason = `Container ${containerName} is still being removed by docker after ${CONTAINER_REMOVAL_TIMEOUT_MS / 1000}s.`
143
147
  const diagnosis = await describeUnremovableContainer(exec, containerId)
144
148
  return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
145
149
  }
@@ -0,0 +1,32 @@
1
+ // Under a 6 GiB memcg, a ~230 MiB tool process contributes only ~38 points
2
+ // from RSS, versus ~230 for a 1.4 GiB agent runtime. +600 puts even that
3
+ // tool ahead of the runtime without using the maximum (+1000) and making
4
+ // tiny helper processes unconditional first victims. Children inherit the
5
+ // score across fork/exec. The shell writes its OWN score before any command
6
+ // can fork; writing /proc/<pid> from the parent after spawn would race.
7
+ const OOM_SCORE_ADJ = 600
8
+ const SCORE_PATH = '/proc/self/oom_score_adj'
9
+ const PRIORITY_SCRIPT = `if [ -w ${SCORE_PATH} ]; then printf '%s\\n' ${OOM_SCORE_ADJ} 2>/dev/null >${SCORE_PATH} || :; fi`
10
+
11
+ function priorityScript(scorePath: string): string {
12
+ // The default is pre-rendered; alternate paths are solely a test seam for
13
+ // an absent or write-failing procfs, never derived from a tool command.
14
+ return scorePath === SCORE_PATH
15
+ ? PRIORITY_SCRIPT
16
+ : `if [ -w '${scorePath}' ]; then printf '%s\\n' ${OOM_SCORE_ADJ} 2>/dev/null >'${scorePath}' || :; fi`
17
+ }
18
+
19
+ // Shell built-in printf runs in this process. An absent/unwritable procfs
20
+ // (including non-container dev environments) fails open without changing the
21
+ // command's output, exit status, environment, or timeout behavior.
22
+ export function withToolOomPriorityCommand(command: string, scorePath = SCORE_PATH): string {
23
+ return process.platform === 'linux' ? `${priorityScript(scorePath)}\n${command}` : command
24
+ }
25
+
26
+ export function withToolOomPriorityArgv(command: string[], scorePath = SCORE_PATH): string[] {
27
+ return process.platform === 'linux'
28
+ ? ['/bin/sh', '-c', `${priorityScript(scorePath)}\nexec "$@"`, 'tool-oom-priority', ...command]
29
+ : command
30
+ }
31
+
32
+ export { OOM_SCORE_ADJ }
@@ -4,6 +4,8 @@ import { promptWithFallback, resolveFallbackChain } from '@/agent/model-fallback
4
4
  import type { SessionOrigin } from '@/agent/session-origin'
5
5
  import { getConfig } from '@/config'
6
6
  import type { ModelRef } from '@/config/providers'
7
+ import { readOomEvents, toolOomNote } from '@/container/oom-events'
8
+ import { withToolOomPriorityArgv } from '@/container/tool-oom-priority'
7
9
  import type { HookBus } from '@/plugin'
8
10
  import type { Stream, Unsubscribe } from '@/stream'
9
11
 
@@ -143,7 +145,7 @@ export function createCronConsumer({
143
145
  if (job.kind === 'prompt') {
144
146
  if (!(await runPrompt(job, createSessionForCron, stream, logger))) outcome = 'failed'
145
147
  } else if (job.kind === 'exec') {
146
- await runExec(job, cwd)
148
+ await runExec(job, cwd, logger)
147
149
  } else {
148
150
  if (invokeHandler === undefined) {
149
151
  throw new Error(
@@ -347,7 +349,7 @@ function describe(err: unknown): string {
347
349
  return err instanceof Error ? err.message : String(err)
348
350
  }
349
351
 
350
- async function runExec(job: ExecJob, cwd: string): Promise<void> {
352
+ async function runExec(job: ExecJob, cwd: string, logger: CronConsumerLogger): Promise<void> {
351
353
  const [cmd, ...args] = job.command
352
354
  if (!cmd) throw new Error(`exec job ${job.id}: empty command`)
353
355
  // Inject TYPECLAW_PARENT_ORIGIN_JSON so a child that proxies into the
@@ -363,8 +365,9 @@ async function runExec(job: ExecJob, cwd: string): Promise<void> {
363
365
  ...(job.scheduledByRole !== undefined ? { scheduledByRole: job.scheduledByRole } : {}),
364
366
  ...(job.scheduledByOrigin !== undefined ? { scheduledByOrigin: job.scheduledByOrigin } : {}),
365
367
  }
368
+ const oomBefore = readOomEvents()
366
369
  const proc = Bun.spawn({
367
- cmd: [cmd, ...args],
370
+ cmd: withToolOomPriorityArgv([cmd, ...args]),
368
371
  cwd,
369
372
  stdout: 'ignore',
370
373
  stderr: 'pipe',
@@ -375,8 +378,11 @@ async function runExec(job: ExecJob, cwd: string): Promise<void> {
375
378
  })
376
379
  const stderrText = new Response(proc.stderr).text()
377
380
  const [code, stderr] = await Promise.all([proc.exited, stderrText])
381
+ const note = toolOomNote(oomBefore, code, 'cron exec', { log: logger.warn })
378
382
  if (code !== 0) {
379
- throw new Error(`exec job ${job.id} exited with code ${code}: ${stderr.trim() || 'no stderr'}`)
383
+ throw new Error(
384
+ `exec job ${job.id} exited with code ${code}: ${stderr.trim() || 'no stderr'}${note === null ? '' : `\n${note}`}`,
385
+ )
380
386
  }
381
387
  }
382
388
 
@@ -3,7 +3,7 @@ import type { Reloadable, ReloadResult } from '@/reload'
3
3
 
4
4
  import { loadCron } from './index'
5
5
  import type { JobDiff, Scheduler } from './scheduler'
6
- import type { CronJob } from './schema'
6
+ import type { CronJob, HandlerJob, ParsedCronJob } from './schema'
7
7
 
8
8
  export type CreateCronReloadableOptions = {
9
9
  cwd: string
@@ -74,7 +74,28 @@ async function doReload({
74
74
  return { scope: 'cron', ok: false, reason: `apply failed (schedule unchanged): ${message}` }
75
75
  }
76
76
 
77
- return { scope: 'cron', ok: true, summary: formatSummary(diff, nextJobs.length), details: diff }
77
+ return { scope: 'cron', ok: true, summary: formatSummary(diff, nextJobs.length), details: plainDiff(diff) }
78
+ }
79
+
80
+ // `details` lands in the reload tool result, which pi structuredClones on every
81
+ // later request in that session. A plugin handler job's function would make
82
+ // that clone throw DataCloneError and wedge the session, so details carry data only.
83
+ type CronJobData = ParsedCronJob | Omit<HandlerJob, 'handler'>
84
+ type CronReloadDetails = Record<keyof JobDiff, CronJobData[]>
85
+
86
+ function plainDiff(diff: JobDiff): CronReloadDetails {
87
+ const plain = (jobs: CronJob[]): CronJobData[] =>
88
+ jobs.map((job) => {
89
+ if (job.kind !== 'handler') return job
90
+ const { handler: _handler, ...data } = job
91
+ return data
92
+ })
93
+ return {
94
+ added: plain(diff.added),
95
+ removed: plain(diff.removed),
96
+ updated: plain(diff.updated),
97
+ unchanged: plain(diff.unchanged),
98
+ }
78
99
  }
79
100
 
80
101
  function formatSummary(diff: JobDiff, total: number): string {
@@ -17,6 +17,7 @@ import {
17
17
  type DockerExec,
18
18
  } from '@/container'
19
19
  import { homeRoot, isDaemonReachable, send } from '@/hostd'
20
+ import { hostBunVersionWarning } from '@/hostd/version'
20
21
  import { resolveBaseImageVersion } from '@/init/cli-version'
21
22
  import { buildDockerfile, DOCKERFILE } from '@/init/dockerfile'
22
23
  import { detectMissingDeps } from '@/init/ensure-deps'
@@ -99,7 +100,7 @@ function dockerDaemonFix(reason: DockerUnavailableReason): string {
99
100
  }
100
101
  }
101
102
 
102
- function bunRuntime(): DoctorCheck {
103
+ export function bunRuntime(bunVersion: string | undefined = process.versions.bun): DoctorCheck {
103
104
  return {
104
105
  name: 'runtime.bun-available',
105
106
  category: 'runtime',
@@ -113,7 +114,18 @@ function bunRuntime(): DoctorCheck {
113
114
  fix: { description: 'Install Bun (https://bun.sh) and ensure the typeclaw CLI runs under it.' },
114
115
  }
115
116
  }
116
- return { status: 'ok', message: `Bun ${process.versions.bun ?? 'present'}` }
117
+ const outdated = hostBunVersionWarning(bunVersion)
118
+ if (outdated) {
119
+ return {
120
+ status: 'warning',
121
+ message: outdated,
122
+ fix: {
123
+ description:
124
+ 'Run `bun upgrade`. The host daemon respawns on the new runtime at the next `typeclaw start` or `restart`.',
125
+ },
126
+ }
127
+ }
128
+ return { status: 'ok', message: `Bun ${bunVersion ?? 'present'}` }
117
129
  },
118
130
  }
119
131
  }