typeclaw 0.48.13 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/scripts/dump-system-prompt.ts +52 -110
- package/src/agent/index.ts +144 -72
- package/src/agent/live-sessions.ts +2 -2
- package/src/agent/live-subagents.ts +1 -0
- package/src/agent/plugin-tools.ts +53 -11
- package/src/agent/prompt-cache-key.ts +42 -0
- package/src/agent/provider-error.ts +179 -10
- package/src/agent/restart-handoff/index.ts +7 -14
- package/src/agent/reviewer-activity.ts +68 -0
- package/src/agent/session-origin.ts +33 -0
- package/src/agent/subagents.ts +13 -1
- package/src/agent/system-prompt.ts +68 -202
- package/src/agent/todo/continuation-policy.ts +12 -4
- package/src/agent/tool-file-safety.ts +113 -14
- package/src/agent/tool-not-found-nudge.ts +1 -1
- package/src/agent/tools/channel-history-render.ts +2 -1
- package/src/agent/tools/channel-reply.ts +23 -5
- package/src/agent/tools/channel-send.ts +49 -7
- package/src/agent/tools/curl-impersonate.ts +11 -2
- package/src/agent/tools/post-github-review.ts +77 -15
- package/src/agent/tools/skip-response.ts +2 -2
- package/src/agent/tools/spawn-subagent.ts +163 -12
- package/src/agent/tools/subagent-output.ts +8 -0
- package/src/bundled-plugins/backup/README.md +10 -9
- package/src/bundled-plugins/backup/runner.ts +140 -58
- package/src/bundled-plugins/backup/subagents.ts +58 -26
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +34 -2
- package/src/bundled-plugins/github-cli-auth/granted-push.ts +135 -0
- package/src/bundled-plugins/github-cli-auth/index.ts +83 -11
- package/src/bundled-plugins/github-cli-auth/review-account.ts +23 -0
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +9 -2
- package/src/bundled-plugins/github-cli-auth/review-recorder.ts +45 -7
- package/src/bundled-plugins/operator/operator.ts +15 -4
- package/src/bundled-plugins/reviewer/skills/code-review.ts +1 -1
- package/src/bundled-plugins/security/index.ts +50 -2
- package/src/bundled-plugins/security/policies/git-exfil.ts +24 -12
- package/src/channels/adapters/discord-bot-reactions.ts +18 -0
- package/src/channels/adapters/discord-bot.ts +122 -67
- package/src/channels/adapters/discord-reactions.ts +18 -0
- package/src/channels/adapters/discord.ts +114 -8
- package/src/channels/adapters/github/inbound.ts +103 -8
- package/src/channels/adapters/github/index.ts +251 -19
- package/src/channels/adapters/github/last-alive.ts +27 -0
- package/src/channels/adapters/github/outbound.ts +54 -18
- package/src/channels/adapters/github/pending-review-store.ts +141 -0
- package/src/channels/adapters/github/permission-guidance.ts +1 -0
- package/src/channels/adapters/github/reactions.ts +100 -0
- package/src/channels/adapters/github/reconcile-cooldown-store.ts +197 -67
- package/src/channels/adapters/github/reconcile-open-prs.ts +418 -33
- package/src/channels/adapters/github/recovery.ts +106 -0
- package/src/channels/adapters/github/review-check.ts +510 -0
- package/src/channels/adapters/github/review-submitter.ts +34 -2
- package/src/channels/adapters/instagram.ts +32 -14
- package/src/channels/adapters/kakaotalk.ts +22 -7
- package/src/channels/adapters/line.ts +18 -3
- package/src/channels/adapters/outbound-account.ts +20 -0
- package/src/channels/adapters/own-reactions.ts +66 -0
- package/src/channels/adapters/recovery-correlation.ts +214 -0
- package/src/channels/adapters/slack-bot-classify.ts +18 -11
- package/src/channels/adapters/slack-bot-dedupe.ts +111 -28
- package/src/channels/adapters/slack-bot-reactions.ts +18 -0
- package/src/channels/adapters/slack-bot.ts +125 -57
- package/src/channels/adapters/slack-classify.ts +25 -4
- package/src/channels/adapters/slack-inbound-revision.ts +96 -0
- package/src/channels/adapters/slack-reactions.ts +18 -0
- package/src/channels/adapters/slack-thread-ts.ts +6 -0
- package/src/channels/adapters/slack.ts +48 -3
- package/src/channels/adapters/teams.ts +14 -4
- package/src/channels/adapters/telegram-bot.ts +56 -10
- package/src/channels/adapters/webex-bot.ts +12 -2
- package/src/channels/adapters/webex.ts +12 -2
- package/src/channels/background-handoff.ts +434 -0
- package/src/channels/background-obligations.ts +893 -0
- package/src/channels/continuity-types.ts +271 -0
- package/src/channels/github-review-check.ts +598 -0
- package/src/channels/github-review-turn-ledger.ts +102 -28
- package/src/channels/github-review-verdict-coordinator.ts +72 -12
- package/src/channels/github-token-bridge.ts +11 -5
- package/src/channels/inbound-journal.ts +1738 -0
- package/src/channels/index.ts +6 -1
- package/src/channels/manager.ts +17 -7
- package/src/channels/recovery-dispatcher.ts +264 -0
- package/src/channels/recovery-notice.ts +94 -0
- package/src/channels/recovery-outbox.ts +459 -0
- package/src/channels/router.ts +2539 -615
- package/src/channels/schema.ts +8 -1
- package/src/channels/subagent-completion-bridge.ts +17 -14
- package/src/channels/types.ts +121 -16
- package/src/channels/waiting-reactions.ts +664 -0
- package/src/cli/compose.ts +7 -4
- package/src/cli/hostd.ts +49 -3
- package/src/cli/inspect.ts +11 -14
- package/src/coding-agent/broker-body.ts +38 -0
- package/src/coding-agent/broker.ts +290 -0
- package/src/coding-agent/claude.ts +95 -0
- package/src/coding-agent/codex.ts +104 -0
- package/src/coding-agent/index.ts +506 -0
- package/src/coding-agent/omo-state.ts +69 -0
- package/src/coding-agent/omo-stream.ts +98 -0
- package/src/coding-agent/omo.ts +192 -0
- package/src/coding-agent/runner.ts +263 -0
- package/src/coding-agent/tool.ts +62 -0
- package/src/coding-agent/worktree.ts +355 -0
- package/src/compose/restart.ts +10 -3
- package/src/compose/start.ts +20 -3
- package/src/config/config.ts +10 -3
- package/src/config/providers.ts +12 -9
- package/src/container/exit-reason.ts +44 -0
- package/src/container/index.ts +5 -0
- package/src/container/log-archive.ts +51 -13
- package/src/container/memory-limit.ts +1 -1
- package/src/container/memory-oversubscription.ts +36 -6
- package/src/container/oom-events.ts +86 -0
- package/src/container/restart.ts +2 -0
- package/src/container/shared.ts +8 -1
- package/src/container/start.ts +25 -13
- package/src/container/stop.ts +6 -2
- package/src/container/tool-oom-priority.ts +32 -0
- package/src/cron/consumer.ts +10 -4
- package/src/cron/reloadable.ts +23 -2
- package/src/doctor/checks.ts +14 -2
- package/src/hostd/daemon.ts +20 -10
- package/src/hostd/spawn.ts +106 -5
- package/src/hostd/version.ts +20 -0
- package/src/hostd/watchdog.ts +241 -0
- package/src/init/dockerfile.ts +57 -592
- package/src/init/index.ts +2 -3
- package/src/mcp/client.ts +3 -1
- package/src/operations/classify-tool-outcome.ts +45 -1
- package/src/operations/incidents.ts +10 -5
- package/src/plugin/core-tool-names.ts +1 -1
- package/src/plugin/define.ts +1 -0
- package/src/plugin/index.ts +1 -0
- package/src/plugin/types.ts +6 -1
- package/src/run/background-handoff-boot.ts +86 -0
- package/src/run/index.ts +83 -83
- package/src/run/resource-sample.ts +19 -0
- package/src/sandbox/build.ts +4 -0
- package/src/sandbox/coding-agent.ts +214 -0
- package/src/sandbox/policy.ts +1 -0
- package/src/sandbox/runtime-private.ts +4 -1
- package/src/secrets/coding-agent-auth.ts +97 -0
- package/src/secrets/index.ts +0 -16
- package/src/server/command-runner.ts +6 -2
- package/src/server/index.ts +101 -40
- package/src/skills/typeclaw-channel-github/SKILL.md +7 -5
- package/src/skills/typeclaw-claude-code/SKILL.md +25 -260
- package/src/skills/typeclaw-claude-code/references/auth-flow.md +7 -23
- package/src/skills/typeclaw-claude-code/references/headless-sessions.md +11 -0
- package/src/skills/typeclaw-claude-code/references/results-and-worktrees.md +13 -0
- package/src/skills/typeclaw-codex-cli/SKILL.md +35 -270
- package/src/skills/typeclaw-codex-cli/references/auth-flow.md +13 -20
- package/src/skills/typeclaw-codex-cli/references/headless-sessions.md +26 -0
- package/src/skills/typeclaw-codex-cli/references/results-and-worktrees.md +28 -0
- package/src/skills/typeclaw-config/SKILL.md +21 -21
- package/src/skills/typeclaw-config/references/dockerfile.md +14 -13
- package/src/skills/typeclaw-cron/SKILL.md +1 -1
- package/src/skills/typeclaw-github-contributing/SKILL.md +2 -2
- package/src/skills/typeclaw-memory/SKILL.md +3 -3
- package/src/skills/typeclaw-omo-cli/SKILL.md +52 -0
- package/src/skills/typeclaw-omo-cli/references/auth-flow.md +11 -0
- package/src/skills/typeclaw-omo-cli/references/headless-sessions.md +11 -0
- package/src/skills/typeclaw-omo-cli/references/results-and-worktrees.md +23 -0
- package/src/skills/typeclaw-plugins/SKILL.md +15 -15
- package/src/skills/typeclaw-skills/SKILL.md +19 -2
- package/src/skills/typeclaw-troubleshooting/SKILL.md +15 -2
- package/typeclaw.schema.json +12 -1
- package/src/secrets/claude-credentials-json.ts +0 -129
- package/src/secrets/codex-auth-json.ts +0 -102
- package/src/secrets/export-claude-credentials-file.ts +0 -270
- package/src/secrets/export-codex-auth-file.ts +0 -250
- package/src/skills/typeclaw-claude-code/references/stop-hook.md +0 -101
- package/src/skills/typeclaw-claude-code/references/tmux-driving.md +0 -243
- package/src/skills/typeclaw-codex-cli/references/stop-hook.md +0 -92
- package/src/skills/typeclaw-codex-cli/references/tmux-driving.md +0 -239
|
@@ -4,7 +4,7 @@ import { join } from 'node:path'
|
|
|
4
4
|
|
|
5
5
|
import { DEFAULT_LOG_RETENTION_DAYS, MAX_LOG_RETENTION_DAYS } from '@/config'
|
|
6
6
|
|
|
7
|
-
import { resolveDockerBinary, sanitizeDockerStderr } from './shared'
|
|
7
|
+
import { defaultDockerExec, resolveDockerBinary, sanitizeDockerStderr, type DockerExecResult } from './shared'
|
|
8
8
|
|
|
9
9
|
export type DockerLogArchiveResult =
|
|
10
10
|
| { ok: true; status: 'archived'; path: string }
|
|
@@ -49,6 +49,7 @@ export type StreamCaptureResult = {
|
|
|
49
49
|
|
|
50
50
|
export type DockerLogArchiveRuntime = {
|
|
51
51
|
capture?: (input: StreamCaptureInput) => Promise<StreamCaptureResult>
|
|
52
|
+
inspectLogConfig?: (args: string[], options: { cwd: string; signal: AbortSignal }) => Promise<DockerExecResult>
|
|
52
53
|
nonce?: () => string
|
|
53
54
|
now?: () => Date
|
|
54
55
|
resolveDockerBinary?: () => string | null
|
|
@@ -74,6 +75,43 @@ const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/
|
|
|
74
75
|
const NONCE_PATTERN = /^[a-f0-9]{32}$/
|
|
75
76
|
const ARCHIVE_FILENAME_PATTERN = /^[a-f0-9]{64}-(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z-[a-f0-9]{32}\.log$/
|
|
76
77
|
const PARTIAL_FILENAME_PATTERN = /^\.[a-f0-9]{64}-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z-[a-f0-9]{32}\.partial$/
|
|
78
|
+
const DOCKER_LOG_INSPECT_TIMEOUT_MS = 10_000
|
|
79
|
+
|
|
80
|
+
async function archiveByteLimit(
|
|
81
|
+
agentDir: string,
|
|
82
|
+
containerId: string,
|
|
83
|
+
inspect: NonNullable<DockerLogArchiveRuntime['inspectLogConfig']>,
|
|
84
|
+
): Promise<number> {
|
|
85
|
+
try {
|
|
86
|
+
const result = await inspect(['inspect', '--format', '{{json .HostConfig.LogConfig}}', containerId], {
|
|
87
|
+
cwd: agentDir,
|
|
88
|
+
signal: AbortSignal.timeout(DOCKER_LOG_INSPECT_TIMEOUT_MS),
|
|
89
|
+
})
|
|
90
|
+
if (result.exitCode !== 0) return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
91
|
+
const config: unknown = JSON.parse(result.stdout)
|
|
92
|
+
if (typeof config !== 'object' || config === null || !('Type' in config) || config.Type !== 'json-file')
|
|
93
|
+
return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
94
|
+
if (!('Config' in config) || typeof config.Config !== 'object' || config.Config === null)
|
|
95
|
+
return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
96
|
+
const opts = config.Config
|
|
97
|
+
if (!('max-size' in opts) || typeof opts['max-size'] !== 'string') return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
98
|
+
const maxFiles = 'max-file' in opts ? opts['max-file'] : '1'
|
|
99
|
+
// Docker parses max-file with strconv.Atoi, so "05" is 5; only positive counts are bounded.
|
|
100
|
+
if (typeof maxFiles !== 'string' || !/^\+?\d+$/.test(maxFiles) || Number(maxFiles) < 1)
|
|
101
|
+
return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
102
|
+
// Docker json-file uses go-units FromHumanSize: decimal powers even for "MiB".
|
|
103
|
+
const sizeMatch = /^(\+?(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?)(?: ?(b|[kmgtp](?:i?b)?))?$/i.exec(opts['max-size'])
|
|
104
|
+
if (!sizeMatch) return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
105
|
+
const prefix = sizeMatch[2]?.toLowerCase()[0]
|
|
106
|
+
const multiplier = prefix && prefix !== 'b' ? 1000 ** ('kmgtp'.indexOf(prefix) + 1) : 1
|
|
107
|
+
const size = Math.trunc(Number(sizeMatch[1]) * multiplier)
|
|
108
|
+
const capacity = size * Number(maxFiles)
|
|
109
|
+
if (size <= 0 || !Number.isSafeInteger(capacity)) return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
110
|
+
return Math.min(MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES, Math.max(MAX_DOCKER_LOG_ARCHIVE_BYTES, capacity))
|
|
111
|
+
} catch {
|
|
112
|
+
return MAX_DOCKER_LOG_ARCHIVE_BYTES
|
|
113
|
+
}
|
|
114
|
+
}
|
|
77
115
|
|
|
78
116
|
export async function archiveContainerLogs(
|
|
79
117
|
{ agentDir, containerId, retentionDays = DEFAULT_LOG_RETENTION_DAYS }: Parameters<DockerLogArchiver>[0],
|
|
@@ -106,18 +144,21 @@ export async function archiveContainerLogs(
|
|
|
106
144
|
if (binary === null) throw new Error('Docker binary could not be resolved for log archival')
|
|
107
145
|
|
|
108
146
|
const capture = runtime.capture ?? streamProcessOutput
|
|
147
|
+
const maxBytes = await archiveByteLimit(agentDir, containerId, runtime.inspectLogConfig ?? defaultDockerExec)
|
|
109
148
|
const captured = await capture({
|
|
110
149
|
args: ['logs', '--timestamps', containerId],
|
|
111
150
|
binary,
|
|
112
151
|
cwd: agentDir,
|
|
113
|
-
maxBytes
|
|
152
|
+
maxBytes,
|
|
114
153
|
output,
|
|
115
154
|
timeoutMs: DOCKER_LOG_ARCHIVE_TIMEOUT_MS,
|
|
116
155
|
})
|
|
117
156
|
if (captured.terminationBoundExceeded) throw new Error('docker logs did not settle after SIGKILL')
|
|
118
157
|
if (captured.timedOut) throw new Error(`docker logs timed out after ${DOCKER_LOG_ARCHIVE_TIMEOUT_MS}ms`)
|
|
119
158
|
if (captured.overflowed) {
|
|
120
|
-
throw new Error(
|
|
159
|
+
throw new Error(
|
|
160
|
+
`docker logs exceeded the ${maxBytes}-byte archive limit. Container ${containerId} was preserved; export both log streams with docker logs --timestamps ${containerId} > private-log-file 2>&1 (choose a private destination); confirm the export succeeded before removing it with docker rm ${containerId} and retrying.`,
|
|
161
|
+
)
|
|
121
162
|
}
|
|
122
163
|
if (captured.exitCode !== 0) {
|
|
123
164
|
const detail = sanitizeDockerStderr(captured.stderrExcerpt)
|
|
@@ -128,13 +169,16 @@ export async function archiveContainerLogs(
|
|
|
128
169
|
|
|
129
170
|
await output.chmod(0o600)
|
|
130
171
|
const incomingSize = (await output.stat()).size
|
|
131
|
-
|
|
172
|
+
if (incomingSize > MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES) {
|
|
173
|
+
throw new Error('Docker log archive exceeds total archive capacity')
|
|
174
|
+
}
|
|
132
175
|
await output.close()
|
|
133
176
|
output = null
|
|
134
177
|
|
|
135
178
|
await rename(temporaryPath, allocation.finalPath)
|
|
136
179
|
temporaryPath = null
|
|
137
180
|
await chmod(allocation.finalPath, 0o600)
|
|
181
|
+
await pruneToCapacity(logsDir, allocation.finalPath)
|
|
138
182
|
return { ok: true, status: 'archived', path: allocation.finalPath }
|
|
139
183
|
} catch (error) {
|
|
140
184
|
const cleanupErrors = await cleanupTemp(output, temporaryPath)
|
|
@@ -334,10 +378,7 @@ async function pruneByAgeAndRemoveStalePartials(logsDir: string, ageCutoff: numb
|
|
|
334
378
|
}
|
|
335
379
|
}
|
|
336
380
|
|
|
337
|
-
async function pruneToCapacity(logsDir: string,
|
|
338
|
-
if (incomingSize > MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES) {
|
|
339
|
-
throw new Error('Docker log archive exceeds total archive capacity')
|
|
340
|
-
}
|
|
381
|
+
async function pruneToCapacity(logsDir: string, publishedPath: string): Promise<void> {
|
|
341
382
|
const snapshots: Array<{ name: string; path: string; size: number; timestamp: number }> = []
|
|
342
383
|
for (const entry of await collectDirectoryEntries(logsDir)) {
|
|
343
384
|
if (!entry.isFile()) continue
|
|
@@ -351,11 +392,8 @@ async function pruneToCapacity(logsDir: string, incomingSize: number): Promise<v
|
|
|
351
392
|
let totalBytes = snapshots.reduce((total, snapshot) => total + snapshot.size, 0)
|
|
352
393
|
let count = snapshots.length
|
|
353
394
|
for (const snapshot of snapshots) {
|
|
354
|
-
if (
|
|
355
|
-
|
|
356
|
-
totalBytes + incomingSize <= MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES
|
|
357
|
-
)
|
|
358
|
-
break
|
|
395
|
+
if (count <= MAX_DOCKER_LOG_ARCHIVE_SNAPSHOTS && totalBytes <= MAX_DOCKER_LOG_ARCHIVE_TOTAL_BYTES) break
|
|
396
|
+
if (snapshot.path === publishedPath) continue
|
|
359
397
|
await removeRaceSafe(snapshot.path)
|
|
360
398
|
count -= 1
|
|
361
399
|
totalBytes -= snapshot.size
|
|
@@ -14,7 +14,7 @@ export const DEFAULT_CONTAINER_MEMORY_BYTES = 6 * BYTES_PER_GIB
|
|
|
14
14
|
// Memory left to the Docker VM itself (dockerd, containerd, the guest kernel,
|
|
15
15
|
// page cache) when the default does not fit. Without it a single agent sized to
|
|
16
16
|
// the whole machine reproduces the exhaustion this limit exists to prevent.
|
|
17
|
-
const HOST_HEADROOM_BYTES = 2 * BYTES_PER_GIB
|
|
17
|
+
export const HOST_HEADROOM_BYTES = 2 * BYTES_PER_GIB
|
|
18
18
|
|
|
19
19
|
// Below this a container cannot boot the agent at all, so clamping further down
|
|
20
20
|
// would trade a slow host for an agent that never starts.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { COMPOSE_PROJECT } from './compose-project'
|
|
2
|
+
import { HOST_HEADROOM_BYTES, readDockerTotalMemory } from './memory-limit'
|
|
2
3
|
import type { DockerExec } from './shared'
|
|
3
4
|
|
|
4
5
|
// `bytes: null` means the container runs with NO memory limit. That is a
|
|
@@ -33,7 +34,8 @@ export type OversubscriptionWarning = {
|
|
|
33
34
|
// crowded host. The operator gets the arithmetic and decides.
|
|
34
35
|
export function decideMemoryOversubscription(options: {
|
|
35
36
|
running: AgentMemoryClaim[]
|
|
36
|
-
|
|
37
|
+
// Omitted for a fleet-wide check run after every agent is already up.
|
|
38
|
+
incoming?: AgentMemoryClaim
|
|
37
39
|
totalMemoryBytes: number | undefined
|
|
38
40
|
}): OversubscriptionWarning | null {
|
|
39
41
|
const { running, incoming, totalMemoryBytes } = options
|
|
@@ -41,8 +43,10 @@ export function decideMemoryOversubscription(options: {
|
|
|
41
43
|
// A restart re-runs start for a container that is already listed, so drop any
|
|
42
44
|
// running claim under the incoming name before summing. Counting it twice
|
|
43
45
|
// would fire the warning on a plain `typeclaw restart` of a single agent.
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
+
const claims =
|
|
47
|
+
incoming === undefined
|
|
48
|
+
? running
|
|
49
|
+
: [...running.filter((claim) => claim.containerName !== incoming.containerName), incoming]
|
|
46
50
|
const unbounded = claims.filter((claim) => claim.bytes === null)
|
|
47
51
|
const claimedBytes = claims.reduce((sum, claim) => sum + (claim.bytes ?? 0), 0)
|
|
48
52
|
|
|
@@ -63,9 +67,10 @@ export function decideMemoryOversubscription(options: {
|
|
|
63
67
|
|
|
64
68
|
export function formatOversubscriptionWarning(warning: OversubscriptionWarning): string[] {
|
|
65
69
|
const gib = (bytes: number): string => `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GiB`
|
|
70
|
+
const width = warning.claims.reduce((w, claim) => Math.max(w, claim.containerName.length), 0)
|
|
66
71
|
const roster = warning.claims
|
|
67
|
-
.
|
|
68
|
-
.
|
|
72
|
+
.toSorted((a, b) => a.containerName.localeCompare(b.containerName))
|
|
73
|
+
.map((claim) => ` ${claim.containerName.padEnd(width)} ${claim.bytes === null ? 'unlimited' : gib(claim.bytes)}`)
|
|
69
74
|
|
|
70
75
|
const capacity = warning.totalMemoryBytes === null ? null : gib(warning.totalMemoryBytes)
|
|
71
76
|
const names = warning.unbounded.map((claim) => claim.containerName).join(', ')
|
|
@@ -78,11 +83,36 @@ export function formatOversubscriptionWarning(warning: OversubscriptionWarning):
|
|
|
78
83
|
const remedy =
|
|
79
84
|
warning.unbounded.length > 0
|
|
80
85
|
? ['An unbounded agent can exhaust the host on its own. Restart it to apply', 'the current limit.']
|
|
81
|
-
:
|
|
86
|
+
: oversubscriptionRemedy(warning.claimedBytes)
|
|
82
87
|
|
|
83
88
|
return [headline, ...roster, ...remedy]
|
|
84
89
|
}
|
|
85
90
|
|
|
91
|
+
function oversubscriptionRemedy(claimedBytes: number): string[] {
|
|
92
|
+
const targetGib = Math.ceil((claimedBytes + HOST_HEADROOM_BYTES) / (1024 * 1024 * 1024))
|
|
93
|
+
return [
|
|
94
|
+
'If they peak together the host can still exhaust. Either:',
|
|
95
|
+
' • Stop an agent: run `typeclaw stop` in its folder.',
|
|
96
|
+
` • Give Docker at least ${targetGib}GiB of memory, then restart the runtime:`,
|
|
97
|
+
' Docker Desktop: Settings → Resources → Memory limit → Apply & restart',
|
|
98
|
+
` OrbStack: orb config set memory_mib ${targetGib * 1024}`,
|
|
99
|
+
` Colima: colima stop && colima start --memory ${targetGib}`,
|
|
100
|
+
' Linux Engine: Docker uses host RAM directly; stop an agent instead.',
|
|
101
|
+
]
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Fleet-wide variant for `typeclaw compose`: agents start concurrently, so a
|
|
105
|
+
// per-agent check sees whichever siblings happened to be up and repeats the same
|
|
106
|
+
// advice once per agent. Checking once after the whole fleet settled reports the
|
|
107
|
+
// real total exactly once.
|
|
108
|
+
export async function readFleetMemoryOversubscription(exec: DockerExec): Promise<OversubscriptionWarning | null> {
|
|
109
|
+
const [running, totalMemoryBytes] = await Promise.all([
|
|
110
|
+
readRunningAgentMemoryClaims(exec),
|
|
111
|
+
readDockerTotalMemory(exec),
|
|
112
|
+
])
|
|
113
|
+
return decideMemoryOversubscription({ running, totalMemoryBytes })
|
|
114
|
+
}
|
|
115
|
+
|
|
86
116
|
// Reads the memory cap Docker actually applied to every running agent. Docker
|
|
87
117
|
// reports 0 for a container created with no limit, which is UNLIMITED, not
|
|
88
118
|
// missing — an agent started before this field existed is precisely the case
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs'
|
|
2
|
+
|
|
3
|
+
export type OomEventCounts = { oom: number; oomKill: number }
|
|
4
|
+
export type OomEventsReader = () => OomEventCounts | null
|
|
5
|
+
|
|
6
|
+
// Docker's private cgroup namespace exposes its own v2 root here. Cgroup v1,
|
|
7
|
+
// dev hosts and unreadable/malformed files provide no reliable pair: fail open.
|
|
8
|
+
export function readOomEvents(path = '/sys/fs/cgroup/memory.events'): OomEventCounts | null {
|
|
9
|
+
try {
|
|
10
|
+
const events = readFileSync(path, 'utf8')
|
|
11
|
+
const oom = /^oom (\d+)$/m.exec(events)
|
|
12
|
+
const oomKill = /^oom_kill (\d+)$/m.exec(events)
|
|
13
|
+
if (oom === null || oomKill === null) return null
|
|
14
|
+
const counts = { oom: Number(oom[1]), oomKill: Number(oomKill[1]) }
|
|
15
|
+
return Number.isSafeInteger(counts.oom) && Number.isSafeInteger(counts.oomKill) ? counts : null
|
|
16
|
+
} catch {
|
|
17
|
+
return null
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function readMemoryLimitMiB(path = '/sys/fs/cgroup/memory.max'): number | null {
|
|
22
|
+
try {
|
|
23
|
+
const bytes = readFileSync(path, 'utf8').trim()
|
|
24
|
+
if (!/^\d+$/.test(bytes)) return null
|
|
25
|
+
const value = Number(bytes)
|
|
26
|
+
return Number.isSafeInteger(value) ? Math.round(value / 1048576) : null
|
|
27
|
+
} catch {
|
|
28
|
+
return null
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const claimedCounts = new Map<number, string>()
|
|
33
|
+
const reportedCounts = new Set<number>()
|
|
34
|
+
|
|
35
|
+
// memory.events is CONTAINER-wide: a concurrent unrelated OOM may increase
|
|
36
|
+
// either counter during this command. A limit event plus kill is evidence of
|
|
37
|
+
// container pressure, not proof that the observed child was the victim.
|
|
38
|
+
export function toolOomNote(
|
|
39
|
+
before: OomEventCounts | null,
|
|
40
|
+
exitCode: number,
|
|
41
|
+
tool: string,
|
|
42
|
+
options: { read?: OomEventsReader; limitMiB?: () => number | null; log?: (line: string) => void } = {},
|
|
43
|
+
): string | null {
|
|
44
|
+
if (exitCode !== 137 || before === null) return null
|
|
45
|
+
const after = (options.read ?? readOomEvents)()
|
|
46
|
+
if (after === null || after.oomKill <= before.oomKill) return null
|
|
47
|
+
// Only a single new event can be claimed unambiguously. A sampler may
|
|
48
|
+
// already have logged it before this command's stdout finished draining.
|
|
49
|
+
if (after.oomKill === before.oomKill + 1) claimedCounts.set(after.oomKill, tool)
|
|
50
|
+
const limit = (options.limitMiB ?? readMemoryLimitMiB)()
|
|
51
|
+
const reachedLimit = after.oom > before.oom
|
|
52
|
+
const detail = reachedLimit
|
|
53
|
+
? `the container reached its memory limit${limit === null ? '' : ` (${limit} MiB)`}`
|
|
54
|
+
: `without a container memory-limit event${limit === null ? '' : ` (configured limit ${limit} MiB)`}`
|
|
55
|
+
if (!reportedCounts.has(after.oomKill)) {
|
|
56
|
+
options.log?.(`[tool] ${tool} subprocess SIGKILL; OOM kill observed while command ran; ${detail}`)
|
|
57
|
+
}
|
|
58
|
+
return `An OOM kill was observed while this command ran (exit 137); ${detail}. Reduce parallelism or memory use before retrying.`
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function unclaimedOomEvents(
|
|
62
|
+
previous: OomEventCounts,
|
|
63
|
+
current: OomEventCounts,
|
|
64
|
+
): { from: number; to: number; claimed: string[]; reachedLimit: boolean } | null {
|
|
65
|
+
if (current.oomKill <= previous.oomKill) return null
|
|
66
|
+
const claimed: string[] = []
|
|
67
|
+
let unclaimed = false
|
|
68
|
+
for (let count = previous.oomKill + 1; count <= current.oomKill; count++) {
|
|
69
|
+
const tool = claimedCounts.get(count)
|
|
70
|
+
if (tool === undefined) {
|
|
71
|
+
unclaimed = true
|
|
72
|
+
reportedCounts.add(count)
|
|
73
|
+
} else {
|
|
74
|
+
claimed.push(tool)
|
|
75
|
+
}
|
|
76
|
+
claimedCounts.delete(count)
|
|
77
|
+
}
|
|
78
|
+
// Retain a bounded window for a late tool completion after the sampler's
|
|
79
|
+
// tick. Its model note still arrives, but the operator need not see two lines.
|
|
80
|
+
for (const count of reportedCounts) {
|
|
81
|
+
if (count < current.oomKill - 64) reportedCounts.delete(count)
|
|
82
|
+
}
|
|
83
|
+
return unclaimed
|
|
84
|
+
? { from: previous.oomKill, to: current.oomKill, claimed, reachedLimit: current.oom > previous.oom }
|
|
85
|
+
: null
|
|
86
|
+
}
|
package/src/container/restart.ts
CHANGED
|
@@ -8,6 +8,7 @@ export type RestartOptions = {
|
|
|
8
8
|
forceBuild?: boolean
|
|
9
9
|
streamOutput?: boolean
|
|
10
10
|
onWarning?: (warning: string) => void
|
|
11
|
+
skipMemoryOversubscriptionCheck?: boolean
|
|
11
12
|
cliEntry?: string
|
|
12
13
|
reuseCurrentHostDaemon?: boolean
|
|
13
14
|
currentHostDaemon?: CurrentHostDaemon
|
|
@@ -53,6 +54,7 @@ async function restartWithLease(
|
|
|
53
54
|
forceBuild: options.forceBuild,
|
|
54
55
|
streamOutput: options.streamOutput,
|
|
55
56
|
onWarning: options.onWarning,
|
|
57
|
+
skipMemoryOversubscriptionCheck: options.skipMemoryOversubscriptionCheck,
|
|
56
58
|
cliEntry: options.cliEntry,
|
|
57
59
|
reuseCurrentHostDaemon: options.reuseCurrentHostDaemon,
|
|
58
60
|
currentHostDaemon: options.currentHostDaemon,
|
package/src/container/shared.ts
CHANGED
|
@@ -536,12 +536,19 @@ export async function cleanupRunCorpse(
|
|
|
536
536
|
// `docker run --name <same>` to succeed — Docker's name-reservation table
|
|
537
537
|
// drains independently. waitForRemoval is the fast path; the retry on
|
|
538
538
|
// `docker run` (see isContainerNameConflict) is the safety net.
|
|
539
|
+
//
|
|
540
|
+
// The budget is generous because a draining removal is normal, not stuck:
|
|
541
|
+
// OrbStack has been observed taking ~40s to destroy agents with large
|
|
542
|
+
// writable layers. Timing out early makes `compose restart` report a healthy
|
|
543
|
+
// drain as stuck and tell the operator to restart Docker for nothing.
|
|
544
|
+
export const CONTAINER_REMOVAL_TIMEOUT_MS = 120_000
|
|
545
|
+
|
|
539
546
|
export async function waitForRemoval(
|
|
540
547
|
exec: DockerExec,
|
|
541
548
|
name: string,
|
|
542
549
|
options: { timeoutMs?: number; intervalMs?: number } = {},
|
|
543
550
|
): Promise<boolean> {
|
|
544
|
-
const timeoutMs = options.timeoutMs ??
|
|
551
|
+
const timeoutMs = options.timeoutMs ?? CONTAINER_REMOVAL_TIMEOUT_MS
|
|
545
552
|
const intervalMs = options.intervalMs ?? 100
|
|
546
553
|
const deadline = Date.now() + timeoutMs
|
|
547
554
|
while (Date.now() < deadline) {
|
package/src/container/start.ts
CHANGED
|
@@ -43,6 +43,7 @@ import { hostLocaleIsCjk } from '@/shared/host-locale'
|
|
|
43
43
|
import { type AgentOperationLease, type WithAgentOperationLock, withAgentOperationLock } from './agent-operation-lock'
|
|
44
44
|
import { acquireManagedBuildCache, type ManagedBuildCacheLease } from './build-cache'
|
|
45
45
|
import { COMPOSE_PROJECT } from './compose-project'
|
|
46
|
+
import { inspectContainerExit } from './exit-reason'
|
|
46
47
|
import { archiveContainerLogs, dockerLogsUnavailableWarning, type DockerLogArchiver } from './log-archive'
|
|
47
48
|
import { formatMemorySize, readDockerTotalMemory, resolveMemoryLimit } from './memory-limit'
|
|
48
49
|
import {
|
|
@@ -157,6 +158,8 @@ export type StartOptions = {
|
|
|
157
158
|
// disables process output so child progress and warnings cannot corrupt it.
|
|
158
159
|
streamOutput?: boolean
|
|
159
160
|
onWarning?: (warning: string) => void
|
|
161
|
+
// Compose runs one fleet-wide check after every agent settled instead.
|
|
162
|
+
skipMemoryOversubscriptionCheck?: boolean
|
|
160
163
|
exec?: DockerExec
|
|
161
164
|
// Test seam: allows tests to inject a deterministic port allocator. In
|
|
162
165
|
// production we go through the real kernel via `findFreePort`.
|
|
@@ -272,6 +275,7 @@ async function runStart({
|
|
|
272
275
|
forceBuild = false,
|
|
273
276
|
streamOutput = true,
|
|
274
277
|
onWarning,
|
|
278
|
+
skipMemoryOversubscriptionCheck = false,
|
|
275
279
|
exec = defaultDockerExec,
|
|
276
280
|
allocatePort = findFreePort,
|
|
277
281
|
cliEntry,
|
|
@@ -296,21 +300,27 @@ async function runStart({
|
|
|
296
300
|
const containerName = containerNameFromCwd(cwd)
|
|
297
301
|
const imageTagValue = imageTagFromCwd(cwd)
|
|
298
302
|
const archiveBeforeRemove = async (containerId: string) => {
|
|
303
|
+
const exit = await inspectContainerExit(exec, containerId)
|
|
299
304
|
const archive = await archiveLogs({
|
|
300
305
|
agentDir: cwd,
|
|
301
306
|
containerId,
|
|
302
307
|
retentionDays: (await loadTypeclawConfig(cwd)).logs.retentionDays,
|
|
303
308
|
})
|
|
304
|
-
if (archive.ok) return { ok:
|
|
305
|
-
if (archive.kind === 'failed') return { ok: false as const, reason: archive.reason }
|
|
309
|
+
if (!archive.ok && archive.kind === 'failed') return { ok: false as const, reason: archive.reason }
|
|
306
310
|
// Dispatch on the callback, NOT on streamOutput. `streamOutput` defaults
|
|
307
311
|
// to true, and standalone start/restart both stream Docker build output
|
|
308
312
|
// AND pass a collector so they can render warnings after their spinner
|
|
309
313
|
// settles; keying on streamOutput writes under the live spinner and
|
|
310
314
|
// leaves that collector empty. stderr stays the no-callback fallback.
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
315
|
+
if (!archive.ok) {
|
|
316
|
+
const warning = dockerLogsUnavailableWarning(containerId)
|
|
317
|
+
if (onWarning !== undefined) onWarning(warning)
|
|
318
|
+
else if (streamOutput) process.stderr.write(`${warning}\n`)
|
|
319
|
+
}
|
|
320
|
+
if (exit?.kind === 'stopped' && exit.reason !== null) {
|
|
321
|
+
if (onWarning !== undefined) onWarning(exit.reason)
|
|
322
|
+
else if (streamOutput) process.stderr.write(`typeclaw: ${exit.reason}\n`)
|
|
323
|
+
}
|
|
314
324
|
return { ok: true as const }
|
|
315
325
|
}
|
|
316
326
|
|
|
@@ -601,14 +611,16 @@ async function runStart({
|
|
|
601
611
|
totalMemoryBytes,
|
|
602
612
|
})
|
|
603
613
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
614
|
+
if (!skipMemoryOversubscriptionCheck) {
|
|
615
|
+
await warnOnMemoryOversubscription({
|
|
616
|
+
exec,
|
|
617
|
+
containerName,
|
|
618
|
+
memoryLimitBytes: plan.memoryLimitBytes,
|
|
619
|
+
totalMemoryBytes,
|
|
620
|
+
onWarning,
|
|
621
|
+
streamOutput,
|
|
622
|
+
})
|
|
623
|
+
}
|
|
612
624
|
|
|
613
625
|
let built = false
|
|
614
626
|
if (plan.needsBuild) {
|
package/src/container/stop.ts
CHANGED
|
@@ -2,9 +2,11 @@ import { DEFAULT_LOG_RETENTION_DAYS, loadConfigSync } from '@/config'
|
|
|
2
2
|
import { isDaemonReachable, send as sendToDaemon } from '@/hostd/client'
|
|
3
3
|
|
|
4
4
|
import { type AgentOperationLease, type WithAgentOperationLock, withAgentOperationLock } from './agent-operation-lock'
|
|
5
|
+
import { inspectContainerExit } from './exit-reason'
|
|
5
6
|
import { archiveContainerLogs, dockerLogsUnavailableWarning, type DockerLogArchiver } from './log-archive'
|
|
6
7
|
import {
|
|
7
8
|
classifyRmStderr,
|
|
9
|
+
CONTAINER_REMOVAL_TIMEOUT_MS,
|
|
8
10
|
containerNameFromCwd,
|
|
9
11
|
defaultDockerExec,
|
|
10
12
|
type DockerExec,
|
|
@@ -78,6 +80,7 @@ async function runStop({
|
|
|
78
80
|
}
|
|
79
81
|
}
|
|
80
82
|
const { containerId, running } = parsed
|
|
83
|
+
const staleExit = running ? null : await inspectContainerExit(exec, containerId)
|
|
81
84
|
|
|
82
85
|
// Only call `docker stop` when the container is actually running. A stopped
|
|
83
86
|
// corpse from a prior crash is left around by design (no `--rm`), and
|
|
@@ -100,6 +103,7 @@ async function runStop({
|
|
|
100
103
|
}
|
|
101
104
|
onWarning?.(dockerLogsUnavailableWarning(containerId))
|
|
102
105
|
}
|
|
106
|
+
if (staleExit?.kind === 'stopped' && staleExit.reason !== null) onWarning?.(staleExit.reason)
|
|
103
107
|
|
|
104
108
|
// Containers run without `--rm`, so `docker stop` only stops them. Remove
|
|
105
109
|
// the inspected ID without force after archival. If that same ID resumed,
|
|
@@ -134,12 +138,12 @@ async function runStop({
|
|
|
134
138
|
return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
|
|
135
139
|
}
|
|
136
140
|
if (kind === 'in-progress' && !(await waitForRemoval(exec, containerName))) {
|
|
137
|
-
const reason = `Container ${containerName} is still being removed by docker after
|
|
141
|
+
const reason = `Container ${containerName} is still being removed by docker after ${CONTAINER_REMOVAL_TIMEOUT_MS / 1000}s.`
|
|
138
142
|
const diagnosis = await describeUnremovableContainer(exec, containerId)
|
|
139
143
|
return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
|
|
140
144
|
}
|
|
141
145
|
} else if (!(await waitForRemoval(exec, containerName))) {
|
|
142
|
-
const reason = `Container ${containerName} is still being removed by docker after
|
|
146
|
+
const reason = `Container ${containerName} is still being removed by docker after ${CONTAINER_REMOVAL_TIMEOUT_MS / 1000}s.`
|
|
143
147
|
const diagnosis = await describeUnremovableContainer(exec, containerId)
|
|
144
148
|
return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
|
|
145
149
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Under a 6 GiB memcg, a ~230 MiB tool process contributes only ~38 points
|
|
2
|
+
// from RSS, versus ~230 for a 1.4 GiB agent runtime. +600 puts even that
|
|
3
|
+
// tool ahead of the runtime without using the maximum (+1000) and making
|
|
4
|
+
// tiny helper processes unconditional first victims. Children inherit the
|
|
5
|
+
// score across fork/exec. The shell writes its OWN score before any command
|
|
6
|
+
// can fork; writing /proc/<pid> from the parent after spawn would race.
|
|
7
|
+
const OOM_SCORE_ADJ = 600
|
|
8
|
+
const SCORE_PATH = '/proc/self/oom_score_adj'
|
|
9
|
+
const PRIORITY_SCRIPT = `if [ -w ${SCORE_PATH} ]; then printf '%s\\n' ${OOM_SCORE_ADJ} 2>/dev/null >${SCORE_PATH} || :; fi`
|
|
10
|
+
|
|
11
|
+
function priorityScript(scorePath: string): string {
|
|
12
|
+
// The default is pre-rendered; alternate paths are solely a test seam for
|
|
13
|
+
// an absent or write-failing procfs, never derived from a tool command.
|
|
14
|
+
return scorePath === SCORE_PATH
|
|
15
|
+
? PRIORITY_SCRIPT
|
|
16
|
+
: `if [ -w '${scorePath}' ]; then printf '%s\\n' ${OOM_SCORE_ADJ} 2>/dev/null >'${scorePath}' || :; fi`
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// Shell built-in printf runs in this process. An absent/unwritable procfs
|
|
20
|
+
// (including non-container dev environments) fails open without changing the
|
|
21
|
+
// command's output, exit status, environment, or timeout behavior.
|
|
22
|
+
export function withToolOomPriorityCommand(command: string, scorePath = SCORE_PATH): string {
|
|
23
|
+
return process.platform === 'linux' ? `${priorityScript(scorePath)}\n${command}` : command
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function withToolOomPriorityArgv(command: string[], scorePath = SCORE_PATH): string[] {
|
|
27
|
+
return process.platform === 'linux'
|
|
28
|
+
? ['/bin/sh', '-c', `${priorityScript(scorePath)}\nexec "$@"`, 'tool-oom-priority', ...command]
|
|
29
|
+
: command
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export { OOM_SCORE_ADJ }
|
package/src/cron/consumer.ts
CHANGED
|
@@ -4,6 +4,8 @@ import { promptWithFallback, resolveFallbackChain } from '@/agent/model-fallback
|
|
|
4
4
|
import type { SessionOrigin } from '@/agent/session-origin'
|
|
5
5
|
import { getConfig } from '@/config'
|
|
6
6
|
import type { ModelRef } from '@/config/providers'
|
|
7
|
+
import { readOomEvents, toolOomNote } from '@/container/oom-events'
|
|
8
|
+
import { withToolOomPriorityArgv } from '@/container/tool-oom-priority'
|
|
7
9
|
import type { HookBus } from '@/plugin'
|
|
8
10
|
import type { Stream, Unsubscribe } from '@/stream'
|
|
9
11
|
|
|
@@ -143,7 +145,7 @@ export function createCronConsumer({
|
|
|
143
145
|
if (job.kind === 'prompt') {
|
|
144
146
|
if (!(await runPrompt(job, createSessionForCron, stream, logger))) outcome = 'failed'
|
|
145
147
|
} else if (job.kind === 'exec') {
|
|
146
|
-
await runExec(job, cwd)
|
|
148
|
+
await runExec(job, cwd, logger)
|
|
147
149
|
} else {
|
|
148
150
|
if (invokeHandler === undefined) {
|
|
149
151
|
throw new Error(
|
|
@@ -347,7 +349,7 @@ function describe(err: unknown): string {
|
|
|
347
349
|
return err instanceof Error ? err.message : String(err)
|
|
348
350
|
}
|
|
349
351
|
|
|
350
|
-
async function runExec(job: ExecJob, cwd: string): Promise<void> {
|
|
352
|
+
async function runExec(job: ExecJob, cwd: string, logger: CronConsumerLogger): Promise<void> {
|
|
351
353
|
const [cmd, ...args] = job.command
|
|
352
354
|
if (!cmd) throw new Error(`exec job ${job.id}: empty command`)
|
|
353
355
|
// Inject TYPECLAW_PARENT_ORIGIN_JSON so a child that proxies into the
|
|
@@ -363,8 +365,9 @@ async function runExec(job: ExecJob, cwd: string): Promise<void> {
|
|
|
363
365
|
...(job.scheduledByRole !== undefined ? { scheduledByRole: job.scheduledByRole } : {}),
|
|
364
366
|
...(job.scheduledByOrigin !== undefined ? { scheduledByOrigin: job.scheduledByOrigin } : {}),
|
|
365
367
|
}
|
|
368
|
+
const oomBefore = readOomEvents()
|
|
366
369
|
const proc = Bun.spawn({
|
|
367
|
-
cmd: [cmd, ...args],
|
|
370
|
+
cmd: withToolOomPriorityArgv([cmd, ...args]),
|
|
368
371
|
cwd,
|
|
369
372
|
stdout: 'ignore',
|
|
370
373
|
stderr: 'pipe',
|
|
@@ -375,8 +378,11 @@ async function runExec(job: ExecJob, cwd: string): Promise<void> {
|
|
|
375
378
|
})
|
|
376
379
|
const stderrText = new Response(proc.stderr).text()
|
|
377
380
|
const [code, stderr] = await Promise.all([proc.exited, stderrText])
|
|
381
|
+
const note = toolOomNote(oomBefore, code, 'cron exec', { log: logger.warn })
|
|
378
382
|
if (code !== 0) {
|
|
379
|
-
throw new Error(
|
|
383
|
+
throw new Error(
|
|
384
|
+
`exec job ${job.id} exited with code ${code}: ${stderr.trim() || 'no stderr'}${note === null ? '' : `\n${note}`}`,
|
|
385
|
+
)
|
|
380
386
|
}
|
|
381
387
|
}
|
|
382
388
|
|
package/src/cron/reloadable.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type { Reloadable, ReloadResult } from '@/reload'
|
|
|
3
3
|
|
|
4
4
|
import { loadCron } from './index'
|
|
5
5
|
import type { JobDiff, Scheduler } from './scheduler'
|
|
6
|
-
import type { CronJob } from './schema'
|
|
6
|
+
import type { CronJob, HandlerJob, ParsedCronJob } from './schema'
|
|
7
7
|
|
|
8
8
|
export type CreateCronReloadableOptions = {
|
|
9
9
|
cwd: string
|
|
@@ -74,7 +74,28 @@ async function doReload({
|
|
|
74
74
|
return { scope: 'cron', ok: false, reason: `apply failed (schedule unchanged): ${message}` }
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
-
return { scope: 'cron', ok: true, summary: formatSummary(diff, nextJobs.length), details: diff }
|
|
77
|
+
return { scope: 'cron', ok: true, summary: formatSummary(diff, nextJobs.length), details: plainDiff(diff) }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// `details` lands in the reload tool result, which pi structuredClones on every
|
|
81
|
+
// later request in that session. A plugin handler job's function would make
|
|
82
|
+
// that clone throw DataCloneError and wedge the session, so details carry data only.
|
|
83
|
+
type CronJobData = ParsedCronJob | Omit<HandlerJob, 'handler'>
|
|
84
|
+
type CronReloadDetails = Record<keyof JobDiff, CronJobData[]>
|
|
85
|
+
|
|
86
|
+
function plainDiff(diff: JobDiff): CronReloadDetails {
|
|
87
|
+
const plain = (jobs: CronJob[]): CronJobData[] =>
|
|
88
|
+
jobs.map((job) => {
|
|
89
|
+
if (job.kind !== 'handler') return job
|
|
90
|
+
const { handler: _handler, ...data } = job
|
|
91
|
+
return data
|
|
92
|
+
})
|
|
93
|
+
return {
|
|
94
|
+
added: plain(diff.added),
|
|
95
|
+
removed: plain(diff.removed),
|
|
96
|
+
updated: plain(diff.updated),
|
|
97
|
+
unchanged: plain(diff.unchanged),
|
|
98
|
+
}
|
|
78
99
|
}
|
|
79
100
|
|
|
80
101
|
function formatSummary(diff: JobDiff, total: number): string {
|
package/src/doctor/checks.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
type DockerExec,
|
|
18
18
|
} from '@/container'
|
|
19
19
|
import { homeRoot, isDaemonReachable, send } from '@/hostd'
|
|
20
|
+
import { hostBunVersionWarning } from '@/hostd/version'
|
|
20
21
|
import { resolveBaseImageVersion } from '@/init/cli-version'
|
|
21
22
|
import { buildDockerfile, DOCKERFILE } from '@/init/dockerfile'
|
|
22
23
|
import { detectMissingDeps } from '@/init/ensure-deps'
|
|
@@ -99,7 +100,7 @@ function dockerDaemonFix(reason: DockerUnavailableReason): string {
|
|
|
99
100
|
}
|
|
100
101
|
}
|
|
101
102
|
|
|
102
|
-
function bunRuntime(): DoctorCheck {
|
|
103
|
+
export function bunRuntime(bunVersion: string | undefined = process.versions.bun): DoctorCheck {
|
|
103
104
|
return {
|
|
104
105
|
name: 'runtime.bun-available',
|
|
105
106
|
category: 'runtime',
|
|
@@ -113,7 +114,18 @@ function bunRuntime(): DoctorCheck {
|
|
|
113
114
|
fix: { description: 'Install Bun (https://bun.sh) and ensure the typeclaw CLI runs under it.' },
|
|
114
115
|
}
|
|
115
116
|
}
|
|
116
|
-
|
|
117
|
+
const outdated = hostBunVersionWarning(bunVersion)
|
|
118
|
+
if (outdated) {
|
|
119
|
+
return {
|
|
120
|
+
status: 'warning',
|
|
121
|
+
message: outdated,
|
|
122
|
+
fix: {
|
|
123
|
+
description:
|
|
124
|
+
'Run `bun upgrade`. The host daemon respawns on the new runtime at the next `typeclaw start` or `restart`.',
|
|
125
|
+
},
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return { status: 'ok', message: `Bun ${bunVersion ?? 'present'}` }
|
|
117
129
|
},
|
|
118
130
|
}
|
|
119
131
|
}
|