typeclaw 0.48.11 → 0.48.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/agent/index.ts +11 -1
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +7 -1
- package/src/bundled-plugins/github-cli-auth/index.ts +1 -1
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +9 -1
- package/src/bundled-plugins/memory/dreaming.ts +21 -6
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +86 -16
- package/src/channels/adapters/kakaotalk.ts +7 -8
- package/src/channels/adapters/teams-classify.ts +21 -0
- package/src/channels/adapters/teams-edit.ts +2 -1
- package/src/channels/adapters/teams-format.ts +16 -0
- package/src/channels/adapters/teams.ts +12 -5
- package/src/cli/compose.ts +3 -3
- package/src/cli/docker-preflight.ts +8 -4
- package/src/cli/restart.ts +89 -23
- package/src/cli/start.ts +68 -16
- package/src/cli/status.ts +38 -6
- package/src/cli/stop.ts +5 -2
- package/src/compose/stop.ts +5 -4
- package/src/container/build-cache.ts +807 -0
- package/src/container/compose-project.ts +4 -0
- package/src/container/docker-app.ts +74 -0
- package/src/container/log-archive.ts +33 -5
- package/src/container/memory-limit.ts +80 -0
- package/src/container/memory-oversubscription.ts +125 -0
- package/src/container/restart.ts +6 -1
- package/src/container/shared.ts +43 -16
- package/src/container/start.ts +280 -53
- package/src/container/stop.ts +19 -13
- package/src/container/verify-running.ts +12 -0
- package/src/cron/scheduler.ts +12 -4
- package/src/doctor/bounded-exec.ts +14 -0
- package/src/doctor/checks.ts +40 -15
- package/src/doctor/index.ts +14 -2
- package/src/doctor/plugin-bridge.ts +15 -3
- package/src/hostd/paths.ts +4 -0
- package/src/run/index.ts +16 -0
- package/src/run/resource-sample.ts +171 -0
- package/src/sandbox/index.ts +8 -1
- package/src/sandbox/session-tmp.ts +26 -1
package/src/container/start.ts
CHANGED
|
@@ -36,11 +36,20 @@ import {
|
|
|
36
36
|
type GithubCliProvisionResult,
|
|
37
37
|
type ProvisionGithubCliStoreOptions,
|
|
38
38
|
} from '@/secrets/provision-github-cli-store'
|
|
39
|
+
import { SecretsBackend } from '@/secrets/storage'
|
|
39
40
|
import { isWindows } from '@/shared'
|
|
40
41
|
import { hostLocaleIsCjk } from '@/shared/host-locale'
|
|
41
42
|
|
|
42
43
|
import { type AgentOperationLease, type WithAgentOperationLock, withAgentOperationLock } from './agent-operation-lock'
|
|
43
|
-
import {
|
|
44
|
+
import { acquireManagedBuildCache, type ManagedBuildCacheLease } from './build-cache'
|
|
45
|
+
import { COMPOSE_PROJECT } from './compose-project'
|
|
46
|
+
import { archiveContainerLogs, dockerLogsUnavailableWarning, type DockerLogArchiver } from './log-archive'
|
|
47
|
+
import { formatMemorySize, readDockerTotalMemory, resolveMemoryLimit } from './memory-limit'
|
|
48
|
+
import {
|
|
49
|
+
decideMemoryOversubscription,
|
|
50
|
+
formatOversubscriptionWarning,
|
|
51
|
+
readRunningAgentMemoryClaims,
|
|
52
|
+
} from './memory-oversubscription'
|
|
44
53
|
import { CONTAINER_PORT, TUI_TOKEN_LABEL, findFreePort, isPortAllocatedError, resolveTuiToken } from './port'
|
|
45
54
|
import {
|
|
46
55
|
buildxAvailable,
|
|
@@ -59,7 +68,12 @@ import {
|
|
|
59
68
|
sanitizeDockerConfigJson,
|
|
60
69
|
sanitizeDockerStderr,
|
|
61
70
|
} from './shared'
|
|
62
|
-
import {
|
|
71
|
+
import {
|
|
72
|
+
buildCrashReason,
|
|
73
|
+
createVerifyRunning,
|
|
74
|
+
describeUnremovableContainer,
|
|
75
|
+
type VerifyRunningFn,
|
|
76
|
+
} from './verify-running'
|
|
63
77
|
|
|
64
78
|
const PACKAGE_FILE = 'package.json'
|
|
65
79
|
const TYPECLAW_PACKAGE = 'typeclaw'
|
|
@@ -67,7 +81,6 @@ const DEV_SOURCE_CONTAINER_PATH = '/agent/node_modules/typeclaw'
|
|
|
67
81
|
const BUN_LOCK_FILE = 'bun.lock'
|
|
68
82
|
const DEPENDENCY_FILES = [PACKAGE_FILE, BUN_LOCK_FILE] as const
|
|
69
83
|
const ENV_FILE = '.env'
|
|
70
|
-
const COMPOSE_PROJECT = 'typeclaw'
|
|
71
84
|
const CONTAINER_HOSTD_HOST = 'host.docker.internal'
|
|
72
85
|
const HOST_GATEWAY_ALIAS = `${CONTAINER_HOSTD_HOST}:host-gateway`
|
|
73
86
|
|
|
@@ -82,6 +95,7 @@ export type StartPlan = {
|
|
|
82
95
|
needsBuild: boolean
|
|
83
96
|
hostPort: number
|
|
84
97
|
tuiToken: string | null
|
|
98
|
+
memoryLimitBytes: number
|
|
85
99
|
}
|
|
86
100
|
|
|
87
101
|
export type PlanStartOptions = {
|
|
@@ -99,6 +113,12 @@ export type PlanStartOptions = {
|
|
|
99
113
|
// Omitted when unavailable (native Windows and unusual JS hosts).
|
|
100
114
|
hostIdentity?: { uid: number; gid: number } | null
|
|
101
115
|
gitIdentity?: GitIdentity | null
|
|
116
|
+
// Total memory of the machine the CONTAINER runs on, used only to clamp the
|
|
117
|
+
// default memory limit down when it will not fit. On macOS and Windows that
|
|
118
|
+
// is the Docker VM, not the workstation, so start() reads it from the daemon
|
|
119
|
+
// rather than from os.totalmem(). Omitted by callers that have no exec (and
|
|
120
|
+
// by tests), in which case the default applies unclamped.
|
|
121
|
+
totalMemoryBytes?: number | undefined
|
|
102
122
|
}
|
|
103
123
|
|
|
104
124
|
export type HostDaemonControl = {
|
|
@@ -197,6 +217,8 @@ export type StartOptions = {
|
|
|
197
217
|
archiveLogs?: DockerLogArchiver
|
|
198
218
|
operationLock?: WithAgentOperationLock
|
|
199
219
|
operationLease?: AgentOperationLease
|
|
220
|
+
buildCacheStateDir?: string
|
|
221
|
+
acquireBuildCache?: typeof acquireManagedBuildCache | null
|
|
200
222
|
}
|
|
201
223
|
|
|
202
224
|
export type HostDaemonStatus =
|
|
@@ -267,16 +289,30 @@ async function runStart({
|
|
|
267
289
|
assertConfigWritable = assertAgentConfigWritable,
|
|
268
290
|
provisionGithubCliStore: provisionGithubCliStoreForAgent = provisionGithubCliStore,
|
|
269
291
|
archiveLogs = archiveContainerLogs,
|
|
292
|
+
buildCacheStateDir,
|
|
293
|
+
acquireBuildCache = acquireManagedBuildCache,
|
|
270
294
|
}: StartOptions): Promise<StartResult> {
|
|
271
295
|
try {
|
|
272
296
|
const containerName = containerNameFromCwd(cwd)
|
|
273
297
|
const imageTagValue = imageTagFromCwd(cwd)
|
|
274
|
-
const archiveBeforeRemove = async (containerId: string) =>
|
|
275
|
-
await archiveLogs({
|
|
298
|
+
const archiveBeforeRemove = async (containerId: string) => {
|
|
299
|
+
const archive = await archiveLogs({
|
|
276
300
|
agentDir: cwd,
|
|
277
301
|
containerId,
|
|
278
302
|
retentionDays: (await loadTypeclawConfig(cwd)).logs.retentionDays,
|
|
279
303
|
})
|
|
304
|
+
if (archive.ok) return { ok: true as const }
|
|
305
|
+
if (archive.kind === 'failed') return { ok: false as const, reason: archive.reason }
|
|
306
|
+
// Dispatch on the callback, NOT on streamOutput. `streamOutput` defaults
|
|
307
|
+
// to true, and standalone start/restart both stream Docker build output
|
|
308
|
+
// AND pass a collector so they can render warnings after their spinner
|
|
309
|
+
// settles; keying on streamOutput writes under the live spinner and
|
|
310
|
+
// leaves that collector empty. stderr stays the no-callback fallback.
|
|
311
|
+
const warning = dockerLogsUnavailableWarning(containerId)
|
|
312
|
+
if (onWarning !== undefined) onWarning(warning)
|
|
313
|
+
else if (streamOutput) process.stderr.write(`${warning}\n`)
|
|
314
|
+
return { ok: true as const }
|
|
315
|
+
}
|
|
280
316
|
|
|
281
317
|
// Probe container state BEFORE refreshing Dockerfile/.gitignore: when the
|
|
282
318
|
// container is already running, start() is a no-op and must not produce
|
|
@@ -525,14 +561,16 @@ async function runStart({
|
|
|
525
561
|
}
|
|
526
562
|
}
|
|
527
563
|
if (cleanup === 'stuck') {
|
|
564
|
+
const reason = `Container ${containerName} could not be safely inspected or removed; preserving it and refusing to docker run --name to avoid a conflict.`
|
|
565
|
+
const diagnosis = await describeUnremovableContainer(exec, state.containerId)
|
|
528
566
|
return {
|
|
529
567
|
ok: false,
|
|
530
|
-
reason:
|
|
568
|
+
reason: diagnosis === null ? reason : `${reason} ${diagnosis}`,
|
|
531
569
|
}
|
|
532
570
|
}
|
|
533
571
|
}
|
|
534
572
|
|
|
535
|
-
const
|
|
573
|
+
const previousImageId = await inspectImageId(exec, imageTagValue)
|
|
536
574
|
|
|
537
575
|
// First attempt uses the user's preferred host port (8973 by default, or
|
|
538
576
|
// whatever they passed via --port / typeclaw.json). If it's already bound
|
|
@@ -549,28 +587,64 @@ async function runStart({
|
|
|
549
587
|
|
|
550
588
|
const publishHost = await resolvePublishHost(exec)
|
|
551
589
|
const tuiToken = randomBytes(32).toString('base64url')
|
|
590
|
+
const totalMemoryBytes = await readDockerTotalMemory(exec)
|
|
552
591
|
let plan = await planStart({
|
|
553
592
|
cwd,
|
|
554
593
|
hostPort,
|
|
555
|
-
imageExists:
|
|
594
|
+
imageExists: previousImageId !== null,
|
|
556
595
|
forceBuild: forceBuild || dockerfileRefresh.changed,
|
|
557
596
|
hostdControl,
|
|
558
597
|
publishHost,
|
|
559
598
|
tuiToken,
|
|
560
599
|
platform,
|
|
561
600
|
hostIdentity,
|
|
601
|
+
totalMemoryBytes,
|
|
602
|
+
})
|
|
603
|
+
|
|
604
|
+
await warnOnMemoryOversubscription({
|
|
605
|
+
exec,
|
|
606
|
+
containerName,
|
|
607
|
+
memoryLimitBytes: plan.memoryLimitBytes,
|
|
608
|
+
totalMemoryBytes,
|
|
609
|
+
onWarning,
|
|
610
|
+
streamOutput,
|
|
562
611
|
})
|
|
563
612
|
|
|
564
613
|
let built = false
|
|
565
614
|
if (plan.needsBuild) {
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
streamOutput
|
|
573
|
-
|
|
615
|
+
const buildWarning = (warning: string): void => {
|
|
616
|
+
if (streamOutput) process.stderr.write(`typeclaw: ${warning}\n`)
|
|
617
|
+
else onWarning?.(warning)
|
|
618
|
+
}
|
|
619
|
+
const protection = await protectImageForRebuild(exec, plan.imageTag, previousImageId)
|
|
620
|
+
if (protection.warning !== null) {
|
|
621
|
+
if (streamOutput) process.stderr.write(`typeclaw: ${protection.warning}\n`)
|
|
622
|
+
else onWarning?.(protection.warning)
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
let buildResult: Awaited<ReturnType<typeof runImageBuild>>
|
|
626
|
+
try {
|
|
627
|
+
buildResult = await runImageBuild({
|
|
628
|
+
exec,
|
|
629
|
+
cwd,
|
|
630
|
+
cacheScope: containerName,
|
|
631
|
+
imageTag: plan.imageTag,
|
|
632
|
+
buildContext: plan.buildContext,
|
|
633
|
+
hasBuildx,
|
|
634
|
+
acquireBuildCache,
|
|
635
|
+
buildCacheStateDir,
|
|
636
|
+
streamOutput,
|
|
637
|
+
onWarning: buildWarning,
|
|
638
|
+
})
|
|
639
|
+
} finally {
|
|
640
|
+
if (protection.tag !== null) {
|
|
641
|
+
const releaseWarning = await releaseImageProtection(exec, protection.tag)
|
|
642
|
+
if (releaseWarning !== null) {
|
|
643
|
+
if (streamOutput) process.stderr.write(`typeclaw: ${releaseWarning}\n`)
|
|
644
|
+
else onWarning?.(releaseWarning)
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
}
|
|
574
648
|
if (!buildResult.ok) {
|
|
575
649
|
await cleanupHostDaemonRegistration(containerName, hostd)
|
|
576
650
|
const retryDetail = buildResult.credentialHelperRetried
|
|
@@ -625,6 +699,7 @@ async function runStart({
|
|
|
625
699
|
tuiToken,
|
|
626
700
|
platform,
|
|
627
701
|
hostIdentity,
|
|
702
|
+
totalMemoryBytes,
|
|
628
703
|
})
|
|
629
704
|
}
|
|
630
705
|
} catch (error) {
|
|
@@ -697,6 +772,12 @@ async function runStart({
|
|
|
697
772
|
publishHost,
|
|
698
773
|
tuiToken,
|
|
699
774
|
platform,
|
|
775
|
+
// Every replan must carry the daemon total. planStart resolves the
|
|
776
|
+
// memory limit from it, so dropping it here silently replaces a
|
|
777
|
+
// clamped limit with the unconditional default right before
|
|
778
|
+
// `docker run` — the launch would exceed what the VM can honor while
|
|
779
|
+
// the already-emitted warning still described the clamped figure.
|
|
780
|
+
totalMemoryBytes,
|
|
700
781
|
})
|
|
701
782
|
run = await execRunWithConflictRetry(exec, plan.runArgs, cwd, containerName, archiveBeforeRemove)
|
|
702
783
|
}
|
|
@@ -745,6 +826,16 @@ async function refreshGithubCliStore(
|
|
|
745
826
|
}
|
|
746
827
|
if (refreshed) return null
|
|
747
828
|
|
|
829
|
+
// Only a failure that leaves an EXISTING store to go stale is worth an
|
|
830
|
+
// operator warning; with no store nothing regressed. This credential is
|
|
831
|
+
// OPTIONAL — authenticated `git push` and PR review ride the agent's own
|
|
832
|
+
// per-repo App token (`github.resolveTokenForRepo`, injected via GIT_ASKPASS
|
|
833
|
+
// by `github-cli-auth` and the backup runner), and the host store only backs
|
|
834
|
+
// `gh` invocations the broker cannot scope to a literal owner/repo. Warning
|
|
835
|
+
// unconditionally told every agent to run `gh auth login` for a capability
|
|
836
|
+
// most never use, citing a persisted store that did not exist.
|
|
837
|
+
if (!hasPersistedGithubCliStore(agentDir)) return null
|
|
838
|
+
|
|
748
839
|
return (
|
|
749
840
|
'typeclaw: warning: Could not refresh GitHub CLI credentials from the host. ' +
|
|
750
841
|
'Keeping the previously persisted credential store. Run `gh auth login --hostname github.com` on the host, ' +
|
|
@@ -752,6 +843,16 @@ async function refreshGithubCliStore(
|
|
|
752
843
|
)
|
|
753
844
|
}
|
|
754
845
|
|
|
846
|
+
// An unreadable or malformed secrets.json cannot prove the store is absent, so
|
|
847
|
+
// it takes the warning — silence is reserved for a confirmed no-store agent.
|
|
848
|
+
function hasPersistedGithubCliStore(agentDir: string): boolean {
|
|
849
|
+
try {
|
|
850
|
+
return new SecretsBackend(join(agentDir, 'secrets.json')).tryReadGithubCliSync() !== undefined
|
|
851
|
+
} catch {
|
|
852
|
+
return true
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
755
856
|
function resolveGithubCliDeniedRoots(cwd: string, config: Config): string[] {
|
|
756
857
|
return [
|
|
757
858
|
realpathSync(cwd),
|
|
@@ -808,6 +909,7 @@ export async function planStart({
|
|
|
808
909
|
platform = process.platform,
|
|
809
910
|
hostIdentity = currentHostIdentity(),
|
|
810
911
|
gitIdentity,
|
|
912
|
+
totalMemoryBytes,
|
|
811
913
|
}: PlanStartOptions): Promise<StartPlan> {
|
|
812
914
|
const containerName = containerNameFromCwd(cwd)
|
|
813
915
|
const imageTag = imageTagFromCwd(cwd)
|
|
@@ -815,6 +917,8 @@ export async function planStart({
|
|
|
815
917
|
const devSourcePath = await detectDevSource(cwd)
|
|
816
918
|
const cfg = await loadTypeclawConfig(cwd)
|
|
817
919
|
const mounts = cfg.mounts
|
|
920
|
+
const memoryLimit = resolveMemoryLimit({ totalMemoryBytes })
|
|
921
|
+
const memoryLimitArg = formatMemorySize(memoryLimit.bytes)
|
|
818
922
|
|
|
819
923
|
// No `--rm`: a crashed container's logs MUST survive past exit. Lifecycle
|
|
820
924
|
// cleanup archives them under host-stage <agent>/.typeclaw/logs/ before
|
|
@@ -899,6 +1003,21 @@ export async function planStart({
|
|
|
899
1003
|
'--name',
|
|
900
1004
|
containerName,
|
|
901
1005
|
'--shm-size=2g',
|
|
1006
|
+
// `--memory-swap` equal to `--memory` disables swap for the container, and
|
|
1007
|
+
// that equality is the load-bearing half. Without a cgroup limit at all,
|
|
1008
|
+
// growth is charged to the host VM, and the guest kernel answers memory
|
|
1009
|
+
// pressure by writing to swap: every vCPU ends up in direct reclaim
|
|
1010
|
+
// (shrink_folio_list -> swap_writeout) while the OOM killer frees nothing
|
|
1011
|
+
// reclaimable, and the VM livelocks instead of killing one process. With
|
|
1012
|
+
// the limit the charge fails at the cgroup boundary and a single agent is
|
|
1013
|
+
// killed. Letting the container swap would restore the original livelock
|
|
1014
|
+
// inside the cgroup, which is why these are set as a pair.
|
|
1015
|
+
//
|
|
1016
|
+
// A cgroup limit also makes /dev/shm pages count against the container
|
|
1017
|
+
// rather than the host, which matters because shm is not reclaimable by
|
|
1018
|
+
// killing a process — see the --shm-size note above.
|
|
1019
|
+
`--memory=${memoryLimitArg}`,
|
|
1020
|
+
`--memory-swap=${memoryLimitArg}`,
|
|
902
1021
|
'--security-opt',
|
|
903
1022
|
'seccomp=unconfined',
|
|
904
1023
|
'--security-opt',
|
|
@@ -1047,9 +1166,36 @@ export async function planStart({
|
|
|
1047
1166
|
needsBuild: forceBuild || !imageExists,
|
|
1048
1167
|
hostPort,
|
|
1049
1168
|
tuiToken,
|
|
1169
|
+
memoryLimitBytes: memoryLimit.bytes,
|
|
1050
1170
|
}
|
|
1051
1171
|
}
|
|
1052
1172
|
|
|
1173
|
+
// Surfaces the arithmetic when every agent's cap together exceeds what Docker
|
|
1174
|
+
// actually has. Never blocks: a fleet that has run fine at a nominal
|
|
1175
|
+
// oversubscription must not be bricked by an upgrade, and a refused start is a
|
|
1176
|
+
// worse failure than a crowded host.
|
|
1177
|
+
async function warnOnMemoryOversubscription(options: {
|
|
1178
|
+
exec: DockerExec
|
|
1179
|
+
containerName: string
|
|
1180
|
+
memoryLimitBytes: number
|
|
1181
|
+
totalMemoryBytes: number | undefined
|
|
1182
|
+
onWarning: ((warning: string) => void) | undefined
|
|
1183
|
+
streamOutput: boolean
|
|
1184
|
+
}): Promise<void> {
|
|
1185
|
+
const { exec, containerName, memoryLimitBytes, totalMemoryBytes, onWarning, streamOutput } = options
|
|
1186
|
+
const running = await readRunningAgentMemoryClaims(exec)
|
|
1187
|
+
const warning = decideMemoryOversubscription({
|
|
1188
|
+
running,
|
|
1189
|
+
incoming: { containerName, bytes: memoryLimitBytes },
|
|
1190
|
+
totalMemoryBytes,
|
|
1191
|
+
})
|
|
1192
|
+
if (warning === null) return
|
|
1193
|
+
|
|
1194
|
+
const text = formatOversubscriptionWarning(warning).join('\n')
|
|
1195
|
+
if (onWarning !== undefined) onWarning(text)
|
|
1196
|
+
else if (streamOutput) process.stderr.write(`${text}\n`)
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1053
1199
|
async function resolvePublishHost(exec: DockerExec): Promise<string> {
|
|
1054
1200
|
const result = await exec(['version', '--format', '{{.Server.Platform.Name}}'])
|
|
1055
1201
|
if (result.exitCode === 0 && result.stdout.toLowerCase().includes('docker desktop')) return '0.0.0.0'
|
|
@@ -1090,13 +1236,10 @@ export async function refreshDockerfile(
|
|
|
1090
1236
|
return { changed: true, warnings }
|
|
1091
1237
|
}
|
|
1092
1238
|
|
|
1093
|
-
// Builds the agent image with a
|
|
1094
|
-
//
|
|
1095
|
-
//
|
|
1096
|
-
//
|
|
1097
|
-
// `docker build`. The user sees one successful `typeclaw start` instead of a
|
|
1098
|
-
// buildx-specific dead end. A genuine Dockerfile error fails both paths, so the
|
|
1099
|
-
// retry costs at most one extra attempt before the real error surfaces.
|
|
1239
|
+
// Builds the agent image with a managed-buildx -> current-buildx -> legacy
|
|
1240
|
+
// fallback. A managed builder failure first releases all isolated state and
|
|
1241
|
+
// retries the unchanged BuildKit Dockerfile on Docker's current builder. Only
|
|
1242
|
+
// when that also fails do we strip BuildKit syntax and try `docker build`.
|
|
1100
1243
|
//
|
|
1101
1244
|
// Layered on top is a credential-helper recovery: typeclaw only pulls PUBLIC
|
|
1102
1245
|
// images (the BuildKit syntax frontend + the typeclaw-base FROM), but a broken
|
|
@@ -1110,56 +1253,105 @@ export async function refreshDockerfile(
|
|
|
1110
1253
|
async function runImageBuild(args: {
|
|
1111
1254
|
exec: DockerExec
|
|
1112
1255
|
cwd: string
|
|
1256
|
+
cacheScope: string
|
|
1113
1257
|
imageTag: string
|
|
1114
1258
|
buildContext: string
|
|
1115
1259
|
hasBuildx: boolean
|
|
1260
|
+
acquireBuildCache: typeof acquireManagedBuildCache | null
|
|
1261
|
+
buildCacheStateDir?: string
|
|
1116
1262
|
streamOutput: boolean
|
|
1263
|
+
onWarning: (warning: string) => void
|
|
1117
1264
|
}): Promise<{ ok: true } | { ok: false; exitCode: number; credentialHelperRetried: boolean }> {
|
|
1118
|
-
const {
|
|
1265
|
+
const {
|
|
1266
|
+
exec,
|
|
1267
|
+
cwd,
|
|
1268
|
+
cacheScope,
|
|
1269
|
+
imageTag,
|
|
1270
|
+
buildContext,
|
|
1271
|
+
hasBuildx,
|
|
1272
|
+
acquireBuildCache,
|
|
1273
|
+
buildCacheStateDir,
|
|
1274
|
+
streamOutput,
|
|
1275
|
+
onWarning,
|
|
1276
|
+
} = args
|
|
1277
|
+
let lease: ManagedBuildCacheLease | undefined
|
|
1119
1278
|
const buildArgv = (frontend: 'buildx' | 'legacy'): string[] =>
|
|
1120
1279
|
frontend === 'buildx'
|
|
1121
|
-
? [
|
|
1280
|
+
? [
|
|
1281
|
+
'buildx',
|
|
1282
|
+
'build',
|
|
1283
|
+
...(lease !== undefined ? ['--builder', lease.builder.builderName, ...lease.cacheArgs] : []),
|
|
1284
|
+
'--load',
|
|
1285
|
+
'-t',
|
|
1286
|
+
imageTag,
|
|
1287
|
+
buildContext,
|
|
1288
|
+
]
|
|
1122
1289
|
: ['build', '-t', imageTag, buildContext]
|
|
1123
1290
|
|
|
1124
1291
|
let sanitizedConfig: SanitizedDockerConfig | null = null
|
|
1125
|
-
const attempt = async (
|
|
1292
|
+
const attempt = async (
|
|
1293
|
+
frontend: 'buildx' | 'legacy',
|
|
1294
|
+
config: SanitizedDockerConfig | null,
|
|
1295
|
+
): Promise<DockerExecResult> =>
|
|
1126
1296
|
exec(buildArgv(frontend), {
|
|
1127
1297
|
cwd,
|
|
1128
1298
|
...(streamOutput ? { inheritStdio: true, captureStderr: true } : {}),
|
|
1129
1299
|
...(streamOutput ? {} : { captureStdout: false, maxCapturedStderrBytes: 32 * 1024 }),
|
|
1130
|
-
env:
|
|
1300
|
+
env: { ...(frontend === 'buildx' ? lease?.builder.env : {}), ...config?.env },
|
|
1131
1301
|
})
|
|
1302
|
+
const attemptWithCredentialRecovery = async (
|
|
1303
|
+
frontend: 'buildx' | 'legacy',
|
|
1304
|
+
config: SanitizedDockerConfig | null,
|
|
1305
|
+
): Promise<{ result: DockerExecResult; config: SanitizedDockerConfig | null }> => {
|
|
1306
|
+
let result = await attempt(frontend, config)
|
|
1307
|
+
if (result.exitCode !== 0 && config === null && isMissingDockerCredentialHelper(result.stderr)) {
|
|
1308
|
+
config = await createSanitizedDockerConfig()
|
|
1309
|
+
if (config !== null) result = await attempt(frontend, config)
|
|
1310
|
+
}
|
|
1311
|
+
return { result, config }
|
|
1312
|
+
}
|
|
1132
1313
|
|
|
1133
1314
|
try {
|
|
1134
1315
|
let frontend: 'buildx' | 'legacy' = hasBuildx ? 'buildx' : 'legacy'
|
|
1135
|
-
let result
|
|
1136
|
-
if (
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
if (
|
|
1143
|
-
|
|
1144
|
-
|
|
1316
|
+
let result: DockerExecResult
|
|
1317
|
+
if (frontend === 'buildx' && acquireBuildCache !== null) {
|
|
1318
|
+
const acquired = await acquireBuildCache({
|
|
1319
|
+
exec,
|
|
1320
|
+
cacheScope,
|
|
1321
|
+
...(buildCacheStateDir !== undefined ? { stateDir: buildCacheStateDir } : {}),
|
|
1322
|
+
})
|
|
1323
|
+
if (acquired.ok) {
|
|
1324
|
+
lease = acquired.lease
|
|
1325
|
+
for (const warning of acquired.warnings) onWarning(warning)
|
|
1326
|
+
} else onWarning(`managed build cache unavailable; using Docker's current builder: ${acquired.reason}`)
|
|
1327
|
+
}
|
|
1328
|
+
if (frontend === 'buildx' && lease !== undefined) {
|
|
1329
|
+
const managedLease = lease
|
|
1330
|
+
let managedSuccess = false
|
|
1331
|
+
try {
|
|
1332
|
+
;({ result, config: sanitizedConfig } = await attemptWithCredentialRecovery(frontend, sanitizedConfig))
|
|
1333
|
+
managedSuccess = result.exitCode === 0
|
|
1334
|
+
} finally {
|
|
1335
|
+
const finished = await managedLease.finish(managedSuccess)
|
|
1336
|
+
for (const warning of finished.warnings) onWarning(warning)
|
|
1337
|
+
lease = undefined
|
|
1145
1338
|
}
|
|
1339
|
+
if (result.exitCode !== 0) {
|
|
1340
|
+
onWarning("managed buildx build failed; retrying with Docker's current builder")
|
|
1341
|
+
;({ result, config: sanitizedConfig } = await attemptWithCredentialRecovery(frontend, sanitizedConfig))
|
|
1342
|
+
}
|
|
1343
|
+
} else {
|
|
1344
|
+
;({ result, config: sanitizedConfig } = await attemptWithCredentialRecovery(frontend, sanitizedConfig))
|
|
1146
1345
|
}
|
|
1346
|
+
if (result.exitCode === 0) return { ok: true }
|
|
1147
1347
|
|
|
1148
1348
|
if (frontend === 'buildx') {
|
|
1149
|
-
//
|
|
1150
|
-
//
|
|
1151
|
-
// up with an image. The sanitized config (if any) carries into the retry.
|
|
1349
|
+
// Both managed and current buildx failed. Strip BuildKit-only syntax only
|
|
1350
|
+
// now, preserving the current builder as the first compatibility fallback.
|
|
1152
1351
|
await refreshDockerfile(cwd, { buildKit: false })
|
|
1153
1352
|
frontend = 'legacy'
|
|
1154
|
-
result = await
|
|
1353
|
+
;({ result, config: sanitizedConfig } = await attemptWithCredentialRecovery(frontend, sanitizedConfig))
|
|
1155
1354
|
if (result.exitCode === 0) return { ok: true }
|
|
1156
|
-
if (sanitizedConfig === null && isMissingDockerCredentialHelper(result.stderr)) {
|
|
1157
|
-
sanitizedConfig = await createSanitizedDockerConfig()
|
|
1158
|
-
if (sanitizedConfig) {
|
|
1159
|
-
result = await attempt(frontend)
|
|
1160
|
-
if (result.exitCode === 0) return { ok: true }
|
|
1161
|
-
}
|
|
1162
|
-
}
|
|
1163
1355
|
}
|
|
1164
1356
|
|
|
1165
1357
|
if (sanitizedConfig !== null && streamOutput) {
|
|
@@ -1223,9 +1415,45 @@ export async function refreshGitignore(cwd: string): Promise<void> {
|
|
|
1223
1415
|
// the migration write with a commit on every read path, not only here.
|
|
1224
1416
|
export const commitSystemFile = commitSystemFileShared
|
|
1225
1417
|
|
|
1226
|
-
async function
|
|
1227
|
-
const result = await exec(['image', 'inspect', tag])
|
|
1228
|
-
|
|
1418
|
+
async function inspectImageId(exec: DockerExec, tag: string): Promise<string | null> {
|
|
1419
|
+
const result = await exec(['image', 'inspect', '--format', '{{.Id}}', tag])
|
|
1420
|
+
if (result.exitCode !== 0) return null
|
|
1421
|
+
return result.stdout.trim() || null
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
async function protectImageForRebuild(
|
|
1425
|
+
exec: DockerExec,
|
|
1426
|
+
imageTag: string,
|
|
1427
|
+
previousImageId: string | null,
|
|
1428
|
+
): Promise<{ tag: string | null; warning: string | null }> {
|
|
1429
|
+
if (previousImageId === null) return { tag: null, warning: null }
|
|
1430
|
+
|
|
1431
|
+
const protectionTag = `${imageTag}:rebuild-${randomBytes(8).toString('hex')}`
|
|
1432
|
+
try {
|
|
1433
|
+
const result = await exec(['image', 'tag', previousImageId, protectionTag])
|
|
1434
|
+
if (result.exitCode === 0) return { tag: protectionTag, warning: null }
|
|
1435
|
+
const detail = sanitizeDockerStderr(result.stderr) || `docker image tag exited with code ${result.exitCode}`
|
|
1436
|
+
return {
|
|
1437
|
+
tag: null,
|
|
1438
|
+
warning: `Could not protect existing Docker image ${previousImageId} for cleanup: ${detail}`,
|
|
1439
|
+
}
|
|
1440
|
+
} catch (error) {
|
|
1441
|
+
return {
|
|
1442
|
+
tag: null,
|
|
1443
|
+
warning: `Could not protect existing Docker image ${previousImageId} for cleanup: ${error instanceof Error ? error.message : String(error)}`,
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
async function releaseImageProtection(exec: DockerExec, protectionTag: string): Promise<string | null> {
|
|
1449
|
+
try {
|
|
1450
|
+
const result = await exec(['image', 'rm', '--no-prune', protectionTag])
|
|
1451
|
+
if (result.exitCode === 0) return null
|
|
1452
|
+
const detail = sanitizeDockerStderr(result.stderr) || `docker image rm exited with code ${result.exitCode}`
|
|
1453
|
+
return `Could not remove temporary Docker image protection tag ${protectionTag}: ${detail}`
|
|
1454
|
+
} catch (error) {
|
|
1455
|
+
return `Could not remove temporary Docker image protection tag ${protectionTag}: ${error instanceof Error ? error.message : String(error)}`
|
|
1456
|
+
}
|
|
1229
1457
|
}
|
|
1230
1458
|
|
|
1231
1459
|
type InspectedState =
|
|
@@ -1261,8 +1489,7 @@ async function inspectContainer(exec: DockerExec, name: string): Promise<Inspect
|
|
|
1261
1489
|
// container record behind, and start()'s own port-TOCTOU retry triggers
|
|
1262
1490
|
// this path against that corpse).
|
|
1263
1491
|
//
|
|
1264
|
-
// cleanupRunCorpse refuses to touch a running container and uses non-force rm
|
|
1265
|
-
// so a container that starts after its inspect probe is also preserved. A concurrent
|
|
1492
|
+
// cleanupRunCorpse refuses to touch a running container and uses non-force rm. A concurrent
|
|
1266
1493
|
// legitimate start of the same name (or a foreign-but-named container the
|
|
1267
1494
|
// user wants alive) is surfaced as a hard failure rather than silently
|
|
1268
1495
|
// killed. 'stuck' likewise surfaces — a wedged daemon that won't drain a
|
package/src/container/stop.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { DEFAULT_LOG_RETENTION_DAYS, loadConfigSync } from '@/config'
|
|
|
2
2
|
import { isDaemonReachable, send as sendToDaemon } from '@/hostd/client'
|
|
3
3
|
|
|
4
4
|
import { type AgentOperationLease, type WithAgentOperationLock, withAgentOperationLock } from './agent-operation-lock'
|
|
5
|
-
import { archiveContainerLogs, type DockerLogArchiver } from './log-archive'
|
|
5
|
+
import { archiveContainerLogs, dockerLogsUnavailableWarning, type DockerLogArchiver } from './log-archive'
|
|
6
6
|
import {
|
|
7
7
|
classifyRmStderr,
|
|
8
8
|
containerNameFromCwd,
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
sanitizeDockerStderr,
|
|
14
14
|
waitForRemoval,
|
|
15
15
|
} from './shared'
|
|
16
|
+
import { describeUnremovableContainer } from './verify-running'
|
|
16
17
|
|
|
17
18
|
export type StopPlan = {
|
|
18
19
|
containerName: string
|
|
@@ -26,6 +27,7 @@ export type StopOptions = {
|
|
|
26
27
|
archiveLogs?: DockerLogArchiver
|
|
27
28
|
operationLock?: WithAgentOperationLock
|
|
28
29
|
operationLease?: AgentOperationLease
|
|
30
|
+
onWarning?: (warning: string) => void
|
|
29
31
|
}
|
|
30
32
|
|
|
31
33
|
export async function stop(options: StopOptions): Promise<StopResult> {
|
|
@@ -45,6 +47,7 @@ async function runStop({
|
|
|
45
47
|
cwd,
|
|
46
48
|
exec = defaultDockerExec,
|
|
47
49
|
archiveLogs = archiveContainerLogs,
|
|
50
|
+
onWarning,
|
|
48
51
|
}: StopOptions): Promise<StopResult> {
|
|
49
52
|
const { containerName } = planStop(cwd)
|
|
50
53
|
|
|
@@ -89,10 +92,13 @@ async function runStop({
|
|
|
89
92
|
|
|
90
93
|
const archive = await archiveLogs({ agentDir: cwd, containerId, retentionDays })
|
|
91
94
|
if (!archive.ok) {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
+
if (archive.kind === 'failed') {
|
|
96
|
+
return {
|
|
97
|
+
ok: false,
|
|
98
|
+
reason: `Could not archive logs for container ${containerId}: ${archive.reason}. Preserving the Docker container as the source of truth.`,
|
|
99
|
+
}
|
|
95
100
|
}
|
|
101
|
+
onWarning?.(dockerLogsUnavailableWarning(containerId))
|
|
96
102
|
}
|
|
97
103
|
|
|
98
104
|
// Containers run without `--rm`, so `docker stop` only stops them. Remove
|
|
@@ -123,19 +129,19 @@ async function runStop({
|
|
|
123
129
|
}
|
|
124
130
|
const kind = classifyRmStderr(rmResult.stderr)
|
|
125
131
|
if (kind === null) {
|
|
126
|
-
|
|
132
|
+
const reason = `docker rm failed: ${sanitizeDockerStderr(rmResult.stderr) || 'no stderr'}`
|
|
133
|
+
const diagnosis = await describeUnremovableContainer(exec, containerId)
|
|
134
|
+
return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
|
|
127
135
|
}
|
|
128
136
|
if (kind === 'in-progress' && !(await waitForRemoval(exec, containerName))) {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
}
|
|
137
|
+
const reason = `Container ${containerName} is still being removed by docker after 10s.`
|
|
138
|
+
const diagnosis = await describeUnremovableContainer(exec, containerId)
|
|
139
|
+
return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
|
|
133
140
|
}
|
|
134
141
|
} else if (!(await waitForRemoval(exec, containerName))) {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
}
|
|
142
|
+
const reason = `Container ${containerName} is still being removed by docker after 10s.`
|
|
143
|
+
const diagnosis = await describeUnremovableContainer(exec, containerId)
|
|
144
|
+
return { ok: false, reason: diagnosis === null ? reason : `${reason} ${diagnosis}` }
|
|
139
145
|
}
|
|
140
146
|
|
|
141
147
|
return { ok: true, containerName, running }
|
|
@@ -101,6 +101,18 @@ export async function probeContainer(exec: DockerExec, name: string): Promise<Co
|
|
|
101
101
|
return { kind: 'daemon-error', detail }
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
export async function describeUnremovableContainer(exec: DockerExec, containerRef: string): Promise<string | null> {
|
|
105
|
+
const probe = await probeContainer(exec, containerRef)
|
|
106
|
+
if (probe.kind !== 'status') return null
|
|
107
|
+
if (probe.status === 'removing') {
|
|
108
|
+
return 'Docker is still removing this container, but the removal is not draining. docker rm, including --force, cannot re-enter an in-flight removal. Restart Docker or OrbStack; Docker will restore the container as dead, and TypeClaw can remove it on the next start or restart.'
|
|
109
|
+
}
|
|
110
|
+
if (probe.status === 'dead') {
|
|
111
|
+
return 'Docker marked this container dead because it could not remove its filesystem. Resolve the busy mount or restart Docker, then retry.'
|
|
112
|
+
}
|
|
113
|
+
return null
|
|
114
|
+
}
|
|
115
|
+
|
|
104
116
|
function isLifeStatus(value: string): value is ContainerLifeStatus {
|
|
105
117
|
return (
|
|
106
118
|
value === 'running' ||
|
package/src/cron/scheduler.ts
CHANGED
|
@@ -58,9 +58,17 @@ const consoleLogger: SchedulerLogger = {
|
|
|
58
58
|
error: (m) => console.error(m),
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
|
|
61
|
+
// Timers count down against a monotonic clock, while cron occurrences are wall-clock
|
|
62
|
+
// instants. A machine or VM suspension advances the latter but not the former, so a
|
|
63
|
+
// single day-long timer would resume still holding almost its full delay and fire late
|
|
64
|
+
// by the suspension. There is no portable in-process resume callback to correct that;
|
|
65
|
+
// periodically re-checking the wall clock is the only reliable mechanism.
|
|
66
|
+
//
|
|
67
|
+
// One minute bounds resumed-host convergence while keeping the idle cost modest: each
|
|
68
|
+
// enabled job wakes once per minute (30 jobs are 30 wake-ups/minute, or 43,200/day).
|
|
69
|
+
// An intermediate wake-up only re-arms the immutable occurrence below; it never fires
|
|
70
|
+
// early, because the callback dispatches only after the wall clock reaches nextFire.
|
|
71
|
+
const maxArmMs = 60 * 1000
|
|
64
72
|
|
|
65
73
|
export function createScheduler({
|
|
66
74
|
jobs,
|
|
@@ -119,7 +127,7 @@ export function createScheduler({
|
|
|
119
127
|
fire(live)
|
|
120
128
|
scheduleNext(id)
|
|
121
129
|
},
|
|
122
|
-
Math.min(delay,
|
|
130
|
+
Math.min(delay, maxArmMs),
|
|
123
131
|
)
|
|
124
132
|
handles.set(id, handle)
|
|
125
133
|
}
|