typeclaw 0.39.5 → 0.39.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/todo/continuation-policy.ts +8 -0
- package/src/agent/todo/continuation-state.ts +41 -7
- package/src/agent/todo/continuation-wiring.ts +26 -0
- package/src/bundled-plugins/memory/index.ts +2 -0
- package/src/bundled-plugins/memory/memory-logger.ts +4 -4
- package/src/bundled-plugins/memory/search-command.ts +133 -0
- package/src/channels/engagement.ts +17 -2
- package/src/channels/router.ts +60 -1
- package/src/cli/command-meta.ts +7 -16
- package/src/cli/compose.ts +1 -1
- package/src/cli/cron.ts +1 -1
- package/src/cli/doctor.ts +1 -1
- package/src/cli/dreams.ts +1 -1
- package/src/cli/index.ts +1 -1
- package/src/cli/inspect.ts +1 -1
- package/src/cli/memory.ts +85 -0
- package/src/cli/model.ts +1 -1
- package/src/cli/ui.ts +32 -22
- package/src/inspect/index.ts +2 -3
- package/src/role-claim/client.ts +6 -1
- package/src/run/codex-fetch-observer.ts +26 -10
- package/src/run/fatal-guard.ts +230 -0
- package/src/run/index.ts +123 -27
- package/src/secrets/oauth-xai.ts +8 -2
- package/src/secrets/webex-renewal.ts +3 -0
- package/src/server/index.ts +8 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { defineCommand } from 'citty'
|
|
2
|
+
|
|
3
|
+
import { proxyContainerCommand } from './container-command-client'
|
|
4
|
+
import { dreamsCommand } from './dreams'
|
|
5
|
+
import { requireAgentDir } from './require-agent-dir'
|
|
6
|
+
import { errorLine } from './ui'
|
|
7
|
+
|
|
8
|
+
const MEMORY_SEARCH_COMMAND = 'memory-search'
|
|
9
|
+
|
|
10
|
+
const searchSub = defineCommand({
|
|
11
|
+
meta: {
|
|
12
|
+
name: 'search',
|
|
13
|
+
description: 'vector-search the agent long-term memory inside the running container (host stage)',
|
|
14
|
+
},
|
|
15
|
+
args: {
|
|
16
|
+
query: {
|
|
17
|
+
type: 'positional',
|
|
18
|
+
description: 'the search query',
|
|
19
|
+
required: true,
|
|
20
|
+
},
|
|
21
|
+
topK: {
|
|
22
|
+
type: 'string',
|
|
23
|
+
description: 'return at most N ranked results (default 10)',
|
|
24
|
+
},
|
|
25
|
+
json: {
|
|
26
|
+
type: 'boolean',
|
|
27
|
+
description: 'emit raw JSON results instead of formatted text',
|
|
28
|
+
default: false,
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
async run({ args }) {
|
|
32
|
+
const cwd = requireAgentDir()
|
|
33
|
+
const topK = parseTopK(args.topK)
|
|
34
|
+
if (topK === 'invalid') {
|
|
35
|
+
process.stderr.write(`${errorLine('--topK must be a positive integer')}\n`)
|
|
36
|
+
process.exit(2)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const result = await proxyContainerCommand({
|
|
40
|
+
agentDir: cwd,
|
|
41
|
+
commandName: MEMORY_SEARCH_COMMAND,
|
|
42
|
+
args: {
|
|
43
|
+
query: args.query,
|
|
44
|
+
json: args.json === true,
|
|
45
|
+
...(topK !== undefined ? { topK } : {}),
|
|
46
|
+
},
|
|
47
|
+
stdout: nodeWritable(process.stdout),
|
|
48
|
+
stderr: nodeWritable(process.stderr),
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
if (!result.ok) {
|
|
52
|
+
process.stderr.write(`${errorLine(result.message)}\n`)
|
|
53
|
+
process.exit(result.exitCode)
|
|
54
|
+
}
|
|
55
|
+
process.exit(result.exitCode)
|
|
56
|
+
},
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
export const memoryCommand = defineCommand({
|
|
60
|
+
meta: {
|
|
61
|
+
name: 'memory',
|
|
62
|
+
description: "browse and search the agent's long-term memory",
|
|
63
|
+
},
|
|
64
|
+
subCommands: {
|
|
65
|
+
dreams: dreamsCommand,
|
|
66
|
+
search: searchSub,
|
|
67
|
+
},
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
function parseTopK(raw: unknown): number | undefined | 'invalid' {
|
|
71
|
+
if (typeof raw !== 'string') return undefined
|
|
72
|
+
// Whole-string match, not Number.parseInt — parseInt silently truncates
|
|
73
|
+
// "1.5" to 1 and "10abc" to 10, proxying invalid input as a valid-looking
|
|
74
|
+
// topK. The container's max-50 zod check stays as the upper-bound guard.
|
|
75
|
+
if (!/^[1-9]\d*$/.test(raw)) return 'invalid'
|
|
76
|
+
return Number(raw)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function nodeWritable(stream: NodeJS.WriteStream): WritableStream<Uint8Array> {
|
|
80
|
+
return new WritableStream<Uint8Array>({
|
|
81
|
+
write(chunk) {
|
|
82
|
+
stream.write(chunk)
|
|
83
|
+
},
|
|
84
|
+
})
|
|
85
|
+
}
|
package/src/cli/model.ts
CHANGED
|
@@ -277,7 +277,7 @@ const listSub = defineCommand({
|
|
|
277
277
|
export const modelCommand = defineCommand({
|
|
278
278
|
meta: {
|
|
279
279
|
name: 'model',
|
|
280
|
-
description: 'manage model profiles in typeclaw.json (models.default,
|
|
280
|
+
description: 'manage model profiles in typeclaw.json (models.default, ...)',
|
|
281
281
|
},
|
|
282
282
|
subCommands: {
|
|
283
283
|
set: setSub,
|
package/src/cli/ui.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { emitKeypressEvents as nodeEmitKeypressEvents } from 'node:readline'
|
|
1
2
|
import { styleText } from 'node:util'
|
|
2
3
|
|
|
3
4
|
import { cancel, intro, isCancel, log, note, outro, spinner as clackSpinner } from '@clack/prompts'
|
|
@@ -8,29 +9,38 @@ import { COMPACT_WORDMARK, WORDMARK_LINES, WORDMARK_WIDTH } from '@/shared/wordm
|
|
|
8
9
|
|
|
9
10
|
export { cancel, intro, isCancel, log, note, outro }
|
|
10
11
|
|
|
11
|
-
type ClackInput = Pick<NodeJS.ReadStream, 'isTTY' | '
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
|
|
12
|
+
type ClackInput = Pick<NodeJS.ReadStream, 'isTTY' | 'resume'>
|
|
13
|
+
|
|
14
|
+
type KeypressInterface = { escapeCodeTimeout?: number }
|
|
15
|
+
|
|
16
|
+
type PrepareStdinDeps = {
|
|
17
|
+
emitKeypressEvents?: (stream: NodeJS.ReadableStream, iface: KeypressInterface) => void
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// @clack/core@1.2.0 builds its readline interface with escapeCodeTimeout: 50 so
|
|
21
|
+
// a bare Esc (the cancel key) resolves fast. readline's keypress decoder binds
|
|
22
|
+
// that timeout ONCE, when emitKeypressEvents first installs it on the stream, and
|
|
23
|
+
// is a no-op on every later call — so pre-arming here without it would lock in
|
|
24
|
+
// readline's 500ms default and clack's own 50ms could never take effect.
|
|
25
|
+
const CLACK_ESCAPE_CODE_TIMEOUT_MS = 50
|
|
26
|
+
|
|
27
|
+
// Arm readline's keypress decoder on stdin before a clack picker reads it. Over
|
|
28
|
+
// an SSH pseudo-TTY, arrow keys arrive as raw `\x1b[A`/`\x1b[B` bytes and only
|
|
29
|
+
// become the `{ name: 'up' }` keypress events clack navigates on once that
|
|
30
|
+
// decoder is installed on THIS stream. After a prior raw-mode 'data' consumer or
|
|
31
|
+
// a pi-tui detach (ProcessTerminal.stop() pauses stdin), Bun does NOT re-arm the
|
|
32
|
+
// decoder when clack later calls createInterface({ terminal: true }), so the
|
|
33
|
+
// picker renders but arrows echo as `^[[A` and never advance. We pass clack's
|
|
34
|
+
// own escapeCodeTimeout so the pre-arm matches what clack would have set; raw
|
|
35
|
+
// mode is left to clack. Never pause() — a paused process.stdin does not
|
|
36
|
+
// reliably re-flow under Bun.
|
|
37
|
+
export function prepareStdinForClack(input: ClackInput = process.stdin, deps: PrepareStdinDeps = {}): void {
|
|
24
38
|
if (!input.isTTY) return
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
} catch {
|
|
31
|
-
/* terminal already torn down */
|
|
32
|
-
}
|
|
33
|
-
}
|
|
39
|
+
const emitKeypressEvents =
|
|
40
|
+
deps.emitKeypressEvents ??
|
|
41
|
+
((stream: NodeJS.ReadableStream, iface: KeypressInterface) =>
|
|
42
|
+
nodeEmitKeypressEvents(stream, iface as unknown as Parameters<typeof nodeEmitKeypressEvents>[1]))
|
|
43
|
+
emitKeypressEvents(input as unknown as NodeJS.ReadableStream, { escapeCodeTimeout: CLACK_ESCAPE_CODE_TIMEOUT_MS })
|
|
34
44
|
input.resume()
|
|
35
45
|
}
|
|
36
46
|
|
package/src/inspect/index.ts
CHANGED
|
@@ -85,9 +85,8 @@ export type ResolveInspectResult = { ok: true; target: InspectTarget } | { ok: f
|
|
|
85
85
|
|
|
86
86
|
// Picker phase, split out from the streaming phase so the tail scope is created
|
|
87
87
|
// AFTER the picker, never before. A raw-mode 'data' listener active during the
|
|
88
|
-
// Clack picker
|
|
89
|
-
//
|
|
90
|
-
// "esc does nothing" regression.
|
|
88
|
+
// Clack picker fights Clack for stdin and leaves the later stream in cooked mode
|
|
89
|
+
// — that was the recurring "esc does nothing" regression.
|
|
91
90
|
export async function resolveInspectTarget(opts: Omit<RunInspectOptions, 'signal'>): Promise<ResolveInspectResult> {
|
|
92
91
|
const filterResult = parseFilter(opts.filter)
|
|
93
92
|
if (!filterResult.ok) return { ok: false, exitCode: 2, reason: filterResult.reason }
|
package/src/role-claim/client.ts
CHANGED
|
@@ -13,6 +13,10 @@ export type ClaimSessionOptions = {
|
|
|
13
13
|
role: string
|
|
14
14
|
channel?: string
|
|
15
15
|
ttlMs?: number
|
|
16
|
+
// Defaults to connectTimeoutMs. Split out so a loaded host can't let the
|
|
17
|
+
// open (transport) phase exhaust the budget meant for the connected phase
|
|
18
|
+
// and surface the wrong timeout error. See waitForOpen/waitForConnected.
|
|
19
|
+
openTimeoutMs?: number
|
|
16
20
|
connectTimeoutMs?: number
|
|
17
21
|
onStarted?: (payload: ClaimStartedPayload) => void
|
|
18
22
|
}
|
|
@@ -28,11 +32,12 @@ const DEFAULT_CONNECT_TIMEOUT_MS = 30_000
|
|
|
28
32
|
export async function runClaimSession(opts: ClaimSessionOptions): Promise<ClaimSessionResult> {
|
|
29
33
|
const ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS
|
|
30
34
|
const connectTimeoutMs = opts.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS
|
|
35
|
+
const openTimeoutMs = opts.openTimeoutMs ?? connectTimeoutMs
|
|
31
36
|
const code = generateClaimCode()
|
|
32
37
|
|
|
33
38
|
const ws = new WebSocket(opts.url)
|
|
34
39
|
const displayUrl = redactUrl(opts.url)
|
|
35
|
-
await waitForOpen(ws, displayUrl,
|
|
40
|
+
await waitForOpen(ws, displayUrl, openTimeoutMs)
|
|
36
41
|
await waitForConnected(ws, displayUrl, connectTimeoutMs)
|
|
37
42
|
|
|
38
43
|
try {
|
|
@@ -87,9 +87,16 @@ const consoleLogger: CodexFetchObserverLogger = {
|
|
|
87
87
|
|
|
88
88
|
type InstallState = {
|
|
89
89
|
originalFetch: typeof fetch
|
|
90
|
-
|
|
90
|
+
wrapped: typeof fetch
|
|
91
|
+
claimants: number
|
|
91
92
|
}
|
|
92
93
|
|
|
94
|
+
// Ref-counted so multiple agents in one process (compose, tests) share one
|
|
95
|
+
// `globalThis.fetch` wrapper: the FIRST install wraps fetch, each later install
|
|
96
|
+
// just joins, and `globalThis.fetch` is restored only when the LAST claimant
|
|
97
|
+
// releases. A bare singleton would let one agent's release (e.g. a second
|
|
98
|
+
// agent's boot-failure cleanup) tear down the observer out from under another
|
|
99
|
+
// still-running agent.
|
|
93
100
|
let installed: InstallState | null = null
|
|
94
101
|
|
|
95
102
|
// Returns true when the request is for the Codex Responses endpoint and we
|
|
@@ -340,8 +347,8 @@ export function installCodexFetchObserver(opts: CodexFetchObserverOptions = {}):
|
|
|
340
347
|
}
|
|
341
348
|
const logger = opts.logger ?? consoleLogger
|
|
342
349
|
if (installed !== null) {
|
|
343
|
-
|
|
344
|
-
return installed.
|
|
350
|
+
installed.claimants++
|
|
351
|
+
return makeRelease(installed.wrapped)
|
|
345
352
|
}
|
|
346
353
|
|
|
347
354
|
const codexHost = opts.codexHost ?? DEFAULT_CODEX_HOST
|
|
@@ -421,14 +428,23 @@ export function installCodexFetchObserver(opts: CodexFetchObserverOptions = {}):
|
|
|
421
428
|
|
|
422
429
|
globalThis.fetch = wrapped
|
|
423
430
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
431
|
+
installed = { originalFetch, wrapped, claimants: 1 }
|
|
432
|
+
return makeRelease(wrapped)
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// Each install call gets its own idempotent release. The shared
|
|
436
|
+
// `globalThis.fetch` wrapper is restored only when the final claimant releases,
|
|
437
|
+
// so one agent releasing never disturbs another's still-active observer.
|
|
438
|
+
function makeRelease(wrapped: typeof fetch): () => void {
|
|
439
|
+
let released = false
|
|
440
|
+
return () => {
|
|
441
|
+
if (released || installed === null || installed.wrapped !== wrapped) return
|
|
442
|
+
released = true
|
|
443
|
+
installed.claimants--
|
|
444
|
+
if (installed.claimants > 0) return
|
|
445
|
+
if (globalThis.fetch === installed.wrapped) {
|
|
446
|
+
globalThis.fetch = installed.originalFetch
|
|
428
447
|
}
|
|
429
448
|
installed = null
|
|
430
449
|
}
|
|
431
|
-
|
|
432
|
-
installed = { originalFetch, uninstall }
|
|
433
|
-
return uninstall
|
|
434
450
|
}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
// Process-level crash guard for the container stage. The incident: a Webex KMS
|
|
2
|
+
// request rejected 30s AFTER `listener.on('error')` already fired, so it escaped
|
|
3
|
+
// the adapter as an `unhandledRejection` — which Bun terminates the process on
|
|
4
|
+
// by default, taking Slack/Discord/cron/TUI/websocket down with one flaky dep.
|
|
5
|
+
//
|
|
6
|
+
// The guard NEVER calls `process.exit`. Policy (see `classifyFatalError`) is
|
|
7
|
+
// conservative so "never crash" does not become "pretend the process is healthy":
|
|
8
|
+
// a KNOWN channel-dependency `unhandledRejection` degrades just that channel; an
|
|
9
|
+
// `uncaughtException` or UNKNOWN `unhandledRejection` requests a supervised
|
|
10
|
+
// container restart (corrupt state is safer replaced than swallowed) while
|
|
11
|
+
// continuing best-effort until the host replaces us — it still never self-exits.
|
|
12
|
+
// Restart requests are rate-limited so a per-tick fault can't hammer the daemon.
|
|
13
|
+
|
|
14
|
+
export type FatalGuardKind = 'unhandledRejection' | 'uncaughtException'
|
|
15
|
+
|
|
16
|
+
export type FatalGuardDecision =
|
|
17
|
+
| { action: 'continue'; reason: string; scope?: string }
|
|
18
|
+
| { action: 'restart'; reason: string; scope?: string }
|
|
19
|
+
|
|
20
|
+
export type FatalGuardLogger = {
|
|
21
|
+
warn: (msg: string) => void
|
|
22
|
+
error: (msg: string) => void
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export type FatalGuardOptions = {
|
|
26
|
+
logger?: FatalGuardLogger
|
|
27
|
+
// Ask the host daemon to bounce this container. Returns ok=false (never
|
|
28
|
+
// throws) when no host control endpoint is configured — the guard then logs
|
|
29
|
+
// and continues degraded rather than exiting.
|
|
30
|
+
requestRestart?: (reason: string) => Promise<{ ok: boolean; reason?: string }>
|
|
31
|
+
// Called for a `continue` decision carrying a channel scope, so the channel
|
|
32
|
+
// can be visibly marked degraded (e.g. log/status) instead of silently
|
|
33
|
+
// swallowing the error.
|
|
34
|
+
onDegrade?: (scope: string, reason: string) => void
|
|
35
|
+
now?: () => number
|
|
36
|
+
// Minimum spacing between restart requests. A fault re-firing on every event
|
|
37
|
+
// loop tick must not spam the host daemon. Default 60s.
|
|
38
|
+
restartMinIntervalMs?: number
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type FatalGuard = {
|
|
42
|
+
// Exposed for deterministic tests: invoke the policy without emitting a real
|
|
43
|
+
// process event. Production paths go through the installed listeners.
|
|
44
|
+
handle: (kind: FatalGuardKind, error: unknown) => void
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const DEFAULT_RESTART_MIN_INTERVAL_MS = 60_000
|
|
48
|
+
|
|
49
|
+
const consoleLogger: FatalGuardLogger = {
|
|
50
|
+
warn: (m) => console.warn(m),
|
|
51
|
+
error: (m) => console.error(m),
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// One process gets exactly one pair of OS-level listeners no matter how many
|
|
55
|
+
// callers install a guard (tests, `typeclaw compose`, an embedding host). Each
|
|
56
|
+
// caller's handler is refcounted into this set and receives every event; the
|
|
57
|
+
// listeners are attached on the first install and detached when the last
|
|
58
|
+
// disposer runs. A per-caller `process.on` would otherwise stack N copies that
|
|
59
|
+
// all fire on one rejection and survive a single `dispose()`.
|
|
60
|
+
type Installed = {
|
|
61
|
+
handlers: Set<(kind: FatalGuardKind, error: unknown) => void>
|
|
62
|
+
onUnhandledRejection: (reason: unknown) => void
|
|
63
|
+
onUncaughtException: (error: unknown) => void
|
|
64
|
+
}
|
|
65
|
+
let shared: Installed | null = null
|
|
66
|
+
|
|
67
|
+
export function installFatalGuard(options: FatalGuardOptions = {}): { guard: FatalGuard; dispose: () => void } {
|
|
68
|
+
const logger = options.logger ?? consoleLogger
|
|
69
|
+
const now = options.now ?? Date.now
|
|
70
|
+
const restartMinIntervalMs = options.restartMinIntervalMs ?? DEFAULT_RESTART_MIN_INTERVAL_MS
|
|
71
|
+
let lastRestartRequestAt: number | null = null
|
|
72
|
+
|
|
73
|
+
const handle = (kind: FatalGuardKind, error: unknown): void => {
|
|
74
|
+
// The guard body must never throw — a throwing handler re-enters the very
|
|
75
|
+
// failure mode it exists to contain. Everything is wrapped, with a bare
|
|
76
|
+
// console.error as the last-resort fallback.
|
|
77
|
+
try {
|
|
78
|
+
const decision = classifyFatalError(kind, error)
|
|
79
|
+
const detail = describeError(error)
|
|
80
|
+
logger.error(
|
|
81
|
+
`[fatal-guard] ${kind}: ${decision.reason}${decision.scope !== undefined ? ` scope=${decision.scope}` : ''} action=${decision.action} :: ${detail}`,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
if (decision.action === 'continue') {
|
|
85
|
+
if (decision.scope !== undefined) {
|
|
86
|
+
try {
|
|
87
|
+
options.onDegrade?.(decision.scope, decision.reason)
|
|
88
|
+
} catch (degradeErr) {
|
|
89
|
+
logger.warn(`[fatal-guard] onDegrade(${decision.scope}) threw: ${describeError(degradeErr)}`)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
requestRestart(decision.reason)
|
|
96
|
+
} catch (handlerErr) {
|
|
97
|
+
try {
|
|
98
|
+
console.error(`[fatal-guard] handler failed: ${describeError(handlerErr)}`)
|
|
99
|
+
} catch {
|
|
100
|
+
// Nothing left to do; swallowing here is the whole point.
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const requestRestart = (reason: string): void => {
|
|
106
|
+
if (options.requestRestart === undefined) {
|
|
107
|
+
logger.warn(`[fatal-guard] restart requested (${reason}) but no host control endpoint; continuing degraded`)
|
|
108
|
+
return
|
|
109
|
+
}
|
|
110
|
+
const at = now()
|
|
111
|
+
if (lastRestartRequestAt !== null && at - lastRestartRequestAt < restartMinIntervalMs) {
|
|
112
|
+
logger.warn(`[fatal-guard] restart requested (${reason}) but rate-limited; continuing degraded`)
|
|
113
|
+
return
|
|
114
|
+
}
|
|
115
|
+
lastRestartRequestAt = at
|
|
116
|
+
void options
|
|
117
|
+
.requestRestart(reason)
|
|
118
|
+
.then((result) => {
|
|
119
|
+
if (result.ok) {
|
|
120
|
+
logger.warn(`[fatal-guard] container restart requested: ${reason}`)
|
|
121
|
+
} else {
|
|
122
|
+
logger.warn(
|
|
123
|
+
`[fatal-guard] container restart unavailable (${result.reason ?? 'unknown'}); continuing degraded`,
|
|
124
|
+
)
|
|
125
|
+
}
|
|
126
|
+
})
|
|
127
|
+
.catch((restartErr) => {
|
|
128
|
+
logger.warn(`[fatal-guard] container restart request threw: ${describeError(restartErr)}; continuing degraded`)
|
|
129
|
+
})
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (shared === null) {
|
|
133
|
+
const onUnhandledRejection = (reason: unknown): void => {
|
|
134
|
+
for (const h of shared?.handlers ?? []) h('unhandledRejection', reason)
|
|
135
|
+
}
|
|
136
|
+
const onUncaughtException = (error: unknown): void => {
|
|
137
|
+
for (const h of shared?.handlers ?? []) h('uncaughtException', error)
|
|
138
|
+
}
|
|
139
|
+
shared = { handlers: new Set(), onUnhandledRejection, onUncaughtException }
|
|
140
|
+
process.on('unhandledRejection', onUnhandledRejection)
|
|
141
|
+
process.on('uncaughtException', onUncaughtException)
|
|
142
|
+
}
|
|
143
|
+
shared.handlers.add(handle)
|
|
144
|
+
|
|
145
|
+
let disposed = false
|
|
146
|
+
const dispose = (): void => {
|
|
147
|
+
if (disposed || shared === null) return
|
|
148
|
+
disposed = true
|
|
149
|
+
shared.handlers.delete(handle)
|
|
150
|
+
if (shared.handlers.size === 0) {
|
|
151
|
+
process.off('unhandledRejection', shared.onUnhandledRejection)
|
|
152
|
+
process.off('uncaughtException', shared.onUncaughtException)
|
|
153
|
+
shared = null
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return { guard: { handle }, dispose }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Conservative by design. Only errors with positive evidence that they came
|
|
161
|
+
// from a known external channel dependency are recovered in place; everything
|
|
162
|
+
// else escalates to a supervised restart. Broad rules ("any Error with a code",
|
|
163
|
+
// "any network error") are intentionally avoided — they would mask real bugs.
|
|
164
|
+
export function classifyFatalError(kind: FatalGuardKind, error: unknown): FatalGuardDecision {
|
|
165
|
+
if (kind === 'uncaughtException') {
|
|
166
|
+
// A synchronous throw that reached the top of the stack means an unknown
|
|
167
|
+
// code path is in an undefined state. Replace, don't swallow.
|
|
168
|
+
return { action: 'restart', reason: 'uncaught exception' }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const scope = recognizeExternalChannelDependency(error)
|
|
172
|
+
if (scope !== null) {
|
|
173
|
+
return { action: 'continue', scope, reason: `${scope} async dependency rejection` }
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
return { action: 'restart', reason: 'unknown unhandled rejection' }
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Returns the channel scope to degrade when the rejection carries evidence of
|
|
180
|
+
// an external channel SDK, else null. Matches on structured error codes first,
|
|
181
|
+
// then on stack/module provenance — protocol/diagnostic tokens emitted by the
|
|
182
|
+
// libraries themselves, so English-literal matching is correct here (these are
|
|
183
|
+
// not natural-language user text).
|
|
184
|
+
function recognizeExternalChannelDependency(error: unknown): string | null {
|
|
185
|
+
const code = errorCode(error)
|
|
186
|
+
// Webex KMS (encryption key service) rejections: the incident's exact shape.
|
|
187
|
+
// The `WebexListener` emits an 'error' event for connection failures, but a
|
|
188
|
+
// KMS request that times out 30s later rejects a detached internal promise
|
|
189
|
+
// that never reaches that event.
|
|
190
|
+
if (code === 'KMS_ERROR') return 'webex'
|
|
191
|
+
|
|
192
|
+
const haystack = errorProvenance(error).toLowerCase()
|
|
193
|
+
if (haystack.includes('webex-message-handler') || haystack.includes('agent-messenger/webex')) return 'webex'
|
|
194
|
+
|
|
195
|
+
// Other agent-messenger channel SDKs (slack/discord/telegram/line/kakaotalk).
|
|
196
|
+
// Same containment rationale: one channel's escaped async rejection must not
|
|
197
|
+
// take the whole agent down. Generic `agent-messenger/...` provenance maps to
|
|
198
|
+
// a `channel` scope since the specific platform isn't always recoverable.
|
|
199
|
+
if (haystack.includes('agent-messenger')) return 'channel'
|
|
200
|
+
|
|
201
|
+
return null
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function errorCode(error: unknown): string | null {
|
|
205
|
+
if (error !== null && typeof error === 'object' && 'code' in error) {
|
|
206
|
+
const code = (error as { code: unknown }).code
|
|
207
|
+
if (typeof code === 'string') return code
|
|
208
|
+
}
|
|
209
|
+
return null
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function errorProvenance(error: unknown): string {
|
|
213
|
+
if (error instanceof Error) {
|
|
214
|
+
return `${error.stack ?? ''}\n${error.message}`
|
|
215
|
+
}
|
|
216
|
+
if (error !== null && typeof error === 'object') {
|
|
217
|
+
const stack = 'stack' in error ? String((error as { stack: unknown }).stack ?? '') : ''
|
|
218
|
+
const message = 'message' in error ? String((error as { message: unknown }).message ?? '') : ''
|
|
219
|
+
return `${stack}\n${message}`
|
|
220
|
+
}
|
|
221
|
+
return String(error)
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function describeError(error: unknown): string {
|
|
225
|
+
if (error instanceof Error) {
|
|
226
|
+
const code = errorCode(error)
|
|
227
|
+
return `${error.name}: ${error.message}${code !== null ? ` (code=${code})` : ''}`
|
|
228
|
+
}
|
|
229
|
+
return String(error)
|
|
230
|
+
}
|