typeclaw 0.42.0 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/auth.schema.json +16 -0
- package/package.json +3 -3
- package/secrets.schema.json +16 -0
- package/src/agent/adaptive-thinking-compat.ts +79 -0
- package/src/agent/index.ts +79 -101
- package/src/agent/live-subagents.ts +4 -4
- package/src/agent/plugin-tools.ts +85 -132
- package/src/agent/provider-error.ts +37 -0
- package/src/agent/reload-tool.ts +1 -1
- package/src/agent/session-origin.ts +6 -3
- package/src/agent/subagents.ts +2 -2
- package/src/agent/system-prompt.ts +24 -8
- package/src/agent/tool-result-budget.ts +1 -1
- package/src/agent/tools/channel-react.ts +5 -4
- package/src/agent/tools/spawn-subagent.ts +80 -24
- package/src/agent/tools/stream-snapshot.ts +8 -3
- package/src/bundled-plugins/agent-browser/index.ts +18 -8
- package/src/bundled-plugins/memory/README.md +4 -1
- package/src/bundled-plugins/memory/belief-sentence.ts +50 -0
- package/src/bundled-plugins/memory/dreaming.ts +13 -6
- package/src/bundled-plugins/memory/load-memory.ts +22 -2
- package/src/bundled-plugins/memory/memory-logger.ts +1 -1
- package/src/bundled-plugins/planner/planner.ts +1 -1
- package/src/bundled-plugins/researcher/researcher.ts +2 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +1 -1
- package/src/capabilities/control-plane.ts +24 -0
- package/src/capabilities/index.ts +2 -0
- package/src/capabilities/runtime.ts +30 -0
- package/src/channels/adapters/discord-bot-thread-room.ts +73 -0
- package/src/channels/adapters/discord-bot.ts +9 -1
- package/src/channels/adapters/slack-bot-classify.ts +1 -0
- package/src/channels/adapters/slack-classify.ts +1 -0
- package/src/channels/engagement.ts +33 -0
- package/src/channels/manager.ts +41 -47
- package/src/channels/router.ts +150 -21
- package/src/channels/types.ts +10 -0
- package/src/cli/channel.ts +49 -12
- package/src/cli/command-meta.ts +2 -0
- package/src/cli/hostd.ts +4 -3
- package/src/cli/index.ts +1 -0
- package/src/cli/logs.ts +2 -2
- package/src/cli/mcp.ts +233 -0
- package/src/cli/restart.ts +5 -3
- package/src/cli/shell.ts +2 -2
- package/src/cli/start.ts +2 -2
- package/src/cli/status.ts +2 -2
- package/src/cli/stop.ts +2 -2
- package/src/compose/restart.ts +4 -3
- package/src/compose/start.ts +2 -2
- package/src/compose/stop.ts +2 -2
- package/src/config/config.ts +10 -1
- package/src/config/providers.ts +61 -9
- package/src/container/controller.ts +104 -0
- package/src/container/index.ts +9 -0
- package/src/hostd/daemon.ts +28 -9
- package/src/hostd/protocol.ts +12 -9
- package/src/init/index.ts +1 -1
- package/src/init/instagram-auth.ts +189 -34
- package/src/inspect/logs-item.ts +2 -2
- package/src/mcp/client.ts +16 -6
- package/src/mcp/index.ts +12 -0
- package/src/mcp/manager.ts +9 -4
- package/src/mcp/oauth.ts +212 -0
- package/src/run/index.ts +36 -11
- package/src/run/{codex-fetch-observer.ts → llm-fetch-observer.ts} +182 -82
- package/src/sandbox/availability.ts +2 -2
- package/src/sandbox/build.ts +1 -1
- package/src/sandbox/hidden-paths.ts +59 -0
- package/src/sandbox/index.ts +1 -1
- package/src/sandbox/policy.ts +2 -2
- package/src/secrets/discord-store.ts +4 -13
- package/src/secrets/index.ts +1 -1
- package/src/secrets/instagram-store.ts +4 -13
- package/src/secrets/kakao-store.ts +4 -13
- package/src/secrets/line-store.ts +4 -13
- package/src/secrets/schema.ts +13 -0
- package/src/secrets/secrets-provider.ts +86 -0
- package/src/secrets/slack-store.ts +4 -13
- package/src/secrets/storage.ts +108 -1
- package/src/secrets/webex-store.ts +4 -13
- package/src/server/index.ts +2 -2
- package/src/skills/typeclaw-channel-github/SKILL.md +2 -2
- package/src/skills/typeclaw-channel-instagram/SKILL.md +1 -1
- package/src/skills/typeclaw-config/SKILL.md +5 -3
- package/src/skills/typeclaw-plugins/SKILL.md +1 -1
- package/src/skills/typeclaw-troubleshooting/SKILL.md +1 -1
- package/src/stream/broker.ts +4 -3
- package/src/stream/types.ts +1 -0
- package/typeclaw.schema.json +14 -0
|
@@ -1,57 +1,60 @@
|
|
|
1
|
-
export type
|
|
1
|
+
export type LlmFetchObserverLogger = {
|
|
2
2
|
info: (msg: string) => void
|
|
3
3
|
warn: (msg: string) => void
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
// A provider endpoint the observer should instrument. `match` is host-agnostic
|
|
7
|
+
// on purpose: base URLs are user-configured (ANTHROPIC_BASE_URL, OPENAI_BASE_URL,
|
|
8
|
+
// OpenRouter/LiteLLM/Fireworks/arbitrary proxies), so a host allowlist would be
|
|
9
|
+
// incomplete by design and let a stalled proxy stream slip through unguarded.
|
|
10
|
+
// Path-suffix matching keys off the protocol shape instead of the host.
|
|
11
|
+
export type LlmFetchEndpoint = {
|
|
12
|
+
// Short identifier surfaced in logs as `provider=<label>` and in timeout
|
|
13
|
+
// error messages, so a stall is diagnosable without provider-specific text.
|
|
14
|
+
label: string
|
|
15
|
+
match: (url: URL, method: string) => boolean
|
|
16
|
+
// Per-endpoint timeout overrides. Unset falls back to the observer-level
|
|
17
|
+
// defaults. Set to 0 to disable an individual timer for this endpoint.
|
|
18
|
+
ttfbMs?: number
|
|
19
|
+
idleMs?: number
|
|
20
|
+
overallMs?: number
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type LlmFetchObserverOptions = {
|
|
24
|
+
logger?: LlmFetchObserverLogger
|
|
25
|
+
endpoints?: readonly LlmFetchEndpoint[]
|
|
9
26
|
now?: () => number
|
|
10
27
|
// Override the default pre-headers (TTFB) deadline applied to the outer
|
|
11
|
-
// fetch(). When
|
|
12
|
-
//
|
|
13
|
-
// `
|
|
28
|
+
// fetch(). When a provider silently holds a request without sending response
|
|
29
|
+
// headers, this is the timer that releases the request so `pi-coding-agent`'s
|
|
30
|
+
// `_isRetryableError` can retry. Default: 15_000 ms.
|
|
14
31
|
//
|
|
15
|
-
// Healthy
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
// mode rather than a real request making progress. False-positive cost
|
|
23
|
-
// is one retry (~5s extra); false-negative cost is the full Bun socket
|
|
24
|
-
// deadline (~268s). Aggressive wins.
|
|
32
|
+
// Healthy turns return response headers within ~1s (observed Codex production
|
|
33
|
+
// p50: ~860ms). Pathological-but-healthy upper bounds: TLS handshake on a cold
|
|
34
|
+
// connection (~2s), prompt-prefill on a cache miss with large input (~3s),
|
|
35
|
+
// edge routing slowness (~2s) — sum ~7s. 15s is ~2x that, so anything past it
|
|
36
|
+
// is almost certainly the silent-hang failure mode rather than a real request
|
|
37
|
+
// making progress. False-positive cost is one retry (~5s extra); false-negative
|
|
38
|
+
// cost is the full Bun socket deadline (~268s). Aggressive wins.
|
|
25
39
|
ttfbMs?: number
|
|
26
40
|
// Override the sliding inter-chunk idle deadline applied to the SSE body
|
|
27
41
|
// reader. Resets on every chunk; if no bytes arrive within this window the
|
|
28
|
-
// body stream errors.
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
// reasoning window is genuinely byte-silent) while bounded by the overall
|
|
35
|
-
// cap. Set to 0 to disable just this timer.
|
|
42
|
+
// body stream errors. Doubles as a recovery bound: on a silent stall the user
|
|
43
|
+
// waits this long before the retry fires, so it should not exceed the overall
|
|
44
|
+
// ceiling. Default 120_000 ms. Kept uniform across providers: even though
|
|
45
|
+
// Anthropic/OpenAI-compatible streams usually emit SSE pings, arbitrary proxies
|
|
46
|
+
// may not, and a tighter window risks aborting a valid long reasoning pause.
|
|
47
|
+
// Set to 0 to disable just this timer.
|
|
36
48
|
idleMs?: number
|
|
37
|
-
// Override the absolute wall-clock ceiling on a single
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
// was tuned against a light-turn sample (slowest healthy ~45s, p99 ~30s) and
|
|
47
|
-
// aborted heavy reasoning turns that were still legitimately trickling bytes
|
|
48
|
-
// — PR reviews, the `dreaming` memory consolidation, and long channel threads
|
|
49
|
-
// routinely run a slow-trickle stream past 2min (observed total_ms=120009 with
|
|
50
|
-
// body_bytes>700k on otherwise-progressing turns). Those are NOT the silent
|
|
51
|
-
// hang this timer exists to catch; the sliding `idleMs` (still 120s) already
|
|
52
|
-
// bounds genuine dead air per-chunk, so the wall-clock ceiling only needs to
|
|
53
|
-
// stop a never-terminating stream. 300s caps a real hang at ~5min while giving
|
|
54
|
-
// heavy turns the headroom they need. Set to 0 to disable just this timer.
|
|
49
|
+
// Override the absolute wall-clock ceiling on a single request, measured from
|
|
50
|
+
// fetch start to body completion. Unlike `idleMs`, it does NOT reset on chunk
|
|
51
|
+
// arrival, so it catches a "slow-trickle" stream that emits bytes inside every
|
|
52
|
+
// idle window yet never reaches a terminal SSE event. On expiry the request is
|
|
53
|
+
// aborted with a retryable error, so this also bounds how long a user waits
|
|
54
|
+
// before the retry fires. Default 300_000 ms: heavy reasoning turns (PR reviews,
|
|
55
|
+
// memory consolidation, long channel threads) routinely trickle bytes past 2min
|
|
56
|
+
// on otherwise-progressing turns, so the ceiling only needs to stop a
|
|
57
|
+
// never-terminating stream. Set to 0 to disable just this timer.
|
|
55
58
|
overallMs?: number
|
|
56
59
|
// Schedule fn for tests. Receives (delayMs, callback) and returns a handle
|
|
57
60
|
// the wrapper can pass to `clear`. Default: `setTimeout`/`clearTimeout`.
|
|
@@ -63,28 +66,71 @@ export type TimeoutScheduler = {
|
|
|
63
66
|
clear: (handle: unknown) => void
|
|
64
67
|
}
|
|
65
68
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
const
|
|
69
|
+
// New neutral env vars gate the whole observer and its timeouts; the Codex names
|
|
70
|
+
// stay as backwards-compatible aliases so existing deployments that opted out via
|
|
71
|
+
// TYPECLAW_CODEX_FETCH_OBSERVER=off keep that behavior.
|
|
72
|
+
const ENV_DISABLE_OBSERVER = 'TYPECLAW_LLM_FETCH_OBSERVER'
|
|
73
|
+
const ENV_DISABLE_OBSERVER_LEGACY = 'TYPECLAW_CODEX_FETCH_OBSERVER'
|
|
74
|
+
const ENV_DISABLE_TIMEOUTS = 'TYPECLAW_LLM_TIMEOUTS'
|
|
75
|
+
const ENV_DISABLE_TIMEOUTS_LEGACY = 'TYPECLAW_CODEX_TIMEOUTS'
|
|
76
|
+
const ENV_TTFB_MS = 'TYPECLAW_LLM_TTFB_MS'
|
|
77
|
+
const ENV_IDLE_MS = 'TYPECLAW_LLM_IDLE_MS'
|
|
78
|
+
const ENV_OVERALL_MS = 'TYPECLAW_LLM_OVERALL_MS'
|
|
79
|
+
// Codex keeps its own tuned overrides for the codex endpoint only.
|
|
80
|
+
const ENV_CODEX_TTFB_MS = 'TYPECLAW_CODEX_TTFB_MS'
|
|
81
|
+
const ENV_CODEX_IDLE_MS = 'TYPECLAW_CODEX_IDLE_MS'
|
|
82
|
+
const ENV_CODEX_OVERALL_MS = 'TYPECLAW_CODEX_OVERALL_MS'
|
|
73
83
|
const DEFAULT_TTFB_MS = 15_000
|
|
74
84
|
const DEFAULT_IDLE_MS = 120_000
|
|
75
85
|
const DEFAULT_OVERALL_MS = 300_000
|
|
76
|
-
const LOG_PREFIX = '[
|
|
86
|
+
const LOG_PREFIX = '[llm-fetch]'
|
|
77
87
|
|
|
78
88
|
const defaultScheduler: TimeoutScheduler = {
|
|
79
89
|
set: (delayMs, cb) => setTimeout(cb, delayMs),
|
|
80
90
|
clear: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
81
91
|
}
|
|
82
92
|
|
|
83
|
-
const consoleLogger:
|
|
93
|
+
const consoleLogger: LlmFetchObserverLogger = {
|
|
84
94
|
info: (m) => console.log(m),
|
|
85
95
|
warn: (m) => console.warn(m),
|
|
86
96
|
}
|
|
87
97
|
|
|
98
|
+
// Matches a path regardless of the base-URL prefix a proxy prepends (e.g.
|
|
99
|
+
// `/anthropic/v1/messages`). Trailing slash tolerated for proxies that append one.
|
|
100
|
+
function pathEndsWith(pathname: string, suffix: string): boolean {
|
|
101
|
+
return pathname === suffix || pathname.endsWith(suffix) || pathname.endsWith(`${suffix}/`)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function defaultLlmFetchEndpoints(env: NodeJS.ProcessEnv = process.env): readonly LlmFetchEndpoint[] {
|
|
105
|
+
return [
|
|
106
|
+
{
|
|
107
|
+
label: 'codex',
|
|
108
|
+
// Method check matches the pi-ai provider (only POST hits codex/responses);
|
|
109
|
+
// GETs to the same host (auth probes, etc.) are deliberately ignored.
|
|
110
|
+
match: (url, method) =>
|
|
111
|
+
method === 'POST' && url.hostname === 'chatgpt.com' && url.pathname.includes('/codex/responses'),
|
|
112
|
+
// Only pin a per-endpoint value when the operator explicitly set the
|
|
113
|
+
// Codex-specific var; unset falls through to the generic TYPECLAW_LLM_*
|
|
114
|
+
// (or built-in default) so it isn't masked. See readEnvMsOptional.
|
|
115
|
+
ttfbMs: readEnvMsOptional(env, ENV_CODEX_TTFB_MS),
|
|
116
|
+
idleMs: readEnvMsOptional(env, ENV_CODEX_IDLE_MS),
|
|
117
|
+
overallMs: readEnvMsOptional(env, ENV_CODEX_OVERALL_MS),
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
label: 'anthropic',
|
|
121
|
+
match: (url, method) => method === 'POST' && pathEndsWith(url.pathname, '/v1/messages'),
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
label: 'openai-compatible',
|
|
125
|
+
match: (url, method) =>
|
|
126
|
+
method === 'POST' &&
|
|
127
|
+
(pathEndsWith(url.pathname, '/v1/chat/completions') ||
|
|
128
|
+
pathEndsWith(url.pathname, '/chat/completions') ||
|
|
129
|
+
pathEndsWith(url.pathname, '/v1/responses')),
|
|
130
|
+
},
|
|
131
|
+
]
|
|
132
|
+
}
|
|
133
|
+
|
|
88
134
|
type InstallState = {
|
|
89
135
|
originalFetch: typeof fetch
|
|
90
136
|
wrapped: typeof fetch
|
|
@@ -99,13 +145,14 @@ type InstallState = {
|
|
|
99
145
|
// still-running agent.
|
|
100
146
|
let installed: InstallState | null = null
|
|
101
147
|
|
|
102
|
-
// Returns
|
|
103
|
-
//
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
148
|
+
// Returns the first endpoint whose matcher claims this request, or null. The
|
|
149
|
+
// matched endpoint's per-endpoint timeout overrides win over observer defaults.
|
|
150
|
+
function matchEndpoint(
|
|
151
|
+
input: RequestInfo | URL,
|
|
152
|
+
init: RequestInit | undefined,
|
|
153
|
+
endpoints: readonly LlmFetchEndpoint[],
|
|
154
|
+
): LlmFetchEndpoint | null {
|
|
107
155
|
const method = (init?.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase()
|
|
108
|
-
if (method !== 'POST') return false
|
|
109
156
|
let urlString: string
|
|
110
157
|
if (typeof input === 'string') urlString = input
|
|
111
158
|
else if (input instanceof URL) urlString = input.toString()
|
|
@@ -114,10 +161,12 @@ function shouldObserve(input: RequestInfo | URL, init: RequestInit | undefined,
|
|
|
114
161
|
try {
|
|
115
162
|
parsed = new URL(urlString)
|
|
116
163
|
} catch {
|
|
117
|
-
return
|
|
164
|
+
return null
|
|
118
165
|
}
|
|
119
|
-
|
|
120
|
-
|
|
166
|
+
for (const endpoint of endpoints) {
|
|
167
|
+
if (endpoint.match(parsed, method)) return endpoint
|
|
168
|
+
}
|
|
169
|
+
return null
|
|
121
170
|
}
|
|
122
171
|
|
|
123
172
|
function quote(value: string | null): string {
|
|
@@ -126,6 +175,7 @@ function quote(value: string | null): string {
|
|
|
126
175
|
}
|
|
127
176
|
|
|
128
177
|
function formatLine(fields: {
|
|
178
|
+
provider: string
|
|
129
179
|
status: number | null
|
|
130
180
|
headersMs: number | null
|
|
131
181
|
firstByteMs: number | null
|
|
@@ -138,6 +188,7 @@ function formatLine(fields: {
|
|
|
138
188
|
}): string {
|
|
139
189
|
return [
|
|
140
190
|
LOG_PREFIX,
|
|
191
|
+
`provider=${fields.provider}`,
|
|
141
192
|
`status=${fields.status === null ? 'null' : fields.status}`,
|
|
142
193
|
`headers_ms=${fields.headersMs === null ? 'null' : fields.headersMs}`,
|
|
143
194
|
`first_byte_ms=${fields.firstByteMs === null ? 'null' : fields.firstByteMs}`,
|
|
@@ -150,15 +201,32 @@ function formatLine(fields: {
|
|
|
150
201
|
].join(' ')
|
|
151
202
|
}
|
|
152
203
|
|
|
153
|
-
function readEnvMs(name: string, fallback: number): number {
|
|
154
|
-
|
|
155
|
-
|
|
204
|
+
function readEnvMs(env: NodeJS.ProcessEnv, name: string, fallback: number): number {
|
|
205
|
+
return readEnvMsOptional(env, name) ?? fallback
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// `undefined` when the var is unset/blank/invalid, so a per-endpoint override
|
|
209
|
+
// reader can fall THROUGH to the observer defaults instead of pinning a concrete
|
|
210
|
+
// value. This is what lets an unset `TYPECLAW_CODEX_*_MS` defer to the generic
|
|
211
|
+
// `TYPECLAW_LLM_*_MS` (via `envDefaults`) rather than masking it with the raw
|
|
212
|
+
// default — the precedence is opts > TYPECLAW_CODEX_* (when set) > TYPECLAW_LLM_*
|
|
213
|
+
// > built-in default.
|
|
214
|
+
function readEnvMsOptional(env: NodeJS.ProcessEnv, name: string): number | undefined {
|
|
215
|
+
const raw = env[name]
|
|
216
|
+
if (raw === undefined || raw === '') return undefined
|
|
156
217
|
const parsed = Number.parseInt(raw, 10)
|
|
157
|
-
if (!Number.isFinite(parsed) || parsed < 0) return
|
|
218
|
+
if (!Number.isFinite(parsed) || parsed < 0) return undefined
|
|
158
219
|
return parsed
|
|
159
220
|
}
|
|
160
221
|
|
|
222
|
+
type ResolvedTimeouts = {
|
|
223
|
+
ttfbMs: number
|
|
224
|
+
idleMs: number
|
|
225
|
+
overallMs: number
|
|
226
|
+
}
|
|
227
|
+
|
|
161
228
|
type BodyTapConfig = {
|
|
229
|
+
provider: string
|
|
162
230
|
idleMs: number
|
|
163
231
|
overallMs: number
|
|
164
232
|
scheduler: TimeoutScheduler
|
|
@@ -172,12 +240,13 @@ function attachBodyTimingTap(
|
|
|
172
240
|
retryAfter: string | null,
|
|
173
241
|
requestId: string | null,
|
|
174
242
|
now: () => number,
|
|
175
|
-
logger:
|
|
243
|
+
logger: LlmFetchObserverLogger,
|
|
176
244
|
config: BodyTapConfig,
|
|
177
245
|
): Response {
|
|
178
246
|
if (response.body === null) {
|
|
179
247
|
logger.info(
|
|
180
248
|
formatLine({
|
|
249
|
+
provider: config.provider,
|
|
181
250
|
status,
|
|
182
251
|
headersMs,
|
|
183
252
|
firstByteMs: null,
|
|
@@ -202,6 +271,7 @@ function attachBodyTimingTap(
|
|
|
202
271
|
settled = true
|
|
203
272
|
logger.info(
|
|
204
273
|
formatLine({
|
|
274
|
+
provider: config.provider,
|
|
205
275
|
status,
|
|
206
276
|
headersMs,
|
|
207
277
|
firstByteMs,
|
|
@@ -235,7 +305,9 @@ function attachBodyTimingTap(
|
|
|
235
305
|
if (idleHandle !== null) config.scheduler.clear(idleHandle)
|
|
236
306
|
idleHandle = config.scheduler.set(config.idleMs, () => {
|
|
237
307
|
cause = 'idle_timeout'
|
|
238
|
-
idleController.abort(
|
|
308
|
+
idleController.abort(
|
|
309
|
+
new Error(`${config.provider} SSE body idle for ${config.idleMs}ms (typeclaw observer timeout)`),
|
|
310
|
+
)
|
|
239
311
|
})
|
|
240
312
|
}
|
|
241
313
|
|
|
@@ -253,7 +325,9 @@ function attachBodyTimingTap(
|
|
|
253
325
|
overallHandle = config.scheduler.set(remainingOverallMs, () => {
|
|
254
326
|
cause = 'overall_timeout'
|
|
255
327
|
idleController.abort(
|
|
256
|
-
new Error(
|
|
328
|
+
new Error(
|
|
329
|
+
`${config.provider} SSE body exceeded overall deadline of ${config.overallMs}ms (typeclaw observer timeout)`,
|
|
330
|
+
),
|
|
257
331
|
)
|
|
258
332
|
})
|
|
259
333
|
}
|
|
@@ -341,8 +415,8 @@ function attachBodyTimingTap(
|
|
|
341
415
|
})
|
|
342
416
|
}
|
|
343
417
|
|
|
344
|
-
export function
|
|
345
|
-
if (process.env[ENV_DISABLE_OBSERVER] === 'off') {
|
|
418
|
+
export function installLlmFetchObserver(opts: LlmFetchObserverOptions = {}): () => void {
|
|
419
|
+
if (process.env[ENV_DISABLE_OBSERVER] === 'off' || process.env[ENV_DISABLE_OBSERVER_LEGACY] === 'off') {
|
|
346
420
|
return () => {}
|
|
347
421
|
}
|
|
348
422
|
const logger = opts.logger ?? consoleLogger
|
|
@@ -351,33 +425,55 @@ export function installCodexFetchObserver(opts: CodexFetchObserverOptions = {}):
|
|
|
351
425
|
return makeRelease(installed.wrapped)
|
|
352
426
|
}
|
|
353
427
|
|
|
354
|
-
const
|
|
428
|
+
const endpoints = opts.endpoints ?? defaultLlmFetchEndpoints()
|
|
355
429
|
const now = opts.now ?? Date.now
|
|
356
430
|
const scheduler = opts.scheduler ?? defaultScheduler
|
|
357
|
-
const timeoutsEnabled =
|
|
358
|
-
|
|
359
|
-
const
|
|
360
|
-
|
|
431
|
+
const timeoutsEnabled =
|
|
432
|
+
process.env[ENV_DISABLE_TIMEOUTS] !== 'off' && process.env[ENV_DISABLE_TIMEOUTS_LEGACY] !== 'off'
|
|
433
|
+
const envDefaults: ResolvedTimeouts = {
|
|
434
|
+
ttfbMs: readEnvMs(process.env, ENV_TTFB_MS, DEFAULT_TTFB_MS),
|
|
435
|
+
idleMs: readEnvMs(process.env, ENV_IDLE_MS, DEFAULT_IDLE_MS),
|
|
436
|
+
overallMs: readEnvMs(process.env, ENV_OVERALL_MS, DEFAULT_OVERALL_MS),
|
|
437
|
+
}
|
|
361
438
|
const originalFetch = globalThis.fetch
|
|
362
439
|
|
|
440
|
+
// Precedence per timer: explicit observer-level option wins over the endpoint's
|
|
441
|
+
// own value, which wins over the generic env/default. An explicit `opts.*` means
|
|
442
|
+
// "force this across all endpoints" (the test seam and any global override),
|
|
443
|
+
// so it must beat a per-endpoint tuned value. The master `timeoutsEnabled=off`
|
|
444
|
+
// switch forces every timer to 0 regardless.
|
|
445
|
+
const resolveTimeouts = (endpoint: LlmFetchEndpoint): ResolvedTimeouts => {
|
|
446
|
+
if (!timeoutsEnabled) return { ttfbMs: 0, idleMs: 0, overallMs: 0 }
|
|
447
|
+
return {
|
|
448
|
+
ttfbMs: opts.ttfbMs ?? endpoint.ttfbMs ?? envDefaults.ttfbMs,
|
|
449
|
+
idleMs: opts.idleMs ?? endpoint.idleMs ?? envDefaults.idleMs,
|
|
450
|
+
overallMs: opts.overallMs ?? endpoint.overallMs ?? envDefaults.overallMs,
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
363
454
|
const wrappedImpl = async (
|
|
364
455
|
input: Parameters<typeof fetch>[0],
|
|
365
456
|
init?: Parameters<typeof fetch>[1],
|
|
366
457
|
): Promise<Response> => {
|
|
367
|
-
|
|
458
|
+
const endpoint = matchEndpoint(input, init, endpoints)
|
|
459
|
+
if (endpoint === null) {
|
|
368
460
|
return originalFetch(input, init)
|
|
369
461
|
}
|
|
462
|
+
const timeouts = resolveTimeouts(endpoint)
|
|
463
|
+
const provider = endpoint.label
|
|
370
464
|
const start = now()
|
|
371
465
|
|
|
372
466
|
let ttfbCause: 'ttfb_timeout' | null = null
|
|
373
467
|
let ttfbHandle: unknown = null
|
|
374
468
|
let initWithSignal: RequestInit | undefined = init
|
|
375
|
-
if (ttfbMs > 0) {
|
|
469
|
+
if (timeouts.ttfbMs > 0) {
|
|
376
470
|
const ttfbController = new AbortController()
|
|
377
|
-
ttfbHandle = scheduler.set(ttfbMs, () => {
|
|
471
|
+
ttfbHandle = scheduler.set(timeouts.ttfbMs, () => {
|
|
378
472
|
ttfbCause = 'ttfb_timeout'
|
|
379
473
|
ttfbController.abort(
|
|
380
|
-
new Error(
|
|
474
|
+
new Error(
|
|
475
|
+
`${provider} fetch timed out before response headers after ${timeouts.ttfbMs}ms (typeclaw observer timeout)`,
|
|
476
|
+
),
|
|
381
477
|
)
|
|
382
478
|
})
|
|
383
479
|
const signal = init?.signal ? AbortSignal.any([init.signal, ttfbController.signal]) : ttfbController.signal
|
|
@@ -391,11 +487,14 @@ export function installCodexFetchObserver(opts: CodexFetchObserverOptions = {}):
|
|
|
391
487
|
if (ttfbHandle !== null) scheduler.clear(ttfbHandle)
|
|
392
488
|
const isTtfbAbort = ttfbCause === 'ttfb_timeout'
|
|
393
489
|
const surfacedError = isTtfbAbort
|
|
394
|
-
? new Error(
|
|
490
|
+
? new Error(
|
|
491
|
+
`${provider} fetch timed out before response headers after ${timeouts.ttfbMs}ms (typeclaw observer timeout)`,
|
|
492
|
+
)
|
|
395
493
|
: err
|
|
396
494
|
const message = surfacedError instanceof Error ? surfacedError.message : String(surfacedError)
|
|
397
495
|
logger.info(
|
|
398
496
|
formatLine({
|
|
497
|
+
provider,
|
|
399
498
|
status: null,
|
|
400
499
|
headersMs: null,
|
|
401
500
|
firstByteMs: null,
|
|
@@ -414,8 +513,9 @@ export function installCodexFetchObserver(opts: CodexFetchObserverOptions = {}):
|
|
|
414
513
|
const retryAfter = response.headers.get('retry-after')
|
|
415
514
|
const requestId = response.headers.get('x-request-id')
|
|
416
515
|
return attachBodyTimingTap(response, start, headersMs, response.status, retryAfter, requestId, now, logger, {
|
|
417
|
-
|
|
418
|
-
|
|
516
|
+
provider,
|
|
517
|
+
idleMs: timeouts.idleMs,
|
|
518
|
+
overallMs: timeouts.overallMs,
|
|
419
519
|
scheduler,
|
|
420
520
|
})
|
|
421
521
|
}
|
|
@@ -131,7 +131,7 @@ export function _resetRealProcProbeCacheForTests(): void {
|
|
|
131
131
|
// /proc /proc`: it binds the container's already-real procfs with NO unshare
|
|
132
132
|
// --mount-proc and NO CAP_SYS_ADMIN, so it works where 'real-proc' is rejected
|
|
133
133
|
// (OrbStack). Its security rests entirely on the kernel BLOCKING cross-userns
|
|
134
|
-
// reads of /proc/<agent>/environ — the
|
|
134
|
+
// reads of /proc/<agent>/environ — the OPENAI_API_KEY / GH_TOKEN surface —
|
|
135
135
|
// because bwrap's --unshare-all puts the sandbox in a CHILD user namespace. That
|
|
136
136
|
// block is a kernel fact on every mainstream host, but the consumer must never
|
|
137
137
|
// assume it: a misconfigured runtime that preserves parent-userns creds, or a
|
|
@@ -279,7 +279,7 @@ async function probeProcBind(bwrap: string): Promise<ProcBindProbe> {
|
|
|
279
279
|
let sentinel: Bun.Subprocess | undefined
|
|
280
280
|
try {
|
|
281
281
|
// `env -i` so the sentinel carries ONLY the marker, never the parent's real
|
|
282
|
-
//
|
|
282
|
+
// OPENAI_API_KEY/GH_TOKEN — the probe must not itself plant a real secret,
|
|
283
283
|
// independent of Bun.spawn's env merge/replace semantics. `sleep 30` outlives
|
|
284
284
|
// the sub-second probe by a wide margin (so it cannot exit mid-probe and let
|
|
285
285
|
// a post-exit ESRCH masquerade as the EACCES block), yet is short enough to
|
package/src/sandbox/build.ts
CHANGED
|
@@ -144,7 +144,7 @@ function buildArgv(command: string, policy: SandboxPolicy): string[] {
|
|
|
144
144
|
// --tmpfs /proc, never --proc /proc (OrbStack's kernel blocks
|
|
145
145
|
// mount("proc",...) from user namespaces) and never --dev-bind /proc /proc
|
|
146
146
|
// (leaks the outer container's /proc/N/environ — including
|
|
147
|
-
//
|
|
147
|
+
// OPENAI_API_KEY — into the sandbox). See sandbox.mdx.
|
|
148
148
|
argv.push('--tmpfs', '/proc')
|
|
149
149
|
|
|
150
150
|
// Re-expose ONLY the bun ELF at /proc/self/exe so sandboxed package runners
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { lstat, mkdir, writeFile } from 'node:fs/promises'
|
|
1
2
|
import { join } from 'node:path'
|
|
2
3
|
|
|
3
4
|
import type { SessionOrigin } from '@/agent/session-origin'
|
|
@@ -39,3 +40,61 @@ export function resolveHiddenPaths(
|
|
|
39
40
|
const files = seesSecrets ? [] : SECRET_FILES.map((f) => join(agentDir, f))
|
|
40
41
|
return { dirs, files }
|
|
41
42
|
}
|
|
43
|
+
|
|
44
|
+
// SECURITY / bwrap contract: the mask ops REQUIRE a pre-existing target.
|
|
45
|
+
// `--ro-bind-data` (secret files) and `--tmpfs` (private dirs) create their
|
|
46
|
+
// mount point under the agent-folder bind, which fails "Read-only file system"
|
|
47
|
+
// on a virtiofs/OrbStack bind (bwrap cannot create it from its child user
|
|
48
|
+
// namespace). An agent whose folder legitimately lacks `.env` (keys live in
|
|
49
|
+
// secrets.json) would otherwise have EVERY sandboxed bash call abort at setup.
|
|
50
|
+
// Ensure each target on the REAL host FS first — the runtime owns agentDir RW
|
|
51
|
+
// here (only sandboxed bash sees it RO), so an empty placeholder is harmless and
|
|
52
|
+
// the mask binds over a real empty file, leaking nothing.
|
|
53
|
+
//
|
|
54
|
+
// ENSURE, not FILTER: package-install mode makes the jail root RW
|
|
55
|
+
// (`writableRoot: agentDir`), so dropping an absent secret would let sandboxed
|
|
56
|
+
// code CREATE `.env`/`secrets.json` for real (planting) or expose one created
|
|
57
|
+
// mid-session (TOCTOU). A guaranteed target keeps the mask always rendered over
|
|
58
|
+
// it (last-op-wins), closing that hole for both jail modes. A symlink squatting
|
|
59
|
+
// a target is refused and dropped (a bind would follow it out), and each target
|
|
60
|
+
// is best-effort so one bad entry degrades that mask, never aborts sandboxing.
|
|
61
|
+
export async function ensureHiddenMaskTargets(hidden: HiddenPaths): Promise<HiddenPaths> {
|
|
62
|
+
const dirs = await ensureMaskTargets(hidden.dirs, 'dir')
|
|
63
|
+
const files = await ensureMaskTargets(hidden.files, 'file')
|
|
64
|
+
return { dirs, files }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function ensureMaskTargets(targets: string[], kind: 'dir' | 'file'): Promise<string[]> {
|
|
68
|
+
const results = await Promise.all(targets.map((target) => ensureMaskTarget(target, kind)))
|
|
69
|
+
return targets.filter((_, i) => results[i])
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function ensureMaskTarget(target: string, kind: 'dir' | 'file'): Promise<boolean> {
|
|
73
|
+
try {
|
|
74
|
+
if (kind === 'dir') await mkdir(target, { recursive: true })
|
|
75
|
+
else await ensureEmptyFile(target)
|
|
76
|
+
return await isRealEntry(target, kind)
|
|
77
|
+
} catch {
|
|
78
|
+
return false
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function ensureEmptyFile(target: string): Promise<void> {
|
|
83
|
+
try {
|
|
84
|
+
await writeFile(target, '', { flag: 'wx' })
|
|
85
|
+
} catch {
|
|
86
|
+
// Already exists or lost a creation race; isRealEntry re-validates the kind
|
|
87
|
+
// and rejects a symlink, so an existing regular file passes and anything
|
|
88
|
+
// else is dropped from the mask.
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function isRealEntry(target: string, kind: 'dir' | 'file'): Promise<boolean> {
|
|
93
|
+
try {
|
|
94
|
+
const stats = await lstat(target)
|
|
95
|
+
if (stats.isSymbolicLink()) return false
|
|
96
|
+
return kind === 'dir' ? stats.isDirectory() : stats.isFile()
|
|
97
|
+
} catch {
|
|
98
|
+
return false
|
|
99
|
+
}
|
|
100
|
+
}
|
package/src/sandbox/index.ts
CHANGED
|
@@ -13,7 +13,7 @@ export {
|
|
|
13
13
|
_resetProcBindProbeCacheForTests,
|
|
14
14
|
_resetRealProcProbeCacheForTests,
|
|
15
15
|
} from './availability'
|
|
16
|
-
export { resolveHiddenPaths, type HiddenPaths } from './hidden-paths'
|
|
16
|
+
export { ensureHiddenMaskTargets, resolveHiddenPaths, type HiddenPaths } from './hidden-paths'
|
|
17
17
|
export {
|
|
18
18
|
resolvePackageInstallZones,
|
|
19
19
|
resolveProtectedZones,
|
package/src/sandbox/policy.ts
CHANGED
|
@@ -10,7 +10,7 @@ export type SandboxNetwork = 'none' | 'inherit'
|
|
|
10
10
|
// ALREADY-REAL procfs straight into the sandbox — no `unshare --mount-proc`, no
|
|
11
11
|
// CAP_SYS_ADMIN. A JS package runner's child gets a real /proc/self/{fd,maps,exe}
|
|
12
12
|
// so `bunx`/`bun add`/`bun run <pkg-bin>` stop aborting with Bun's ENOTDIR. The
|
|
13
|
-
// agent runtime's /proc/<agent>/environ (
|
|
13
|
+
// agent runtime's /proc/<agent>/environ (OPENAI_API_KEY, GH_TOKEN) is NOT
|
|
14
14
|
// leaked: build.ts always emits --unshare-user, so the sandboxed bash runs as
|
|
15
15
|
// mapped-root in a CHILD user namespace that is not an ancestor of the agent
|
|
16
16
|
// runtime's userns. The kernel's PTRACE_MODE_READ_FSCREDS check on
|
|
@@ -139,7 +139,7 @@ export type SandboxPolicy = {
|
|
|
139
139
|
// The env the sandbox always re-introduces after `--clearenv`. Anything not
|
|
140
140
|
// listed here (or explicitly named in `env.set` / `env.passthrough` by the
|
|
141
141
|
// consumer) is invisible inside the sandbox. This is the load-bearing leak
|
|
142
|
-
// guard: the container env holds
|
|
142
|
+
// guard: the container env holds OPENAI_API_KEY and GH_TOKEN, and env
|
|
143
143
|
// inheritance is the single highest-risk exfil path for prompt-injected bash.
|
|
144
144
|
// HOME points at /tmp because the sandbox mounts /tmp as a fresh tmpfs.
|
|
145
145
|
//
|
|
@@ -1,13 +1,12 @@
|
|
|
1
|
-
import { sendHttp } from '@/hostd/client'
|
|
2
|
-
|
|
3
1
|
import { type DiscordAccountRecord, type DiscordChannelBlock, discordChannelBlockSchema } from './schema'
|
|
2
|
+
import type { RuntimeSecretsProvider } from './secrets-provider'
|
|
4
3
|
import { SecretsBackend } from './storage'
|
|
5
4
|
|
|
6
5
|
export type SetDiscordAccountInput = DiscordAccountRecord
|
|
7
6
|
|
|
8
7
|
export type SecretsDiscordCredentialStoreOptions =
|
|
9
8
|
| { mode: 'host'; secretsPath: string }
|
|
10
|
-
| { mode: 'container'; secretsPath: string;
|
|
9
|
+
| { mode: 'container'; secretsPath: string; hostProvider: RuntimeSecretsProvider }
|
|
11
10
|
|
|
12
11
|
const EMPTY_BLOCK: DiscordChannelBlock = { currentAccount: null, accounts: {} }
|
|
13
12
|
|
|
@@ -56,7 +55,7 @@ export class SecretsDiscordCredentialStore {
|
|
|
56
55
|
|
|
57
56
|
private readBlock(): DiscordChannelBlock {
|
|
58
57
|
const channels =
|
|
59
|
-
this.options.mode === 'container' ? this.
|
|
58
|
+
this.options.mode === 'container' ? this.options.hostProvider.readChannels() : this.backend.readChannelsSync()
|
|
60
59
|
return parseBlock(channels?.discord)
|
|
61
60
|
}
|
|
62
61
|
|
|
@@ -64,15 +63,7 @@ export class SecretsDiscordCredentialStore {
|
|
|
64
63
|
return this.enqueueWrite(async () => {
|
|
65
64
|
if (this.options.mode === 'container') {
|
|
66
65
|
const next = update(this.readBlock())
|
|
67
|
-
|
|
68
|
-
{
|
|
69
|
-
kind: 'secrets-patch',
|
|
70
|
-
containerName: this.options.containerName,
|
|
71
|
-
patch: { channels: { discord: next } },
|
|
72
|
-
},
|
|
73
|
-
{ url: this.options.hostdUrl, token: this.options.restartToken },
|
|
74
|
-
)
|
|
75
|
-
if (!response.ok) throw new Error(`secrets-patch failed: ${response.reason}`)
|
|
66
|
+
await this.options.hostProvider.writeBackChannelBlock({ discord: next })
|
|
76
67
|
return
|
|
77
68
|
}
|
|
78
69
|
|
package/src/secrets/index.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import { sendHttp } from '@/hostd/client'
|
|
2
|
-
|
|
3
1
|
import { type InstagramAccountRecord, type InstagramChannelBlock, instagramChannelBlockSchema } from './schema'
|
|
2
|
+
import type { RuntimeSecretsProvider } from './secrets-provider'
|
|
4
3
|
import { SecretsBackend } from './storage'
|
|
5
4
|
|
|
6
5
|
export type SecretsInstagramCredentialStoreOptions =
|
|
7
6
|
| { mode: 'host'; secretsPath: string }
|
|
8
|
-
| { mode: 'container'; secretsPath: string;
|
|
7
|
+
| { mode: 'container'; secretsPath: string; hostProvider: RuntimeSecretsProvider }
|
|
9
8
|
|
|
10
9
|
const EMPTY_BLOCK: InstagramChannelBlock = { currentAccount: null, accounts: {} }
|
|
11
10
|
|
|
@@ -62,7 +61,7 @@ export class SecretsInstagramCredentialStore {
|
|
|
62
61
|
|
|
63
62
|
private readBlock(): InstagramChannelBlock {
|
|
64
63
|
const channels =
|
|
65
|
-
this.options.mode === 'container' ? this.
|
|
64
|
+
this.options.mode === 'container' ? this.options.hostProvider.readChannels() : this.backend.readChannelsSync()
|
|
66
65
|
return parseBlock(channels?.instagram)
|
|
67
66
|
}
|
|
68
67
|
|
|
@@ -70,15 +69,7 @@ export class SecretsInstagramCredentialStore {
|
|
|
70
69
|
return this.enqueueWrite(async () => {
|
|
71
70
|
if (this.options.mode === 'container') {
|
|
72
71
|
const next = update(this.readBlock())
|
|
73
|
-
|
|
74
|
-
{
|
|
75
|
-
kind: 'secrets-patch',
|
|
76
|
-
containerName: this.options.containerName,
|
|
77
|
-
patch: { channels: { instagram: next } },
|
|
78
|
-
},
|
|
79
|
-
{ url: this.options.hostdUrl, token: this.options.restartToken },
|
|
80
|
-
)
|
|
81
|
-
if (!response.ok) throw new Error(`secrets-patch failed: ${response.reason}`)
|
|
72
|
+
await this.options.hostProvider.writeBackChannelBlock({ instagram: next })
|
|
82
73
|
return
|
|
83
74
|
}
|
|
84
75
|
|