typeclaw 0.35.0 → 0.35.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "typeclaw",
-  "version": "0.35.0",
+  "version": "0.35.1",
   "homepage": "https://github.com/typeclaw/typeclaw#readme",
   "bugs": {
     "url": "https://github.com/typeclaw/typeclaw/issues"

package/src/agent/plugin-tools.ts

@@ -37,15 +37,16 @@ import type {
 } from '@/plugin'
 import {
   buildSandboxedCommand,
-  canBindProcSafely,
   canMountRealProc,
   DEFAULT_SANDBOX_ENV,
   ensureBwrapAvailable,
   ensureSessionTmpDir,
+  getProcBindSafetyVerdict,
   isPackageInstallCommand,
   mapVirtualTmpPath,
   resolveHiddenPaths,
   resolvePackageInstallZones,
+  resolveProcBindSafetyWithRetry,
   resolveProcSelfExe,
   resolveProtectedZones,
   resolveSandboxSymlinks,
@@ -673,12 +674,12 @@ function subtractMaskedProtected(
 // the kernel permits the mount (canMountRealProc) — it adds PID isolation but
 // needs CAP_SYS_ADMIN (unshare --mount-proc), so it is a deliberate, narrow
 // opt-in; else 'proc-bind' (--ro-bind /proc, NO CAP_SYS_ADMIN) when its userns
-// leak-block is verified safe (canBindProcSafely); else 'tmpfs'. Because
-// sandbox.realProc DEFAULTS FALSE, the first branch is normally skipped and
-// proc-bind is the de-facto default — which is the point: the common path needs
-// no broad outer capability. 'tmpfs' is the last-resort degraded mode where
-// external packages can't run; reached only when BOTH probes fail (e.g. a kernel
-// that would leak cross-userns environ — proc-bind fails closed there).
+// leak-block is verified safe; else 'tmpfs'. Because sandbox.realProc DEFAULTS
+// FALSE, the first branch is normally skipped and proc-bind is the de-facto
+// default — which is the point: the common path needs no broad outer capability.
+// 'tmpfs' is the last-resort degraded mode where external packages can't run;
+// reached only when proc-bind is DEFINITIVELY unavailable (a real cross-userns
+// environ leak → fail closed) or its safety stays unverifiable after retries.
 //
 // Read from the boot-time `config` snapshot, NOT live getConfig(): sandbox is
 // restart-required, and the strategy MUST track the boot-time CAP_SYS_ADMIN
@@ -688,7 +689,16 @@ function subtractMaskedProtected(
 // container lifetime regardless of how many bash calls hit it.
 async function resolveProcStrategy(): Promise<SandboxProcStrategy> {
   if (config.sandbox.realProc && (await canMountRealProc())) return 'real-proc'
-  if (await canBindProcSafely()) return 'proc-bind'
+  // Retry an 'inconclusive' proc-bind probe (transient under load) before
+  // degrading — a single such hiccup must not break external-package runs on a
+  // capable host. 'unsafe' still fails closed with no retry.
+  if (
+    await resolveProcBindSafetyWithRetry(
+      () => getProcBindSafetyVerdict(),
+      (ms) => Bun.sleep(ms),
+    )
+  )
+    return 'proc-bind'
   // Degraded last resort: no working /proc strategy. External package runners
   // (bunx/bun add/bun run <pkg-bin>) will fail with Bun's opaque "NotDir" because
   // /proc/self/{fd,maps} are absent. Warn once so an operator on such an exotic

package/src/inspect/live.ts

@@ -11,9 +11,15 @@ export type StreamLiveOptions = {
   onSubscribed?: (live: boolean) => void
   onError?: (message: string) => void
   connectTimeoutMs?: number
+  heartbeatIntervalMs?: number
+  pongTimeoutMs?: number
+  bufferedAmountCeiling?: number
 }
 
 const DEFAULT_CONNECT_TIMEOUT_MS = 5_000
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 10_000
+const DEFAULT_PONG_TIMEOUT_MS = 30_000
+const DEFAULT_BUFFERED_AMOUNT_CEILING = 1_048_576
 
 export async function* streamLive(opts: StreamLiveOptions): AsyncGenerator<InspectEvent> {
   const WS = opts.WebSocketImpl ?? WebSocket
@@ -26,6 +32,17 @@ export async function* streamLive(opts: StreamLiveOptions): AsyncGenerator<Inspe
   const accumulators = new Map<string, string>()
   const thinkingAccumulators = new Map<string, string>()
 
+  let heartbeat: ReturnType<typeof setInterval> | null = null
+  let awaitingPongSince: number | null = null
+  let supportsPing = false
+
+  const stopHeartbeat = (): void => {
+    if (heartbeat !== null) {
+      clearInterval(heartbeat)
+      heartbeat = null
+    }
+  }
+
   const wake = (): void => {
     if (resolveNext !== null) {
       const fn = resolveNext
@@ -43,13 +60,19 @@ export async function* streamLive(opts: StreamLiveOptions): AsyncGenerator<Inspe
       return
     }
     if (msg.type === 'subscribed') {
+      supportsPing = msg.supportsPing === true
       opts.onSubscribed?.(msg.sessionLive)
       return
     }
+    if (msg.type === 'pong') {
+      awaitingPongSince = null
+      return
+    }
     if (msg.type === 'error') {
       opts.onError?.(msg.message)
       pendingError = msg.message
       closed = true
+      stopHeartbeat()
       try {
         ws.close()
       } catch {
@@ -84,6 +107,7 @@ export async function* streamLive(opts: StreamLiveOptions): AsyncGenerator<Inspe
   })
   ws.addEventListener('close', () => {
     closed = true
+    stopHeartbeat()
     wake()
   })
 
@@ -99,6 +123,7 @@ export async function* streamLive(opts: StreamLiveOptions): AsyncGenerator<Inspe
         'abort',
         () => {
           closed = true
+          stopHeartbeat()
           try {
             ws.close()
           } catch {
@@ -134,25 +159,115 @@ export async function* streamLive(opts: StreamLiveOptions): AsyncGenerator<Inspe
   }
   ws.send(JSON.stringify(subscribe))
 
-  while (true) {
-    if (buffer.length > 0) {
-      const next = buffer.shift()!
-      yield next
-      continue
+  startHeartbeat({
+    ws,
+    intervalMs: opts.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS,
+    pongTimeoutMs: opts.pongTimeoutMs ?? DEFAULT_PONG_TIMEOUT_MS,
+    bufferedAmountCeiling: opts.bufferedAmountCeiling ?? DEFAULT_BUFFERED_AMOUNT_CEILING,
+    supportsPing: () => supportsPing,
+    isAwaitingPongSince: () => awaitingPongSince,
+    setAwaitingPongSince: (at) => {
+      awaitingPongSince = at
+    },
+    setTimer: (timer) => {
+      heartbeat = timer
+    },
+    onDead: () => {
+      closed = true
+      stopHeartbeat()
+      try {
+        ws.close()
+      } catch {
+        /* ignore */
+      }
+      wake()
+    },
+  })
+
+  try {
+    while (true) {
+      if (buffer.length > 0) {
+        const next = buffer.shift()!
+        yield next
+        continue
+      }
+      if (closed) {
+        if (pendingError !== null) throw new Error(pendingError)
+        return
+      }
+      const { event, done } = await new Promise<{ event: InspectEvent | null; done: boolean }>((resolve) => {
+        resolveNext = resolve
+      })
+      if (event !== null) yield event
+      if (done) {
+        if (pendingError !== null) throw new Error(pendingError)
+        return
+      }
+    }
+  } finally {
+    // Also fired when the consumer abandons the generator (break from a
+    // `for await` calls .return()): close the socket so it can't outlive the
+    // viewer, not just the heartbeat timer.
+    stopHeartbeat()
+    closed = true
+    try {
+      ws.close()
+    } catch {
+      /* ignore */
     }
-    if (closed) {
-      if (pendingError !== null) throw new Error(pendingError)
+  }
+}
+
+type HeartbeatOptions = {
+  ws: WebSocket
+  intervalMs: number
+  pongTimeoutMs: number
+  bufferedAmountCeiling: number
+  // Read live: the `subscribed` reply that sets it arrives after the timer is
+  // armed, so a snapshot taken at startHeartbeat time would always be false.
+  supportsPing: () => boolean
+  isAwaitingPongSince: () => number | null
+  setAwaitingPongSince: (at: number | null) => void
+  setTimer: (timer: ReturnType<typeof setInterval>) => void
+  onDead: () => void
+}
+
+// Steady-state liveness watchdog. The connect gate only bounds the OPENING
+// phase; once subscribed, a wedged socket (send queue not draining, no
+// 'close'/'error') would park the read loop forever. The interval fires on the
+// event-loop timer queue independent of the dead socket, so it always runs.
+// Two death signals, both treated as a clean close (return, never throw) so the
+// viewer recovers to the picker:
+//   1. bufferedAmount past a ceiling — our writes are not draining. Always on:
+//      it needs no server cooperation, so it works against any server version.
+//   2. a ping with no pong within the deadline — round-trip liveness lost,
+//      which also covers idle tails (a quiet-but-healthy tail still pongs). Only
+//      armed when the server advertised supportsPing; a pre-heartbeat server
+//      answers an unknown ping with error+close, so probing it would kill the
+//      tail. Such a server degrades to bufferedAmount-only detection.
+function startHeartbeat(opts: HeartbeatOptions): void {
+  let pingId = 0
+  const tick = (): void => {
+    if (opts.ws.bufferedAmount >= opts.bufferedAmountCeiling) {
+      opts.onDead()
       return
     }
-    const { event, done } = await new Promise<{ event: InspectEvent | null; done: boolean }>((resolve) => {
-      resolveNext = resolve
-    })
-    if (event !== null) yield event
-    if (done) {
-      if (pendingError !== null) throw new Error(pendingError)
+    if (!opts.supportsPing()) return
+    const awaiting = opts.isAwaitingPongSince()
+    if (awaiting !== null) {
+      if (Date.now() - awaiting >= opts.pongTimeoutMs) opts.onDead()
       return
     }
+    pingId += 1
+    const ping: InspectClientMessage = { type: 'ping', id: pingId }
+    try {
+      opts.ws.send(JSON.stringify(ping))
+      opts.setAwaitingPongSince(Date.now())
+    } catch {
+      opts.onDead()
+    }
   }
+  opts.setTimer(setInterval(tick, opts.intervalMs))
 }
 
 function frameToEvent(

package/src/sandbox/availability.ts

@@ -138,6 +138,27 @@ export function _resetRealProcProbeCacheForTests(): void {
 // future bwrap flag change, would turn this strategy into a secret leak. So we
 // PROBE it directly before ever selecting it — plant a real secret in a sibling
 // process's env and assert the sandbox cannot read it back.
+// The probe has THREE outcomes, not two — collapsing them to a boolean is what
+// caused the silent-degrade bug this verdict type fixes. 'safe'/'unsafe' are definitive capability
+// facts (the userns block held / a leak was observed); 'inconclusive' is a
+// transient local failure (probe timeout under CPU/IO contention, sentinel dying
+// mid-probe, a bwrap startup hiccup) that proves NOTHING about the host. A caller
+// deciding the /proc strategy must tell these apart: an inconclusive probe must
+// trigger a RETRY, never a fall-through to tmpfs that breaks the whole bash call
+// on a host that is actually capable. 'unsafe' must still fail closed with no
+// retry. canBindProcSafely() keeps the old boolean shape for callers that only
+// need "is proc-bind selectable right now"; getProcBindSafetyVerdict() exposes
+// the third state for the retry-owning strategy resolver.
+export type ProcBindSafetyVerdict = 'safe' | 'unsafe' | 'inconclusive'
+
+// Only DEFINITIVE verdicts are process-global facts worth caching. Caching
+// 'inconclusive' (i.e. its boolean `false`) would PERMANENTLY disable proc-bind
+// for the process — a single slow first bash call would silently break every
+// later bunx until container restart (the exact "works after restart" symptom
+// this whole machinery exists to kill). So the cache type structurally excludes
+// it.
+type CacheableProcBindSafetyVerdict = Exclude<ProcBindSafetyVerdict, 'inconclusive'>
+
 // Keyed by resolved bwrapPath, like ensureBwrapAvailable: the safety answer is a
 // fact about a SPECIFIC bwrap binary, so a caller pinning a non-default path
 // (tests, or a future deployment) must re-probe rather than inherit the default
@@ -145,19 +166,21 @@ export function _resetRealProcProbeCacheForTests(): void {
 // concurrent first callers for one path share a single probe. Both cached
 // process-globally (the answer is a per-container capability fact). Not abortable
 // (see canMountRealProc).
-const procBindProbeCache = new Map<string, boolean>()
-const procBindProbeInFlight = new Map<string, Promise<boolean>>()
-
-// `safe` is the answer; `cacheable` is false for INCONCLUSIVE outcomes (a probe
-// timeout under load, or the sentinel dying mid-probe). Those are transient
-// failure modes, not capability facts, so caching their `safe=false` would
-// PERMANENTLY disable proc-bind for the process — a single slow first bash call
-// would silently break every later bunx until container restart (the exact
-// "works after restart" symptom this whole fix exists to kill). Only a probe that
-// ran to a verdict (definitively safe OR definitively leaking) is cached.
-type ProcBindProbe = { safe: boolean; cacheable: boolean }
+const procBindProbeCache = new Map<string, CacheableProcBindSafetyVerdict>()
+const procBindProbeInFlight = new Map<string, Promise<ProcBindSafetyVerdict>>()
 
-export function canBindProcSafely(options?: { bwrapPath?: string }): Promise<boolean> {
+// `verdict` is the answer; only definitive verdicts are `cacheable`. INCONCLUSIVE
+// outcomes (a probe timeout under load, or the sentinel dying mid-probe) are
+// transient failure modes, not capability facts — see the cache rationale above.
+type ProcBindProbe =
+  | { verdict: CacheableProcBindSafetyVerdict; cacheable: true }
+  | { verdict: 'inconclusive'; cacheable: false }
+
+// The three-state probe, deduped + cached like canBindProcSafely. The strategy
+// resolver (resolveProcStrategy in plugin-tools.ts) consumes this so it can RETRY
+// an 'inconclusive' result before degrading the bash call to tmpfs, while still
+// failing closed on 'unsafe'.
+export function getProcBindSafetyVerdict(options?: { bwrapPath?: string }): Promise<ProcBindSafetyVerdict> {
   const bwrap = options?.bwrapPath ?? 'bwrap'
   const cached = procBindProbeCache.get(bwrap)
   if (cached !== undefined) return Promise.resolve(cached)
@@ -165,9 +188,9 @@ export function canBindProcSafely(options?: { bwrapPath?: string }): Promise<boo
   if (existing !== undefined) return existing
 
   const promise = probeProcBind(bwrap)
-    .then(({ safe, cacheable }) => {
-      if (cacheable) procBindProbeCache.set(bwrap, safe)
-      return safe
+    .then(({ verdict, cacheable }) => {
+      if (cacheable) procBindProbeCache.set(bwrap, verdict)
+      return verdict
     })
     .finally(() => {
       procBindProbeInFlight.delete(bwrap)
@@ -176,9 +199,53 @@ export function canBindProcSafely(options?: { bwrapPath?: string }): Promise<boo
   return promise
 }
 
+// Boolean convenience wrapper: 'safe' is the ONLY verdict that makes proc-bind
+// selectable. 'unsafe' AND 'inconclusive' both map to false — callers that only
+// take a boolean (and do not own a retry budget) must fail closed on either.
+// Derives from the deduped verdict probe, so concurrent callers still share one
+// spawn even though this wrapper's own promise identity differs per call.
+export function canBindProcSafely(options?: { bwrapPath?: string }): Promise<boolean> {
+  return getProcBindSafetyVerdict(options).then((verdict) => verdict === 'safe')
+}
+
+// Default backoff between proc-bind safety re-probes, in ms. Array length = retry
+// count (2 retries after the initial attempt = 3 probes total). The probe is
+// normally sub-ms; it only returns 'inconclusive' under transient CPU/IO
+// contention (e.g. a boot-time storm of concurrent LLM calls saturating the box
+// and tripping the probe's own timeout), so a short staggered wait lets the spike
+// pass before re-proving.
+export const PROC_BIND_RETRY_BACKOFF_MS = [250, 1_000] as const
+
+// proc-bind selection must distinguish "definitely unavailable" from "couldn't
+// verify right now". A DEFINITIVE verdict is final: 'safe'→true; a real userns
+// leak ('unsafe')→false with NO retry. Only an 'inconclusive' verdict (transient
+// probe failure that proves nothing about the host) is retried, because degrading
+// the bash call to tmpfs over a transient hiccup is what silently broke
+// external-package runs on capable hosts. 'inconclusive' is never cached
+// (see the cache type), so each retry re-probes from scratch. After the backoff
+// budget is exhausted we fail CLOSED — an unverified leak-block is never treated
+// as safe. Pure and dependency-injected (probe + sleep) so the retry policy is
+// unit-testable without spawning processes; production passes
+// getProcBindSafetyVerdict and Bun.sleep.
+export async function resolveProcBindSafetyWithRetry(
+  probe: () => Promise<ProcBindSafetyVerdict>,
+  sleep: (ms: number) => Promise<void>,
+  backoffMs: readonly number[] = PROC_BIND_RETRY_BACKOFF_MS,
+): Promise<boolean> {
+  for (let attempt = 0; ; attempt++) {
+    const verdict = await probe()
+    if (verdict === 'safe') return true
+    if (verdict === 'unsafe') return false
+
+    const backoff = backoffMs[attempt]
+    if (backoff === undefined) return false
+    await sleep(backoff)
+  }
+}
+
 const PROC_BIND_PROBE_SECRET = 'TYPECLAW_PROCBIND_PROBE_SECRET'
 
-const INCONCLUSIVE: ProcBindProbe = { safe: false, cacheable: false }
+const INCONCLUSIVE: ProcBindProbe = { verdict: 'inconclusive', cacheable: false }
 
 async function probeProcBind(bwrap: string): Promise<ProcBindProbe> {
   // The sentinel must model the REAL threat geometry: the agent runtime holds
@@ -277,13 +344,13 @@ async function probeProcBind(bwrap: string): Promise<ProcBindProbe> {
     // "non-zero" — a non-zero exit also covers script setup failures (a bwrap that
     // started but couldn't read /proc/self/fd), bwrap startup failures (missing
     // lib, transient mount EBUSY → bwrap's own exit), and an external SIGKILL.
-    // Caching any of those transient failures as a definitive safe=false would
+    // Caching any of those transient failures as a definitive 'unsafe' would
     // PERMANENTLY disable proc-bind — the same cache-poisoning class as the
     // timeout bug. So only the script's two designated codes are cacheable:
     // PROC_BIND_SAFE (clean run, every open blocked) and PROC_BIND_LEAK (an open
     // SUCCEEDED — a real leak). Setup failures use PROC_BIND_SETUP_FAILED, and any
     // other code (bwrap startup, signals, 127) is treated as inconclusive.
-    if (proc.exitCode === PROC_BIND_LEAK) return { safe: false, cacheable: true }
+    if (proc.exitCode === PROC_BIND_LEAK) return { verdict: 'unsafe', cacheable: true }
     if (proc.exitCode !== PROC_BIND_SAFE) return INCONCLUSIVE
     // Final liveness: the in-sandbox blocked-open assertions are only meaningful
     // if the sentinel was alive throughout. Re-read its MARKER from the PARENT —
@@ -293,12 +360,13 @@ async function probeProcBind(bwrap: string): Promise<ProcBindProbe> {
     // kernel liveness, so this marker re-read is the stronger postcondition. A
     // failure here means the sentinel vanished mid-probe → inconclusive.
     if (!(await parentReadsSentinelMarker(sentinelPid))) return INCONCLUSIVE
-    return { safe: true, cacheable: true }
+    return { verdict: 'safe', cacheable: true }
   } catch {
     return INCONCLUSIVE
   } finally {
     try {
       sentinel?.kill()
+      await sentinel?.exited.catch(() => {})
     } catch {
       // killing an already-exited sentinel can throw on some runtimes; cleanup
       // must never propagate out of the probe.

package/src/sandbox/index.ts

@@ -4,7 +4,11 @@ export {
   canBindProcSafely,
   canMountRealProc,
   ensureBwrapAvailable,
+  getProcBindSafetyVerdict,
+  PROC_BIND_RETRY_BACKOFF_MS,
+  resolveProcBindSafetyWithRetry,
   resolveProcSelfExe,
+  type ProcBindSafetyVerdict,
   _resetBwrapAvailabilityCacheForTests,
   _resetProcBindProbeCacheForTests,
   _resetRealProcProbeCacheForTests,

package/src/server/index.ts

@@ -1265,6 +1265,10 @@ function handleInspectMessage(
     ws.close()
     return
   }
+  if (msg.type === 'ping') {
+    sendInspect(ws, { type: 'pong', id: msg.id })
+    return
+  }
   if (msg.type !== 'subscribe' || typeof msg.sessionId !== 'string' || msg.sessionId === '') {
     sendInspect(ws, { type: 'error', message: 'invalid inspect subscription' })
     ws.close()
@@ -1314,7 +1318,7 @@ function handleInspectMessage(
     })
   }
 
-  sendInspect(ws, { type: 'subscribed', sessionId: msg.sessionId, sessionLive: live !== undefined })
+  sendInspect(ws, { type: 'subscribed', sessionId: msg.sessionId, sessionLive: live !== undefined, supportsPing: true })
 }
 
 function extractJobId(target: StreamMessage['target']): string {

package/src/shared/protocol.ts

@@ -44,16 +44,22 @@ export type TunnelLogsServerMessage =
   | { type: 'error'; message: string }
   | { type: 'end' }
 
-export type InspectClientMessage = {
-  type: 'subscribe'
-  sessionId: string
-  // sinceMs is a wall-clock cutoff for backfilling broadcasts from the
-  // in-process Stream ring buffer. The client uses Date.now() - duration;
-  // omit to skip broadcast backfill. AgentSession events are NEVER
-  // backfilled (the session's pi-coding-agent subscribe API delivers
-  // future events only).
-  sinceMs?: number
-}
+export type InspectClientMessage =
+  | {
+      type: 'subscribe'
+      sessionId: string
+      // sinceMs is a wall-clock cutoff for backfilling broadcasts from the
+      // in-process Stream ring buffer. The client uses Date.now() - duration;
+      // omit to skip broadcast backfill. AgentSession events are NEVER
+      // backfilled (the session's pi-coding-agent subscribe API delivers
+      // future events only).
+      sinceMs?: number
+    }
+  // Steady-state liveness probe echoed back as a pong. A live tail is
+  // legitimately quiet for long stretches, so absence of inbound frames cannot
+  // distinguish "idle" from "dead"; a missed pong can. Guards a wedged
+  // WebSocket that stays ESTABLISHED yet never fires 'close'/'error'.
+  | { type: 'ping'; id: number }
 
 export type InspectFramePayload =
   | { kind: 'text_delta'; sessionId: string; delta: string }
@@ -123,9 +129,14 @@ export type InspectFramePayload =
     }
 
 export type InspectServerMessage =
-  | { type: 'subscribed'; sessionId: string; sessionLive: boolean }
+  // supportsPing is the heartbeat capability flag. A pre-heartbeat server omits
+  // it; the client must treat its absence as "no ping support" and never send a
+  // ping (an old server answers an unknown ping with an error + close, killing
+  // the tail). Strict opt-in: only an explicit true arms round-trip probing.
+  | { type: 'subscribed'; sessionId: string; sessionLive: boolean; supportsPing?: true }
   | { type: 'frame'; ts: number; payload: InspectFramePayload }
   | { type: 'error'; message: string }
+  | { type: 'pong'; id: number }
 
 export type ClientMessage =
   | { type: 'prompt'; text: string; delivery?: PromptDelivery }