thinkpool-pair 0.7.356 → 0.7.357
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bridge.mjs +37 -1
- package/claude-session.mjs +231 -12
- package/event-bounds.mjs +1 -1
- package/evidence-citations.mjs +3 -9
- package/evidence-compact.mjs +11 -0
- package/package.json +2 -1
- package/provider-resilience.mjs +148 -5
- package/providers.mjs +61 -2
package/bridge.mjs
CHANGED
|
@@ -47,7 +47,8 @@ import { createPermNotifier, shouldNotifyTurnDone, shouldRecordTurnDone, permiss
|
|
|
47
47
|
// resolveProviderEnv(id) → {ANTHROPIC_BASE_URL,ANTHROPIC_AUTH_TOKEN,ANTHROPIC_MODEL} for a
|
|
48
48
|
// registered custom provider, or null for the built-in/unknown (leave the default env intact).
|
|
49
49
|
// Multi-provider BYOK slice 1: a lane spawned with a `provider` id runs on that endpoint.
|
|
50
|
-
import { resolveProviderEnv, resolveProviderRef, providerNameMap, publicKeyB64, bridgeHostId, announceProviders, listProviders, effectiveLaneModel, sameProviderEnv, providerModel } from './providers.mjs'
|
|
50
|
+
import { resolveProviderEnv, resolveProviderRef, resolveProviderResiliencePolicy, providerNameMap, publicKeyB64, bridgeHostId, announceProviders, listProviders, effectiveLaneModel, sameProviderEnv, providerModel } from './providers.mjs'
|
|
51
|
+
import { createMemoryCircuit, providerResilienceCapAdmission } from './provider-resilience.mjs'
|
|
51
52
|
import { validateProviderSwitch, providerSwitchPlan, BUILTIN_PROVIDER } from './switch-provider.mjs'
|
|
52
53
|
import { createSdkMcpServer, tool } from '@anthropic-ai/claude-agent-sdk'
|
|
53
54
|
import { z } from 'zod'
|
|
@@ -672,6 +673,10 @@ const BRIDGE_STARTED_AT = Date.now()
|
|
|
672
673
|
// top-level only (one bridge per announce); older clients ignore the unknown key.
|
|
673
674
|
const host = (os.hostname() || 'host').split('.')[0].slice(0, 24)
|
|
674
675
|
const hostId = bridgeHostId()
|
|
676
|
+
// Phase 1 circuit state is deliberately memory-only. Its primitive keys state
|
|
677
|
+
// by bridge/provider/exact-model; a process restart clears it and no credential,
|
|
678
|
+
// endpoint, prompt, or raw error ever enters the map.
|
|
679
|
+
const providerResilienceCircuit = createMemoryCircuit()
|
|
675
680
|
|
|
676
681
|
// Repo awareness — the room shows which project this machine is sharing.
|
|
677
682
|
// Cheap reads, no subprocess: directory name + .git/HEAD.
|
|
@@ -3254,6 +3259,33 @@ function openStructured({ id, runtime = 'claude', model, models, effort, resume,
|
|
|
3254
3259
|
flowRole: entry.flowRole,
|
|
3255
3260
|
sideParent: entry.sideParent,
|
|
3256
3261
|
}), canSpawnWorkers ? HERMES_VISIBLE_WORKER_FALLBACK_RULE : ''].filter(Boolean).join('\n\n')
|
|
3262
|
+
const resolvedProviderResilience = runtime === 'claude'
|
|
3263
|
+
? resolveProviderResiliencePolicy(provider)
|
|
3264
|
+
: null
|
|
3265
|
+
let resilienceCapAbsenceLogged = false
|
|
3266
|
+
const providerResilience = resolvedProviderResilience?.policy?.enabled === true
|
|
3267
|
+
? {
|
|
3268
|
+
policy: resolvedProviderResilience.policy,
|
|
3269
|
+
providers: listProviders(),
|
|
3270
|
+
providerId: resolvedProviderResilience.target.providerId,
|
|
3271
|
+
model: resolvedProviderResilience.target.actualConfiguredModel,
|
|
3272
|
+
requestedModel: laneModel || resolvedProviderResilience.target.actualConfiguredModel,
|
|
3273
|
+
bridgeHostId: hostId,
|
|
3274
|
+
circuit: providerResilienceCircuit,
|
|
3275
|
+
// sendTurn creates the controller immediately before the bridge advances
|
|
3276
|
+
// its public turn revision, so project the revision that admission owns.
|
|
3277
|
+
turnRev: () => (Number(entry._turnRev) || 0) + 1,
|
|
3278
|
+
capGate: () => {
|
|
3279
|
+
const budget = entry.flowSessionId ? flowBudgets.get(entry.flowSessionId) : null
|
|
3280
|
+
const admission = providerResilienceCapAdmission(budget)
|
|
3281
|
+
if (!admission.configured && !resilienceCapAbsenceLogged) {
|
|
3282
|
+
resilienceCapAbsenceLogged = true
|
|
3283
|
+
process.stderr.write(`\n ${A.dim}◇ provider resilience cap_not_configured (${String(id).slice(0, 8)})${A.rst}\n`)
|
|
3284
|
+
}
|
|
3285
|
+
return admission
|
|
3286
|
+
},
|
|
3287
|
+
}
|
|
3288
|
+
: null
|
|
3257
3289
|
entry.session = startStructuredSession(runtime, {
|
|
3258
3290
|
// laneModel, NOT the raw `model` param: the SDK's `model` option OVERRIDES the
|
|
3259
3291
|
// ANTHROPIC_MODEL supplied by resolveProviderEnv() in `env` below, so an inherited
|
|
@@ -3360,6 +3392,10 @@ function openStructured({ id, runtime = 'claude', model, models, effort, resume,
|
|
|
3360
3392
|
// per-machine default (provider.mjs/applyProviderEnv). null for built-in/unknown → the
|
|
3361
3393
|
// default Claude env is left exactly as-is (unchanged path).
|
|
3362
3394
|
env: { ...process.env, ...buildConductorEnv({ flowSessionId, mode }), ...(resolveProviderEnv(provider) || {}), TP_MOCKUP_OUTBOX: mockupOutbox },
|
|
3395
|
+
// Same-target custom-provider resilience is a host-local dark-launch
|
|
3396
|
+
// primitive. Missing/disabled policy, built-in Anthropic, Codex, and Hermes
|
|
3397
|
+
// receive null and preserve their established transport behavior exactly.
|
|
3398
|
+
resilience: providerResilience,
|
|
3363
3399
|
onTurnStart: (options = {}) => {
|
|
3364
3400
|
// Hermes promotes /queue items internally, without a second code-turn.
|
|
3365
3401
|
// Advance the lifecycle before its first output and publish the deferred
|
package/claude-session.mjs
CHANGED
|
@@ -25,6 +25,7 @@ import { normalizeClaudeCommandCatalog } from './claude-command-catalog.mjs'
|
|
|
25
25
|
import { evidenceForToolResult } from './evidence-citations.mjs'
|
|
26
26
|
import { THINKPOOL_CASCADE_RULE, THINKPOOL_REMOTE_DELIVERY_RULES, THINKPOOL_RUNTIME_AUTHORITY_RULE, THINKPOOL_RUNTIME_TURN_REMINDER, buildThinkPoolTurnGuidance, createRoomContextSelector, usesFullThinkPoolReminder } from './thinkpool-room-prompt.mjs'
|
|
27
27
|
import { stallDecision, stallEvent, isCompactTurn } from './turn-stall.mjs'
|
|
28
|
+
import { createSameTargetResilienceController, formatResilienceTrace } from './provider-resilience.mjs'
|
|
28
29
|
|
|
29
30
|
// The caret-pulled SDK's real version (^0.3.x auto-upgrades on restart). Resolved
|
|
30
31
|
// once at import by walking up from the package entry to its own package.json.
|
|
@@ -245,7 +246,7 @@ const TP_ROOM_REMINDER = [
|
|
|
245
246
|
'BUILD WORKFLOW (default, no magic word): right-size within your TERMINAL ROLE — a trivial ask or delegated slice you just do; a conductor-capable role with a genuinely decomposable build FIRST writes a short plan in chat, THEN fans worker slices into visible spawn_terminal lanes and verifies them. Worker/leaf/Side/managed Flow roles do not fan out. A person-requested new or separate terminal uses open_main_terminal. Never plan-mode/ExitPlanMode; plans live in chat and lanes in the existing list.',
|
|
246
247
|
].join(' ')
|
|
247
248
|
|
|
248
|
-
export function startClaudeSession({ cwd, model, effort: initialEffort = 'high', resume, env, mode: initialMode = 'default', onEvent, requestPermission, mcpServers, crossPostGate, crossRoomPostGate, didSpawnTarget = null, terminalRolePrompt, rolePrompt, blockSubagents = false, onSubmitPlan = null, onLaneDone = null, onReviewVerdict = null, reviewGate = null, lazy = false, roomContext = null, suggest = true, prepareCwd = null, admitStart = null, resilienceObserver = null }) {
|
|
249
|
+
export function startClaudeSession({ cwd, model, effort: initialEffort = 'high', resume, env, mode: initialMode = 'default', onEvent, requestPermission, mcpServers, crossPostGate, crossRoomPostGate, didSpawnTarget = null, terminalRolePrompt, rolePrompt, blockSubagents = false, onSubmitPlan = null, onLaneDone = null, onReviewVerdict = null, reviewGate = null, lazy = false, roomContext = null, suggest = true, prepareCwd = null, admitStart = null, resilienceObserver = null, resilience = null }) {
|
|
249
250
|
// Per-turn reminder + live ROOM NOW tail. roomContext (bridge-supplied) returns the
|
|
250
251
|
// room's CURRENT state — sibling lanes, active git worktrees — or null. The static
|
|
251
252
|
// rules keep the agent aware of the room's FEATURES; the live tail keeps it aware of
|
|
@@ -272,6 +273,7 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
272
273
|
let persistedSessionId = resume || null
|
|
273
274
|
let lastTurnText = null // the most recent turn text, so a bad-resume recovery can re-deliver it
|
|
274
275
|
let lastTurnReminder = null
|
|
276
|
+
let lastTurnBlocks = null // exact current prompt blocks; same-target retry must replay byte-for-byte
|
|
275
277
|
let closed = false
|
|
276
278
|
// Lazy boot (2026-07-02): a RESTORED-IDLE terminal returns a full session object but
|
|
277
279
|
// defers the expensive query() cold-start (MCP + settingSources, ~50s each) until its
|
|
@@ -352,6 +354,14 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
352
354
|
// Phase 0 measurement seam. The observer receives only a coarse recovery
|
|
353
355
|
// reason/count; it cannot alter this session's existing retry behavior.
|
|
354
356
|
const observeRecovery = (reason, detail = {}) => { try { resilienceObserver?.({ reason, restartCount, ...detail }) } catch { /* observer is strictly read-only */ } }
|
|
357
|
+
// Phase 1 is opt-in and applies only to a named non-Anthropic custom target. Keep
|
|
358
|
+
// disabled/built-in sessions on the established RESTART_MAX recovery path exactly.
|
|
359
|
+
const resilienceEnabled = !!(resilience?.policy?.enabled && resilience?.providerId && resilience.providerId !== 'anthropic')
|
|
360
|
+
let turnResilience = null
|
|
361
|
+
let resilienceDeadlineTimer = null
|
|
362
|
+
let resilienceRetryPending = false
|
|
363
|
+
let resilienceTurnRevision = 0
|
|
364
|
+
let abortPending = false
|
|
355
365
|
let restartTimer = null // the pending auto-restart backoff — cancelled by end()
|
|
356
366
|
// Force-stop a true wedge (item 3): no result, no error, just silence past
|
|
357
367
|
// FORCE_STOP_MS. As of the 2026-07-08 hardening we no longer just surface an error
|
|
@@ -400,6 +410,7 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
400
410
|
// cleared on the next result — "all teardown has arrived" isn't observable. `success`
|
|
401
411
|
// results are never swallowed, so the flushed steer's completion always surfaces.
|
|
402
412
|
let interrupting = false
|
|
413
|
+
let interruptingRevision = null
|
|
403
414
|
let interruptTimer = null
|
|
404
415
|
const INTERRUPT_SWALLOW_MS = 6000
|
|
405
416
|
// ── Haiku suggestion fallback ──
|
|
@@ -423,6 +434,59 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
423
434
|
// (`result`/`error` — covers a no-op "Not enough messages to compact" and a Stop). After
|
|
424
435
|
// that the normal wedge timeline applies again.
|
|
425
436
|
const emit = (evt) => { lastEvtTs = Date.now(); if (evt && evt.kind !== 'stalled') stalledSent = false; if (evt && (evt.kind === 'compaction' || evt.kind === 'result' || evt.kind === 'error')) compacting = false; emitRaw(evt) }
|
|
437
|
+
// The controller only receives its own allowlisted records. Room notes are likewise
|
|
438
|
+
// rendered from that projection, never from an SDK error, URL, credential, or prompt.
|
|
439
|
+
const resilienceRecord = (record) => {
|
|
440
|
+
try { resilience?.onRecord?.(record) } catch { /* observer is never control flow */ }
|
|
441
|
+
if (record?.outcome === 'retrying' || record?.outcome === 'cap_blocked' || record?.outcome === 'circuit_open') {
|
|
442
|
+
emit({ kind: 'note', text: formatResilienceTrace(record) })
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
const clearResilienceDeadline = () => {
|
|
446
|
+
if (resilienceDeadlineTimer == null) return
|
|
447
|
+
try { (resilience?.clearTimer || clearTimeout)(resilienceDeadlineTimer) } catch { /* deadline cleanup is best-effort */ }
|
|
448
|
+
resilienceDeadlineTimer = null
|
|
449
|
+
}
|
|
450
|
+
const armResilienceDeadline = () => {
|
|
451
|
+
clearResilienceDeadline()
|
|
452
|
+
if (!turnResilience) return
|
|
453
|
+
const timeoutMs = Number(resilience?.policy?.timeoutMs)
|
|
454
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return
|
|
455
|
+
const schedule = resilience?.setTimer || setTimeout
|
|
456
|
+
resilienceDeadlineTimer = schedule(() => {
|
|
457
|
+
resilienceDeadlineTimer = null
|
|
458
|
+
if (closed || !turnResilience || !turnActive) return
|
|
459
|
+
// This is a bridge-authored timeout classification, never a raw provider
|
|
460
|
+
// error. Abort the wedged iterator, then reuse the same controller budget.
|
|
461
|
+
turnResilience.failure({ message: 'timeout' })
|
|
462
|
+
turnActive = false
|
|
463
|
+
try { qAc?.abort() } catch { /* the disposal wait below still fails closed */ }
|
|
464
|
+
retryResilientTurn()
|
|
465
|
+
}, timeoutMs)
|
|
466
|
+
}
|
|
467
|
+
const createTurnResilience = () => {
|
|
468
|
+
if (!resilienceEnabled) return null
|
|
469
|
+
try {
|
|
470
|
+
const controller = createSameTargetResilienceController({
|
|
471
|
+
runtime: 'claude', providers: resilience.providers || [], providerId: resilience.providerId,
|
|
472
|
+
model: resilience.model || opts.model || model,
|
|
473
|
+
requestedModel: resilience.requestedModel || resilience.model || opts.model || model,
|
|
474
|
+
policy: resilience.policy, bridgeHostId: resilience.bridgeHostId,
|
|
475
|
+
circuit: resilience.circuit, capGate: resilience.capGate,
|
|
476
|
+
traceId: resilience.traceId,
|
|
477
|
+
turnRev: typeof resilience.turnRev === 'function' ? resilience.turnRev() : resilience.turnRev,
|
|
478
|
+
onRecord: resilienceRecord,
|
|
479
|
+
})
|
|
480
|
+
controller.start()
|
|
481
|
+
return controller
|
|
482
|
+
} catch { return null }
|
|
483
|
+
}
|
|
484
|
+
const admitResilientSubmission = async () => {
|
|
485
|
+
if (!turnResilience) return true
|
|
486
|
+
let admission = null
|
|
487
|
+
try { admission = await turnResilience.preflight() } catch { return false }
|
|
488
|
+
return admission?.admitted === true
|
|
489
|
+
}
|
|
426
490
|
const stallTimer = setInterval(() => {
|
|
427
491
|
const quiet = Date.now() - lastEvtTs
|
|
428
492
|
const action = stallDecision({ turnActive, awaitingUser, quietMs: quiet, stallMs: STALL_MS, forceStopMs: FORCE_STOP_MS, stalledSent, stallRetried, compacting, compactForceStopMs: COMPACT_FORCE_STOP_MS })
|
|
@@ -432,7 +496,23 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
432
496
|
const ev = stallEvent(action, quiet)
|
|
433
497
|
if (ev) emitRaw(ev)
|
|
434
498
|
if (action === 'status') { stalledSent = true; return }
|
|
435
|
-
if (action === 'retry') {
|
|
499
|
+
if (action === 'retry') {
|
|
500
|
+
forceStopped = true
|
|
501
|
+
if (turnResilience) {
|
|
502
|
+
// The legacy watchdog must never create a second retry ledger. Route
|
|
503
|
+
// its terminal timeout through the same controller; visible output or
|
|
504
|
+
// an exhausted attempt budget makes the following preflight fail closed.
|
|
505
|
+
observeRecovery('resilience_stall', { quietMs: quiet })
|
|
506
|
+
turnResilience.failure({ message: 'timeout' })
|
|
507
|
+
turnActive = false
|
|
508
|
+
try { qAc?.abort() } catch { /* disposal is awaited by the shared retry path */ }
|
|
509
|
+
retryResilientTurn()
|
|
510
|
+
return
|
|
511
|
+
}
|
|
512
|
+
observeRecovery('stall_replay', { quietMs: quiet })
|
|
513
|
+
retryStalledTurn(quiet)
|
|
514
|
+
return
|
|
515
|
+
}
|
|
436
516
|
// 'giveup' — the one auto-retry ALSO stalled past FORCE_STOP_MS. Fall back to the
|
|
437
517
|
// pre-2026-07-08 behavior: force-stop the turn so between-turns updates unblock, and
|
|
438
518
|
// let the human resend. The wedged loop is left in place; if it later throws, the
|
|
@@ -686,6 +766,7 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
686
766
|
// On deny, permissionDecisionReason IS what the model receives as the
|
|
687
767
|
// tool error — make it a real instruction, not an opaque tag.
|
|
688
768
|
const denied = decision === 'deny'
|
|
769
|
+
if (denied && turnResilience) turnResilience.failure({ permissionDenied: true })
|
|
689
770
|
return {
|
|
690
771
|
continue: true,
|
|
691
772
|
hookSpecificOutput: {
|
|
@@ -951,6 +1032,10 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
951
1032
|
}
|
|
952
1033
|
break
|
|
953
1034
|
case 'assistant':
|
|
1035
|
+
clearResilienceDeadline()
|
|
1036
|
+
// Any assistant block (including a tool_use) has crossed the replay
|
|
1037
|
+
// boundary. A same-target retry is only safe before visible output.
|
|
1038
|
+
turnResilience?.visibleOutput()
|
|
954
1039
|
// Stamp tool-call start times so tool_result can report a duration.
|
|
955
1040
|
for (const b of (m.message?.content || [])) {
|
|
956
1041
|
if (b?.type === 'tool_use' && b.id) toolStart.set(b.id, { at: Date.now(), name: b.name, input: b.input })
|
|
@@ -966,6 +1051,8 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
966
1051
|
// tool_result blocks arrive on the user-role echo
|
|
967
1052
|
for (const b of (m.message?.content || [])) {
|
|
968
1053
|
if (b?.type === 'tool_result') {
|
|
1054
|
+
clearResilienceDeadline()
|
|
1055
|
+
turnResilience?.visibleOutput({ tool: true })
|
|
969
1056
|
const start = toolStart.get(b.tool_use_id)
|
|
970
1057
|
if (start != null) toolStart.delete(b.tool_use_id)
|
|
971
1058
|
const evidence = evidenceForToolResult(start?.name, b.content)
|
|
@@ -990,6 +1077,7 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
990
1077
|
break
|
|
991
1078
|
}
|
|
992
1079
|
case 'result':
|
|
1080
|
+
clearResilienceDeadline()
|
|
993
1081
|
if (m.session_id) sessionId = m.session_id
|
|
994
1082
|
// Bad resume target: the CLI can't find the session we tried to resume — a forked
|
|
995
1083
|
// id that was never persisted, or a pruned transcript. It surfaces as an is_error
|
|
@@ -1019,7 +1107,26 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
1019
1107
|
// abort() already emitted the one canonical aborted boundary. `success` (and any
|
|
1020
1108
|
// non-teardown subtype) always emits; only the aborted/error_during_execution pair
|
|
1021
1109
|
// the interrupt churns out is dropped, and only inside the timer-bounded window.
|
|
1022
|
-
|
|
1110
|
+
// This guard MUST precede resilient `is_error` handling: the second teardown echo
|
|
1111
|
+
// is itself an is_error result and may arrive after the next turn has installed a
|
|
1112
|
+
// new controller. Letting it reach that controller would retry the wrong prompt.
|
|
1113
|
+
if (interrupting && (m.subtype === 'aborted' || m.subtype === 'error_during_execution')) {
|
|
1114
|
+
// A late teardown echo from the stopped turn may arrive after the
|
|
1115
|
+
// next turn is accepted. Swallow the echo, but only mutate liveness
|
|
1116
|
+
// while the stopped revision still owns the lane.
|
|
1117
|
+
if (resilienceTurnRevision === interruptingRevision) turnActive = false
|
|
1118
|
+
break
|
|
1119
|
+
}
|
|
1120
|
+
// Some SDK transport failures arrive as an error result instead of a
|
|
1121
|
+
// thrown iterator error. Treat them identically, without serializing
|
|
1122
|
+
// `m.errors` or `m.result` into the room event stream.
|
|
1123
|
+
if (m.is_error && turnResilience) {
|
|
1124
|
+
if (resilienceRetryPending) break
|
|
1125
|
+
turnResilience.failure({ message: Array.isArray(m.errors) ? m.errors.join(' ') : m.result, status: m.status ?? m.statusCode, code: m.code })
|
|
1126
|
+
turnActive = false
|
|
1127
|
+
retryResilientTurn()
|
|
1128
|
+
break
|
|
1129
|
+
}
|
|
1023
1130
|
turnBaseOut = 0; curMsgOut = 0 // reset the live token count for the next turn
|
|
1024
1131
|
turnActive = false // turn settled → stall watchdog stands down
|
|
1025
1132
|
// A model switch requested mid-turn was deferred — apply it now the turn is done.
|
|
@@ -1027,6 +1134,7 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
1027
1134
|
// doesn't await itself; the setTimeout runs after this iterator yields.
|
|
1028
1135
|
if (pendingSwitch) { pendingSwitch = false; setTimeout(() => { if (!closed) recreateForSwitch() }, 0) }
|
|
1029
1136
|
if (m.subtype === 'success') { restartCount = 0; if (sessionId) persistedSessionId = sessionId } // ONLY a real success refills the
|
|
1137
|
+
if (m.subtype === 'success') { turnResilience?.success(); turnResilience = null }
|
|
1030
1138
|
// auto-restart budget (else a flapping connection that lands one aborted turn between
|
|
1031
1139
|
// drops refills every cycle past RESTART_MAX) — and marks this session id durably
|
|
1032
1140
|
// persisted so a later model-switch re-create can safely resume it.
|
|
@@ -1080,11 +1188,22 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
1080
1188
|
}
|
|
1081
1189
|
}
|
|
1082
1190
|
} catch (e) {
|
|
1191
|
+
clearResilienceDeadline()
|
|
1083
1192
|
if (closed) return
|
|
1084
1193
|
// This query was intentionally aborted (a model switch superseded it, or the
|
|
1085
1194
|
// session is ending) — not a real error. Stay silent; the re-create owns what's next.
|
|
1086
1195
|
if (myAc.signal.aborted) return
|
|
1087
1196
|
const msg = e?.message || String(e)
|
|
1197
|
+
// Enabled custom-provider turns have a strict, per-turn controller. It
|
|
1198
|
+
// classifies the raw transport error locally, then either admits one exact
|
|
1199
|
+
// replay before any output or terminates without putting provider details in
|
|
1200
|
+
// the room. The legacy path below is intentionally untouched when disabled.
|
|
1201
|
+
if (turnResilience && turnActive) {
|
|
1202
|
+
turnResilience.failure({ message: msg, status: e?.status ?? e?.statusCode, code: e?.code })
|
|
1203
|
+
turnActive = false
|
|
1204
|
+
retryResilientTurn()
|
|
1205
|
+
return
|
|
1206
|
+
}
|
|
1088
1207
|
// onEvent is also the bridge's synchronous busy-edge sampling point. The
|
|
1089
1208
|
// query has already ended here, so publish the failure only after clearing
|
|
1090
1209
|
// the turn. Otherwise a non-recoverable SDK error leaves the roster on its
|
|
@@ -1192,12 +1311,63 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
1192
1311
|
emitRaw({ kind: 'note', text: 'retrying the stalled turn on a fresh connection' })
|
|
1193
1312
|
}
|
|
1194
1313
|
}
|
|
1195
|
-
|
|
1314
|
+
// A resilience retry has to wait for the failed Query's disposal: otherwise the
|
|
1315
|
+
// replay races its session-file lock. Admission is deliberately before runQuery(),
|
|
1316
|
+
// so a refused cap/circuit sends no second provider submission.
|
|
1317
|
+
const retryResilientTurn = () => {
|
|
1318
|
+
if (closed || abortPending || resilienceRetryPending) return
|
|
1319
|
+
const retryingTurn = turnResilience
|
|
1320
|
+
const retryingRevision = resilienceTurnRevision
|
|
1321
|
+
const retryingBlocks = lastTurnBlocks
|
|
1322
|
+
resilienceRetryPending = true
|
|
1323
|
+
const dying = qDone
|
|
1324
|
+
const oldInput = input
|
|
1325
|
+
input = makeInputStream()
|
|
1326
|
+
try { oldInput.end() } catch { /* noop */ }
|
|
1327
|
+
restartTimer = setTimeout(async () => {
|
|
1328
|
+
restartTimer = null
|
|
1329
|
+
try { await dying } catch { /* disposal failure still fails closed below */ }
|
|
1330
|
+
if (closed || !resilienceRetryPending || turnResilience !== retryingTurn || resilienceTurnRevision !== retryingRevision) return
|
|
1331
|
+
if (!(await admitResilientSubmission())) {
|
|
1332
|
+
resilienceRetryPending = false
|
|
1333
|
+
finishResilientFailure()
|
|
1334
|
+
return
|
|
1335
|
+
}
|
|
1336
|
+
if (closed || !resilienceRetryPending || turnResilience !== retryingTurn || resilienceTurnRevision !== retryingRevision) return
|
|
1337
|
+
resilienceRetryPending = false
|
|
1338
|
+
turnActive = true
|
|
1339
|
+
lastEvtTs = Date.now()
|
|
1340
|
+
stalledSent = false
|
|
1341
|
+
runQuery()
|
|
1342
|
+
// This is the exact previously accepted prompt block array, not a rebuilt
|
|
1343
|
+
// string/reminder. It is only reached before assistant or tool output.
|
|
1344
|
+
if (retryingBlocks) input.push(retryingBlocks)
|
|
1345
|
+
armResilienceDeadline()
|
|
1346
|
+
}, 0)
|
|
1347
|
+
}
|
|
1348
|
+
const finishResilientFailure = () => {
|
|
1349
|
+
clearResilienceDeadline()
|
|
1350
|
+
turnActive = false
|
|
1351
|
+
turnResilience = null
|
|
1352
|
+
// The failed Query is gone, but the lane remains recoverable for a later
|
|
1353
|
+
// human turn. `started=false` ensures that turn creates a fresh Query rather
|
|
1354
|
+
// than pushing into the ended stream.
|
|
1355
|
+
started = false
|
|
1356
|
+
emit({
|
|
1357
|
+
kind: 'error',
|
|
1358
|
+
message: 'The configured provider could not complete this turn. Nothing else was sent.',
|
|
1359
|
+
recoverable: true,
|
|
1360
|
+
})
|
|
1361
|
+
}
|
|
1362
|
+
// A custom resilient session defers its otherwise-eager query creation until a
|
|
1363
|
+
// turn clears the injected admission gate. Disabled and built-in sessions retain
|
|
1364
|
+
// their existing eager start and RESTART_MAX behavior.
|
|
1365
|
+
if (!lazy && !resilienceEnabled) runQuery()
|
|
1196
1366
|
|
|
1197
1367
|
return {
|
|
1198
1368
|
// A lazy (restored-idle) session cold-boots the query on its first turn. input.push
|
|
1199
1369
|
// is queue-backed, so the pushed turn buffers and runs once the query is ready.
|
|
1200
|
-
sendTurn(text) { if (
|
|
1370
|
+
sendTurn(text) { if (closed || abortPending || resilienceRetryPending) return false; if (!started && !admitColdStart()) return false; if (!started && prepareCwd) { try { const next = prepareCwd(); if (next) { cwd = next; opts.cwd = next } } catch { /* keep original cwd */ } } turnActive = true; sawSuggestion = false; lastTurnText = String(text); if (sugTimer) { clearTimeout(sugTimer); sugTimer = null } lastEvtTs = Date.now(); stalledSent = false; stallRetried = false; forceStopped = false;
|
|
1201
1371
|
const t = String(text)
|
|
1202
1372
|
const promptIndex = userPromptNo++
|
|
1203
1373
|
const thisTurnForceFull = forceFullReminder
|
|
@@ -1216,11 +1386,34 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
1216
1386
|
// recognized). So a slash command goes CLEAN; conversational turns keep the reminder.
|
|
1217
1387
|
// Normal turns never start with "/" (composeAgentStdin prepends the preamble), and the
|
|
1218
1388
|
// web already routes "/"-prefixed input as a command (pane.jsx), so this matches intent.
|
|
1219
|
-
|
|
1220
|
-
|
|
1389
|
+
lastTurnBlocks = /^\s*\//.test(t) ? [{ type: 'text', text: t }] : [{ type: 'text', text: t }, { type: 'text', text: lastTurnReminder }]
|
|
1390
|
+
resilienceTurnRevision++
|
|
1391
|
+
turnResilience = createTurnResilience()
|
|
1392
|
+
if (turnResilience) {
|
|
1393
|
+
const submittingTurn = turnResilience
|
|
1394
|
+
const submittingRevision = resilienceTurnRevision
|
|
1395
|
+
const submittingBlocks = lastTurnBlocks
|
|
1396
|
+
// Keep the public sendTurn edge synchronous. The prompt is held locally until
|
|
1397
|
+
// the asynchronous bridge cap admits it; a rejection never reaches the SDK.
|
|
1398
|
+
void admitResilientSubmission().then((admitted) => {
|
|
1399
|
+
// Stop or a newer human turn can land while cap admission is pending.
|
|
1400
|
+
// In either case this exact turn no longer owns the submission edge.
|
|
1401
|
+
if (closed || !turnActive || turnResilience !== submittingTurn || resilienceTurnRevision !== submittingRevision) return
|
|
1402
|
+
if (!admitted) { finishResilientFailure(); return }
|
|
1403
|
+
if (!started) runQuery({ admitted: true })
|
|
1404
|
+
input.push(submittingBlocks)
|
|
1405
|
+
armResilienceDeadline()
|
|
1406
|
+
}).catch(() => {
|
|
1407
|
+
if (!closed && turnActive && turnResilience === submittingTurn && resilienceTurnRevision === submittingRevision) finishResilientFailure()
|
|
1408
|
+
})
|
|
1409
|
+
} else {
|
|
1410
|
+
if (!started) runQuery({ admitted: true })
|
|
1411
|
+
input.push(lastTurnBlocks)
|
|
1412
|
+
}
|
|
1413
|
+
},
|
|
1221
1414
|
// Cold-boot the query WITHOUT sending a turn — the background warmer calls this on
|
|
1222
1415
|
// lazily-restored idle terminals so they're ready before the user clicks them.
|
|
1223
|
-
warm() { if (!started && !closed) runQuery() },
|
|
1416
|
+
warm() { if (!resilienceEnabled && !started && !closed) runQuery() },
|
|
1224
1417
|
get started() { return started },
|
|
1225
1418
|
// Set the permission mode — Claude Code's ⇧⇥ cycle. setPermissionMode is a
|
|
1226
1419
|
// streaming control request (drives plan-mode behaviour SDK-side); the local
|
|
@@ -1274,25 +1467,51 @@ export function startClaudeSession({ cwd, model, effort: initialEffort = 'high',
|
|
|
1274
1467
|
// Graceful interrupt (Esc / Stop) — stops the current turn but keeps the
|
|
1275
1468
|
// session alive for the next one. ac.abort() is teardown only (end()).
|
|
1276
1469
|
async abort() {
|
|
1470
|
+
if (abortPending) return
|
|
1471
|
+
abortPending = true
|
|
1472
|
+
try {
|
|
1277
1473
|
// Swallow the SDK's redundant interrupt teardown (aborted + error_during_execution +
|
|
1278
1474
|
// re-init) — we emit the one canonical boundary below. Set BEFORE interrupt so the
|
|
1279
1475
|
// teardown results, which arrive async on the query loop, are caught; the timer bounds
|
|
1280
1476
|
// the window so a genuine later turn failure still surfaces.
|
|
1281
|
-
|
|
1477
|
+
const retryWasPending = resilienceRetryPending
|
|
1478
|
+
const retryDisposal = retryWasPending ? qDone : null
|
|
1479
|
+
if (turnActive || retryWasPending) {
|
|
1480
|
+
resilienceTurnRevision++
|
|
1481
|
+
// Keep the retry-pending ingress guard armed while Stop awaits the old
|
|
1482
|
+
// process. The revision invalidates the retry itself; the flag prevents
|
|
1483
|
+
// a new human turn from entering its reader-less replacement stream.
|
|
1484
|
+
if (!retryWasPending) resilienceRetryPending = false
|
|
1485
|
+
clearResilienceDeadline()
|
|
1486
|
+
turnResilience?.failure({ aborted: true })
|
|
1487
|
+
if (retryWasPending) {
|
|
1488
|
+
clearTimeout(restartTimer)
|
|
1489
|
+
restartTimer = null
|
|
1490
|
+
started = false
|
|
1491
|
+
}
|
|
1282
1492
|
interrupting = true
|
|
1493
|
+
interruptingRevision = resilienceTurnRevision
|
|
1283
1494
|
clearTimeout(interruptTimer)
|
|
1284
|
-
interruptTimer = setTimeout(() => { interrupting = false }, INTERRUPT_SWALLOW_MS)
|
|
1495
|
+
interruptTimer = setTimeout(() => { interrupting = false; interruptingRevision = null }, INTERRUPT_SWALLOW_MS)
|
|
1285
1496
|
}
|
|
1286
1497
|
try { await q?.interrupt?.() } catch { /* noop */ }
|
|
1498
|
+
// A failed query may already be inside its bounded disposal while its
|
|
1499
|
+
// retry owns no live SDK turn. Keep Stop's barrier open until that process
|
|
1500
|
+
// is actually gone so a later human turn cannot race its session lock.
|
|
1501
|
+
if (retryDisposal) { try { await retryDisposal } catch { /* already disposed */ } }
|
|
1502
|
+
if (retryWasPending) resilienceRetryPending = false
|
|
1287
1503
|
// interrupt() stops the turn but emits NO terminal message. Without one the
|
|
1288
1504
|
// struct stream ends on a non-terminal event, so on resume the SDK treats the
|
|
1289
1505
|
// turn as INCOMPLETE and auto-continues it — the 2026-06-22 "zombie turn that
|
|
1290
1506
|
// resumes itself" storm. Emit a terminal result so the turn is marked DONE +
|
|
1291
1507
|
// persists across refresh. Guarded on turnActive to avoid a double-emit if the
|
|
1292
1508
|
// SDK already surfaced one for the interrupt. (restored from 0.7.49)
|
|
1293
|
-
if (turnActive) { turnActive = false; emit({ kind: 'result', subtype: 'aborted', sessionId }) }
|
|
1509
|
+
if (turnActive || retryWasPending) { turnActive = false; emit({ kind: 'result', subtype: 'aborted', sessionId }) }
|
|
1510
|
+
} finally {
|
|
1511
|
+
abortPending = false
|
|
1512
|
+
}
|
|
1294
1513
|
},
|
|
1295
|
-
end() { closed = true; clearInterval(stallTimer); clearTimeout(interruptTimer); clearTimeout(restartTimer); if (sugTimer) clearTimeout(sugTimer); input.end(); try { ac.abort() } catch { /* noop */ } },
|
|
1514
|
+
end() { closed = true; clearResilienceDeadline(); clearInterval(stallTimer); clearTimeout(interruptTimer); clearTimeout(restartTimer); if (sugTimer) clearTimeout(sugTimer); input.end(); try { ac.abort() } catch { /* noop */ } },
|
|
1296
1515
|
get sessionId() { return sessionId },
|
|
1297
1516
|
get mode() { return mode },
|
|
1298
1517
|
get turnActive() { return turnActive }, // a turn is in flight (gates between-turns update restart — Slice 3 Contract #1)
|
package/event-bounds.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { compactEvidenceEnvelope } from './evidence-
|
|
1
|
+
import { compactEvidenceEnvelope } from './evidence-compact.mjs'
|
|
2
2
|
|
|
3
3
|
/* Browser-safe structured-event bounding shared by the bridge broadcast path and
|
|
4
4
|
the web client's persisted reload snapshot. Large Codex/Claude tool payloads
|
package/evidence-citations.mjs
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
// Strict, runtime-neutral EvidenceEnvelopeV1 parsing. Only an actual
|
|
2
2
|
// project-context evidence tool result can create room evidence in S1.
|
|
3
3
|
import { createHash } from 'node:crypto'
|
|
4
|
+
import { EVIDENCE_CAP, compactEvidenceEnvelope } from './evidence-compact.mjs'
|
|
5
|
+
|
|
6
|
+
export { EVIDENCE_CAP, compactEvidenceEnvelope }
|
|
4
7
|
|
|
5
8
|
export const EVIDENCE_START = '<<<TP_EVIDENCE_V1>>>'
|
|
6
9
|
export const EVIDENCE_END = '<<<END_TP_EVIDENCE_V1>>>'
|
|
7
|
-
export const EVIDENCE_CAP = 8
|
|
8
10
|
const LEVELS = new Set(['exact', 'section', 'source', 'unavailable'])
|
|
9
11
|
const WARNINGS = new Set(['NO_MATCH', 'LOW_LEXICAL_MATCH', 'UNCALIBRATED_RELEVANCE', 'SECTION_ONLY', 'SOURCE_ONLY', 'SOURCE_UNAVAILABLE', 'SPAN_ANCHOR_MISMATCH', 'TRANSCRIPT_BOUNDED'])
|
|
10
12
|
const safeText = (value, max) => typeof value === 'string' && value.length <= max && !/(?:https?:\/\/|file:|bearer\s|token=|signature=|x-amz-|\.\.[\\/]|^\/|[A-Za-z]:[\\/])/i.test(value)
|
|
@@ -46,11 +48,3 @@ export function parseEvidenceEnvelope(toolName, content) {
|
|
|
46
48
|
}
|
|
47
49
|
|
|
48
50
|
export function evidenceForToolResult(toolName, content) { return parseEvidenceEnvelope(toolName, content) || undefined }
|
|
49
|
-
|
|
50
|
-
export function compactEvidenceEnvelope(envelope, maxExcerpt = 600) {
|
|
51
|
-
if (!envelope?.citations?.length) return envelope
|
|
52
|
-
return { ...envelope, citations: envelope.citations.slice(0, EVIDENCE_CAP).map((citation) => {
|
|
53
|
-
if (typeof citation.excerpt !== 'string' || citation.excerpt.length <= maxExcerpt) return citation
|
|
54
|
-
return { ...citation, excerpt: undefined, excerptSha256: undefined, locator: { ...citation.locator, level: 'unavailable' }, assessment: { ...citation.assessment, confidence: 'unavailable', warnings: [...new Set([...citation.assessment.warnings, 'TRANSCRIPT_BOUNDED', 'SOURCE_UNAVAILABLE'])].slice(0, 8) } }
|
|
55
|
-
}) }
|
|
56
|
-
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Browser-safe evidence bounding shared by the bridge and replay client.
|
|
2
|
+
// Parsing and digest verification stay in evidence-citations.mjs (Node-only).
|
|
3
|
+
export const EVIDENCE_CAP = 8
|
|
4
|
+
|
|
5
|
+
export function compactEvidenceEnvelope(envelope, maxExcerpt = 600) {
|
|
6
|
+
if (!envelope?.citations?.length) return envelope
|
|
7
|
+
return { ...envelope, citations: envelope.citations.slice(0, EVIDENCE_CAP).map((citation) => {
|
|
8
|
+
if (typeof citation.excerpt !== 'string' || citation.excerpt.length <= maxExcerpt) return citation
|
|
9
|
+
return { ...citation, excerpt: undefined, excerptSha256: undefined, locator: { ...citation.locator, level: 'unavailable' }, assessment: { ...citation.assessment, confidence: 'unavailable', warnings: [...new Set([...citation.assessment.warnings, 'TRANSCRIPT_BOUNDED', 'SOURCE_UNAVAILABLE'])].slice(0, 8) } }
|
|
10
|
+
}) }
|
|
11
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "thinkpool-pair",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.357",
|
|
4
4
|
"description": "Connect Claude Code, Codex, or Hermes on your computer to a Thinkpool Code room.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -59,6 +59,7 @@
|
|
|
59
59
|
"agent-notify.mjs",
|
|
60
60
|
"agent-detect.mjs",
|
|
61
61
|
"code-event-contract.mjs",
|
|
62
|
+
"evidence-compact.mjs",
|
|
62
63
|
"evidence-citations.mjs",
|
|
63
64
|
"error-recovery.mjs",
|
|
64
65
|
"pair-control-authority.mjs",
|
package/provider-resilience.mjs
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
/*
|
|
2
2
|
* Provider resilience policy primitives.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* attempt ledger, but enabling this module cannot add a retry or fallback.
|
|
4
|
+
* This module never makes a network call or reads provider credentials. Phase 1
|
|
5
|
+
* adds an injected same-target controller, but transport remains owned by the
|
|
6
|
+
* Claude session and fallback remains unavailable.
|
|
8
7
|
*/
|
|
9
8
|
import { isSecretFreeRoomPayload } from './runtime-contract.mjs'
|
|
10
9
|
|
|
11
10
|
export const RESILIENCE_VERSION = 1
|
|
12
11
|
export const MAX_NETWORK_ATTEMPTS = 3
|
|
12
|
+
export const MAX_SAME_TARGET_ATTEMPTS = 2
|
|
13
13
|
export const TRANSIENT_FAILURES = new Set(['timeout', 'network', 'rate_limit', 'overload', 'upstream_5xx'])
|
|
14
14
|
const RETRYABLE = [...TRANSIENT_FAILURES]
|
|
15
15
|
const TRACE_FIELDS = new Set(['traceId', 'turnRev', 'attempt', 'targetProviderId', 'targetProviderName', 'requestedModel', 'actualConfiguredModel', 'phase', 'outcome', 'failureClass', 'elapsedMs', 'retryAfterMs', 'fallbackFrom'])
|
|
@@ -183,6 +183,147 @@ export function createMemoryCircuit({ now = () => Date.now() } = {}) {
|
|
|
183
183
|
}
|
|
184
184
|
}
|
|
185
185
|
|
|
186
|
+
/**
|
|
187
|
+
* Apply the only authoritative provider-work cap currently available at the
|
|
188
|
+
* bridge boundary. Ordinary/uncapped lanes remain admissible, but the result
|
|
189
|
+
* names that absence explicitly so the host can record `cap_not_configured`
|
|
190
|
+
* rather than pretending an approval occurred.
|
|
191
|
+
*/
|
|
192
|
+
export function providerResilienceCapAdmission(budget) {
|
|
193
|
+
if (!budget || budget.capTokens == null) {
|
|
194
|
+
return { allowed: true, configured: false, reason: 'cap_not_configured' }
|
|
195
|
+
}
|
|
196
|
+
const capTokens = Number(budget.capTokens)
|
|
197
|
+
const spentTokens = Number(budget.spentTokens)
|
|
198
|
+
const allowed = Number.isFinite(capTokens)
|
|
199
|
+
&& capTokens >= 0
|
|
200
|
+
&& Number.isFinite(spentTokens)
|
|
201
|
+
&& spentTokens >= 0
|
|
202
|
+
&& budget.killed !== true
|
|
203
|
+
&& spentTokens < capTokens
|
|
204
|
+
return { allowed, configured: true, reason: allowed ? 'within_cap' : 'cap_blocked' }
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Pure, injected Phase-1 controller for a single exact custom Claude target.
|
|
209
|
+
*
|
|
210
|
+
* The bridge/session integration owns transport and invokes this controller at
|
|
211
|
+
* turn start, before each submission, after visible output, and at terminal
|
|
212
|
+
* success/failure. It intentionally has no fallback path and cannot resolve a
|
|
213
|
+
* built-in Anthropic target. A caller must inject both the authoritative cap
|
|
214
|
+
* admission gate and the bridge-scoped memory circuit; missing or broken gates
|
|
215
|
+
* deny the submission rather than guessing that it is safe.
|
|
216
|
+
*/
|
|
217
|
+
export function createSameTargetResilienceController({
|
|
218
|
+
runtime = 'claude', providers = [], providerId, model, requestedModel = model,
|
|
219
|
+
policy, bridgeHostId = null, circuit, capGate, traceId = 'local', turnRev = null,
|
|
220
|
+
onRecord = null,
|
|
221
|
+
} = {}) {
|
|
222
|
+
let checked
|
|
223
|
+
try { checked = validateResiliencePolicy(policy, { providers, primaryProviderId: providerId }) } catch { checked = { ok: false } }
|
|
224
|
+
let target = null
|
|
225
|
+
try { target = resolveAttemptTarget({ runtime, providers, providerId, model }) } catch { /* malformed registry is not retryable */ }
|
|
226
|
+
const configured = checked?.ok && checked.value.enabled && target && runtime === 'claude'
|
|
227
|
+
const circuitReady = !!circuit && typeof circuit.preflight === 'function' && typeof circuit.failure === 'function' && typeof circuit.success === 'function'
|
|
228
|
+
const ceiling = configured ? Math.min(checked.value.maxAttempts, MAX_SAME_TARGET_ATTEMPTS) : 0
|
|
229
|
+
const targetKey = target && { bridgeHostId, providerId: target.providerId, model: target.actualConfiguredModel }
|
|
230
|
+
const base = target && {
|
|
231
|
+
traceId,
|
|
232
|
+
turnRev,
|
|
233
|
+
targetProviderId: target.providerId,
|
|
234
|
+
targetProviderName: target.targetProviderName,
|
|
235
|
+
requestedModel,
|
|
236
|
+
actualConfiguredModel: target.actualConfiguredModel,
|
|
237
|
+
phase: 'same_target',
|
|
238
|
+
}
|
|
239
|
+
let started = false
|
|
240
|
+
let submissions = 0
|
|
241
|
+
let awaitingOutcome = false
|
|
242
|
+
let retryEligible = false
|
|
243
|
+
let outputVisible = false
|
|
244
|
+
let toolOutputVisible = false
|
|
245
|
+
let aborted = false
|
|
246
|
+
let permissionDenied = false
|
|
247
|
+
let finished = false
|
|
248
|
+
|
|
249
|
+
const makeRecord = (outcome, extra = {}, attempt = submissions) => safeTraceRecord({
|
|
250
|
+
...base,
|
|
251
|
+
attempt: Math.max(1, attempt),
|
|
252
|
+
outcome,
|
|
253
|
+
...extra,
|
|
254
|
+
})
|
|
255
|
+
const emit = (outcome, extra) => {
|
|
256
|
+
const record = makeRecord(outcome, extra)
|
|
257
|
+
try { onRecord?.(record) } catch { /* trace observers are never control flow */ }
|
|
258
|
+
return record
|
|
259
|
+
}
|
|
260
|
+
const blocked = (outcome = 'failed', extra) => ({ admitted: false, record: emit(outcome, extra) })
|
|
261
|
+
const replaySafe = () => isReplaySafe({ outputVisible, toolOutputVisible, aborted, permissionDenied })
|
|
262
|
+
|
|
263
|
+
return {
|
|
264
|
+
/** Mark the controller as belonging to this turn; no provider work occurs here. */
|
|
265
|
+
start() { started = true },
|
|
266
|
+
|
|
267
|
+
/** Once any assistant/tool output is visible, automatic replay is permanently disabled. */
|
|
268
|
+
visibleOutput({ tool = false } = {}) {
|
|
269
|
+
outputVisible = true
|
|
270
|
+
if (tool) toolOutputVisible = true
|
|
271
|
+
retryEligible = false
|
|
272
|
+
},
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Admit exactly one upcoming submission. The cap gate is awaited before
|
|
276
|
+
* transport, and any false/unknown/throwing result denies the attempt.
|
|
277
|
+
*/
|
|
278
|
+
async preflight() {
|
|
279
|
+
if (!started) started = true
|
|
280
|
+
if (!configured || !circuitReady || finished || awaitingOutcome || submissions >= ceiling) return blocked()
|
|
281
|
+
if (submissions > 0 && (!retryEligible || !replaySafe())) return blocked()
|
|
282
|
+
const pending = makeRecord(submissions === 0 ? 'started' : 'retrying', {}, submissions + 1)
|
|
283
|
+
let admitted = false
|
|
284
|
+
try {
|
|
285
|
+
const result = await capGate?.(pending)
|
|
286
|
+
admitted = result === true || result?.allowed === true
|
|
287
|
+
} catch { admitted = false }
|
|
288
|
+
if (!admitted) return blocked('cap_blocked')
|
|
289
|
+
let circuitState
|
|
290
|
+
try { circuitState = circuit.preflight(targetKey) } catch { return blocked() }
|
|
291
|
+
if (!circuitState?.ok) return blocked('circuit_open')
|
|
292
|
+
submissions += 1
|
|
293
|
+
awaitingOutcome = true
|
|
294
|
+
retryEligible = false
|
|
295
|
+
return { admitted: true, record: emit(submissions === 1 ? 'started' : 'retrying') }
|
|
296
|
+
},
|
|
297
|
+
|
|
298
|
+
/** Classify a raw transport failure locally and expose only a coarse trace. */
|
|
299
|
+
failure(input = {}) {
|
|
300
|
+
if (!awaitingOutcome || finished) return null
|
|
301
|
+
awaitingOutcome = false
|
|
302
|
+
const failureClass = classifyProviderFailure(input)
|
|
303
|
+
if (failureClass === 'user_abort') aborted = true
|
|
304
|
+
if (failureClass === 'permission_denied') permissionDenied = true
|
|
305
|
+
let circuitUsable = true
|
|
306
|
+
if (TRANSIENT_FAILURES.has(failureClass)) {
|
|
307
|
+
try {
|
|
308
|
+
const state = circuit.failure(targetKey, checked.value, failureClass)
|
|
309
|
+
if (state?.state === 'open') circuitUsable = false
|
|
310
|
+
} catch { circuitUsable = false }
|
|
311
|
+
}
|
|
312
|
+
retryEligible = circuitUsable && checked.value.retryOn.includes(failureClass) && replaySafe() && submissions < ceiling
|
|
313
|
+
return emit('failed', { failureClass })
|
|
314
|
+
},
|
|
315
|
+
|
|
316
|
+
/** Close/reset the injected circuit after a completed provider response. */
|
|
317
|
+
success() {
|
|
318
|
+
if (!awaitingOutcome || finished) return null
|
|
319
|
+
awaitingOutcome = false
|
|
320
|
+
finished = true
|
|
321
|
+
try { circuit.success(targetKey) } catch { /* a completed response is never replayed */ }
|
|
322
|
+
return emit('succeeded')
|
|
323
|
+
},
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
186
327
|
export function safeTraceRecord(input = {}) {
|
|
187
328
|
const out = {}
|
|
188
329
|
for (const key of TRACE_FIELDS) {
|
|
@@ -190,6 +331,8 @@ export function safeTraceRecord(input = {}) {
|
|
|
190
331
|
if (TRACE_TEXT_FIELDS.has(key)) {
|
|
191
332
|
const value = safeTraceText(input[key])
|
|
192
333
|
if (value) out[key] = value
|
|
334
|
+
} else if (key === 'turnRev') {
|
|
335
|
+
if (Number.isSafeInteger(input[key]) && input[key] >= 0) out[key] = input[key]
|
|
193
336
|
} else out[key] = input[key]
|
|
194
337
|
}
|
|
195
338
|
out.attempt = Math.max(1, Math.min(Number(out.attempt) || 1, MAX_NETWORK_ATTEMPTS))
|
|
@@ -204,7 +347,7 @@ export function formatResilienceTrace(input) {
|
|
|
204
347
|
const r = safeTraceRecord(input)
|
|
205
348
|
const target = [r.targetProviderName || r.targetProviderId, r.actualConfiguredModel].filter(Boolean).join(' / ') || 'configured provider'
|
|
206
349
|
const failure = r.failureClass ? ` (${r.failureClass.replaceAll('_', ' ')})` : ''
|
|
207
|
-
if (r.outcome === 'retrying') return `${target} unavailable${failure} —
|
|
350
|
+
if (r.outcome === 'retrying') return `${target} unavailable${failure} — retrying once before output (attempt ${r.attempt} of ${MAX_SAME_TARGET_ATTEMPTS}).`
|
|
208
351
|
if (r.outcome === 'circuit_open') return `${target} is temporarily unavailable; its local circuit is open.`
|
|
209
352
|
if (r.outcome === 'cap_blocked') return `Additional provider work was blocked by the applicable cap.`
|
|
210
353
|
if (r.outcome === 'succeeded') return `${target} completed.`
|
package/providers.mjs
CHANGED
|
@@ -43,6 +43,12 @@ import fs from 'node:fs'
|
|
|
43
43
|
import path from 'node:path'
|
|
44
44
|
import crypto from 'node:crypto'
|
|
45
45
|
import { claudeish } from './event-id.mjs'
|
|
46
|
+
import {
|
|
47
|
+
DEFAULT_RESILIENCE_POLICY,
|
|
48
|
+
resilienceProjection,
|
|
49
|
+
resolveAttemptTarget,
|
|
50
|
+
validateResiliencePolicy,
|
|
51
|
+
} from './provider-resilience.mjs'
|
|
46
52
|
|
|
47
53
|
const DIR = path.join(os.homedir(), '.thinkpool-pair')
|
|
48
54
|
const REG_FILE = path.join(DIR, 'providers.json')
|
|
@@ -54,7 +60,7 @@ export const BUILTIN_ID = 'anthropic'
|
|
|
54
60
|
function ensureDir() { try { fs.mkdirSync(DIR, { recursive: true, mode: 0o700 }) } catch { /* noop */ } }
|
|
55
61
|
|
|
56
62
|
// ── registry load / save ───────────────────────────────────────────────
|
|
57
|
-
/** @returns {Array<{id:string,name:string,baseUrl:string,model?:string,key:string,addedAt:number}>} */
|
|
63
|
+
/** @returns {Array<{id:string,name:string,baseUrl:string,model?:string,key:string,addedAt:number,resilience?:object}>} */
|
|
58
64
|
export function loadProviders() {
|
|
59
65
|
try {
|
|
60
66
|
const arr = JSON.parse(fs.readFileSync(REG_FILE, 'utf8'))
|
|
@@ -244,6 +250,9 @@ export function addProviderModel({ id, model, name } = {}) {
|
|
|
244
250
|
baseUrl: src.baseUrl, // copied host-side
|
|
245
251
|
model: nextModel,
|
|
246
252
|
key: src.key, // copied host-side — never re-sent by the client
|
|
253
|
+
// A cloned credential is not consent to retry it. Each row needs its own
|
|
254
|
+
// explicit Phase-1 opt-in, even when both rows share the same endpoint/key.
|
|
255
|
+
resilience: { ...DEFAULT_RESILIENCE_POLICY, retryOn: [...DEFAULT_RESILIENCE_POLICY.retryOn], circuit: { ...DEFAULT_RESILIENCE_POLICY.circuit }, fallback: [] },
|
|
247
256
|
addedAt: Date.now(),
|
|
248
257
|
})
|
|
249
258
|
saveProviders(arr)
|
|
@@ -322,6 +331,55 @@ export function providerModel(id) {
|
|
|
322
331
|
return loadProviders().find((x) => x.id === id)?.model || null
|
|
323
332
|
}
|
|
324
333
|
|
|
334
|
+
/**
|
|
335
|
+
* Persist one custom provider row's Phase-1 same-target policy. This is a
|
|
336
|
+
* host-local primitive: account RPC/UI wiring deliberately lives elsewhere.
|
|
337
|
+
*
|
|
338
|
+
* `policy` may be a complete v1 policy, or a boolean for the conservative
|
|
339
|
+
* Phase-1 preset. Fallback declarations are rejected here: alternate targets
|
|
340
|
+
* require the later explicit-consent flow, so this API can never turn one on.
|
|
341
|
+
*/
|
|
342
|
+
export function setProviderResiliencePolicy(id, policy) {
|
|
343
|
+
if (!id || id === BUILTIN_ID) return { ok: false, error: 'the built-in Anthropic provider cannot use resilience policy' }
|
|
344
|
+
const arr = loadProviders()
|
|
345
|
+
const provider = arr.find((p) => p.id === id)
|
|
346
|
+
if (!provider) return { ok: false, error: 'no such provider' }
|
|
347
|
+
|
|
348
|
+
const input = typeof policy === 'boolean'
|
|
349
|
+
? { ...DEFAULT_RESILIENCE_POLICY, enabled: policy, retryOn: [...DEFAULT_RESILIENCE_POLICY.retryOn], circuit: { ...DEFAULT_RESILIENCE_POLICY.circuit }, fallback: [] }
|
|
350
|
+
: policy
|
|
351
|
+
const checked = validateResiliencePolicy(input, { providers: arr, primaryProviderId: id })
|
|
352
|
+
if (!checked.ok) return { ok: false, error: checked.error }
|
|
353
|
+
if (checked.value.fallback.length) return { ok: false, error: 'fallback is not available in Phase 1' }
|
|
354
|
+
// An enabled controller must have a configured exact model for its safe
|
|
355
|
+
// target record. A disabled policy remains valid for legacy/no-model rows.
|
|
356
|
+
if (checked.value.enabled && !resolveAttemptTarget({ providers: arr, providerId: id, model: provider.model })) {
|
|
357
|
+
return { ok: false, error: 'an enabled resilience policy requires an exact configured model' }
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const next = arr.map((p) => p.id === id ? { ...p, resilience: checked.value } : p)
|
|
361
|
+
saveProviders(next)
|
|
362
|
+
return { ok: true, resilience: resilienceProjection(checked.value) }
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* Resolve a custom provider's persisted policy for bridge wiring without ever
|
|
367
|
+
* returning host credentials or endpoint data. Missing policy is safely
|
|
368
|
+
* disabled; malformed stored policy, built-in, unknown, and model-less targets
|
|
369
|
+
* fail closed with null.
|
|
370
|
+
*/
|
|
371
|
+
export function resolveProviderResiliencePolicy(id) {
|
|
372
|
+
if (!id || id === BUILTIN_ID) return null
|
|
373
|
+
const providers = loadProviders()
|
|
374
|
+
const provider = providers.find((p) => p.id === id)
|
|
375
|
+
if (!provider) return null
|
|
376
|
+
const checked = validateResiliencePolicy(provider.resilience, { providers, primaryProviderId: id })
|
|
377
|
+
if (!checked.ok) return null
|
|
378
|
+
const target = resolveAttemptTarget({ providers, providerId: id, model: provider.model })
|
|
379
|
+
if (!target) return null
|
|
380
|
+
return { policy: resilienceProjection(checked.value), target }
|
|
381
|
+
}
|
|
382
|
+
|
|
325
383
|
// ── read-only projections (never expose the raw key) ────────────────────
|
|
326
384
|
/** Masked list for the dashboard: keyHint = last 4 chars only. Built-in first. */
|
|
327
385
|
export function listProviders() {
|
|
@@ -331,8 +389,9 @@ export function listProviders() {
|
|
|
331
389
|
model: p.model || null,
|
|
332
390
|
group: providerGroup(p),
|
|
333
391
|
keyHint: keyHint(p.key),
|
|
392
|
+
resilience: resilienceProjection(p.resilience),
|
|
334
393
|
}))
|
|
335
|
-
return [{ id: BUILTIN_ID, name: 'Anthropic (Claude)', model: null, group: BUILTIN_ID, keyHint: null }, ...custom]
|
|
394
|
+
return [{ id: BUILTIN_ID, name: 'Anthropic (Claude)', model: null, group: BUILTIN_ID, keyHint: null, resilience: null }, ...custom]
|
|
336
395
|
}
|
|
337
396
|
|
|
338
397
|
/** Name-only projection for the announce/presence payload — NO key, NO baseUrl. */
|