pi-maestro-teammate 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/extension/index.ts +51 -5
- package/src/models/model-circuit-breaker.ts +22 -5
- package/src/runs/execution.ts +157 -42
- package/src/runs/retry.ts +43 -43
- package/src/tui/render.ts +6 -20
- package/types/extension/index.d.ts +15 -0
- package/types/runs/execution.d.ts +6 -0
- package/types/runs/retry.d.ts +2 -2
package/package.json
CHANGED
package/src/extension/index.ts
CHANGED
|
@@ -886,6 +886,14 @@ function toolAction(name: string): string {
|
|
|
886
886
|
return `using ${name}`;
|
|
887
887
|
}
|
|
888
888
|
|
|
889
|
+
function formatRetryDelay(delayMs: number): string {
|
|
890
|
+
const seconds = Math.max(0, Math.ceil(delayMs / 1_000));
|
|
891
|
+
if (seconds < 60) return `${seconds}s`;
|
|
892
|
+
const minutes = Math.floor(seconds / 60);
|
|
893
|
+
const remainder = seconds % 60;
|
|
894
|
+
return remainder === 0 ? `${minutes}m` : `${minutes}m ${remainder}s`;
|
|
895
|
+
}
|
|
896
|
+
|
|
889
897
|
function agentWidgetRows(agents: ActiveAgent[]): AgentWidgetRow[] {
|
|
890
898
|
const rows = new Map<string, AgentWidgetRow>();
|
|
891
899
|
const directAgents = new Map(agents.map((agent) => [agent.correlationId, agent]));
|
|
@@ -913,7 +921,11 @@ function agentWidgetRows(agents: ActiveAgent[]): AgentWidgetRow[] {
|
|
|
913
921
|
: status === "sleeping"
|
|
914
922
|
? "sleeping"
|
|
915
923
|
: status === "retrying"
|
|
916
|
-
?
|
|
924
|
+
? direct?.retry
|
|
925
|
+
? `retry ${direct.retry.attempt}/${direct.retry.maxRetries} in ${formatRetryDelay(
|
|
926
|
+
direct.retry.nextRetryAt - Date.now(),
|
|
927
|
+
)}`
|
|
928
|
+
: "retrying"
|
|
917
929
|
: status === "pending"
|
|
918
930
|
? "waiting for dependencies"
|
|
919
931
|
: status === "failed"
|
|
@@ -1251,6 +1263,32 @@ export function rejectAllChildProxyRequests(
|
|
|
1251
1263
|
}
|
|
1252
1264
|
|
|
1253
1265
|
/** @internal Exported for lifecycle regression tests. */
|
|
1266
|
+
export type IpcSender = (
|
|
1267
|
+
message: Record<string, unknown>,
|
|
1268
|
+
callback: (error: Error | null) => void,
|
|
1269
|
+
) => boolean;
|
|
1270
|
+
|
|
1271
|
+
/**
|
|
1272
|
+
* Builds the IPC sender the teammate proxy uses to talk to its parent.
|
|
1273
|
+
*
|
|
1274
|
+
* Node's IPC `send` reads `this.connected` internally, so detaching it from its
|
|
1275
|
+
* owner (`const send = proc.send`) leaves `this` undefined in module scope and
|
|
1276
|
+
* throws "Cannot read properties of undefined (reading 'connected')" on the
|
|
1277
|
+
* first call — which broke every proxied teammate tool in a nested child.
|
|
1278
|
+
* Binding the owner keeps the proxied call working. Returns undefined when no
|
|
1279
|
+
* live IPC channel exists.
|
|
1280
|
+
*/
|
|
1281
|
+
export function createIpcSender(
|
|
1282
|
+
// Node's IPC process.send is a loosely-typed boundary (message: any); any[]
|
|
1283
|
+
// is the signature both process.send and test fakes satisfy without casts.
|
|
1284
|
+
proc: { connected?: boolean; send?: (...args: any[]) => boolean } = process,
|
|
1285
|
+
): IpcSender | undefined {
|
|
1286
|
+
const rawSend = proc.send;
|
|
1287
|
+
if (typeof rawSend !== "function" || proc.connected === false) return undefined;
|
|
1288
|
+
const send = rawSend.bind(proc);
|
|
1289
|
+
return (message, callback) => send(message, callback);
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1254
1292
|
export function createChildProxyRequest(
|
|
1255
1293
|
pendingRequests: ChildProxyPendingRequests,
|
|
1256
1294
|
requestId: string,
|
|
@@ -1562,8 +1600,8 @@ export default function registerTeammateExtension(
|
|
|
1562
1600
|
params: unknown,
|
|
1563
1601
|
signal?: AbortSignal,
|
|
1564
1602
|
): Promise<TeammateToolResult<T>> {
|
|
1565
|
-
const send =
|
|
1566
|
-
if (
|
|
1603
|
+
const send = createIpcSender();
|
|
1604
|
+
if (!send) {
|
|
1567
1605
|
throw new Error("IPC not available. Teammate proxy requires IPC channel.");
|
|
1568
1606
|
}
|
|
1569
1607
|
const requestId = randomUUID();
|
|
@@ -1577,7 +1615,7 @@ export default function registerTeammateExtension(
|
|
|
1577
1615
|
params,
|
|
1578
1616
|
correlationId: process.env.PI_TEAMMATE_CORRELATION_ID,
|
|
1579
1617
|
},
|
|
1580
|
-
|
|
1618
|
+
send,
|
|
1581
1619
|
CHILD_PROXY_TIMEOUT_MS,
|
|
1582
1620
|
signal,
|
|
1583
1621
|
);
|
|
@@ -5378,6 +5416,7 @@ export async function handleProxyRequest(
|
|
|
5378
5416
|
const reportChildStatus = (
|
|
5379
5417
|
status: ChildAgentCallSnapshot["status"],
|
|
5380
5418
|
progress?: AgentProgress,
|
|
5419
|
+
retryMessage?: string,
|
|
5381
5420
|
): void => {
|
|
5382
5421
|
onChildStatus?.({
|
|
5383
5422
|
agent: activeAgent.agent,
|
|
@@ -5396,6 +5435,9 @@ export async function handleProxyRequest(
|
|
|
5396
5435
|
outputTokens: progress.outputTokens,
|
|
5397
5436
|
...(progress.lastMessage ? { lastMessage: truncateUtf8Tail(progress.lastMessage, AGENT_BUFFER_LIMITS.lastResultBytes) } : {}),
|
|
5398
5437
|
} : {}),
|
|
5438
|
+
...(!progress?.lastMessage && retryMessage
|
|
5439
|
+
? { lastMessage: truncateUtf8Tail(retryMessage, AGENT_BUFFER_LIMITS.lastResultBytes) }
|
|
5440
|
+
: {}),
|
|
5399
5441
|
});
|
|
5400
5442
|
};
|
|
5401
5443
|
|
|
@@ -5583,7 +5625,11 @@ export async function handleProxyRequest(
|
|
|
5583
5625
|
}),
|
|
5584
5626
|
onRetry: (retry) => {
|
|
5585
5627
|
applyAgentRetryState(state, retry);
|
|
5586
|
-
reportChildStatus(
|
|
5628
|
+
reportChildStatus(
|
|
5629
|
+
"retrying",
|
|
5630
|
+
undefined,
|
|
5631
|
+
`retry ${retry.attempt}/${retry.maxRetries} in ${formatRetryDelay(retry.delayMs)}: ${retry.error}`,
|
|
5632
|
+
);
|
|
5587
5633
|
},
|
|
5588
5634
|
onTurnComplete: (result) => {
|
|
5589
5635
|
const lastMessage = displayMessageForResult(result);
|
|
@@ -40,6 +40,8 @@ interface MutableModelCircuit {
|
|
|
40
40
|
generation: number;
|
|
41
41
|
openedAt?: number;
|
|
42
42
|
halfOpenTrialInProgress: boolean;
|
|
43
|
+
/** Timestamp when the circuit entered HALF_OPEN; used as a watchdog deadline. */
|
|
44
|
+
halfOpenEnteredAt?: number;
|
|
43
45
|
}
|
|
44
46
|
|
|
45
47
|
export class ModelCircuitBreaker {
|
|
@@ -74,11 +76,23 @@ export class ModelCircuitBreaker {
|
|
|
74
76
|
}
|
|
75
77
|
|
|
76
78
|
if (circuit.state === "HALF_OPEN") {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
79
|
+
// Watchdog: if the HALF_OPEN trial has not settled within cooldownMs,
|
|
80
|
+
// the trial holder likely crashed or leaked. Re-open so the normal
|
|
81
|
+
// OPEN → cooldown → HALF_OPEN recovery cycle can proceed.
|
|
82
|
+
// Skipped when cooldownMs is 0 because the watchdog would fire
|
|
83
|
+
// immediately and break the single-trial invariant.
|
|
84
|
+
if (this.cooldownMs > 0
|
|
85
|
+
&& circuit.halfOpenEnteredAt !== undefined
|
|
86
|
+
&& this.now() >= circuit.halfOpenEnteredAt + this.cooldownMs) {
|
|
87
|
+
this.open(circuit);
|
|
88
|
+
// Fall through to the OPEN branch below (retryAt check).
|
|
89
|
+
} else {
|
|
90
|
+
return {
|
|
91
|
+
allowed: false,
|
|
92
|
+
model,
|
|
93
|
+
state: "HALF_OPEN",
|
|
94
|
+
};
|
|
95
|
+
}
|
|
82
96
|
}
|
|
83
97
|
|
|
84
98
|
const retryAt = this.retryAt(circuit);
|
|
@@ -93,6 +107,7 @@ export class ModelCircuitBreaker {
|
|
|
93
107
|
|
|
94
108
|
circuit.state = "HALF_OPEN";
|
|
95
109
|
circuit.halfOpenTrialInProgress = true;
|
|
110
|
+
circuit.halfOpenEnteredAt = this.now();
|
|
96
111
|
return {
|
|
97
112
|
allowed: true,
|
|
98
113
|
model,
|
|
@@ -111,6 +126,7 @@ export class ModelCircuitBreaker {
|
|
|
111
126
|
circuit.consecutiveFailures = 0;
|
|
112
127
|
circuit.openedAt = undefined;
|
|
113
128
|
circuit.halfOpenTrialInProgress = false;
|
|
129
|
+
circuit.halfOpenEnteredAt = undefined;
|
|
114
130
|
if (acquisition.state === "HALF_OPEN") circuit.generation += 1;
|
|
115
131
|
}
|
|
116
132
|
|
|
@@ -171,6 +187,7 @@ export class ModelCircuitBreaker {
|
|
|
171
187
|
circuit.state = "OPEN";
|
|
172
188
|
circuit.openedAt = this.now();
|
|
173
189
|
circuit.halfOpenTrialInProgress = false;
|
|
190
|
+
circuit.halfOpenEnteredAt = undefined;
|
|
174
191
|
circuit.generation += 1;
|
|
175
192
|
}
|
|
176
193
|
|
package/src/runs/execution.ts
CHANGED
|
@@ -952,7 +952,9 @@ function isFallbackModelError(messages: Array<{ role: string; content: string }>
|
|
|
952
952
|
}
|
|
953
953
|
|
|
954
954
|
function resultFailureMessage(messages: Array<{ role: string; content: string }>): string {
|
|
955
|
-
|
|
955
|
+
const newestFirst = [...messages].reverse();
|
|
956
|
+
return newestFirst.find((message) => isRetryableProviderError(message.content))?.content
|
|
957
|
+
?? newestFirst.find((message) => message.role === "system")?.content
|
|
956
958
|
?? messages.at(-1)?.content
|
|
957
959
|
?? "Unknown teammate failure";
|
|
958
960
|
}
|
|
@@ -972,6 +974,102 @@ function waitForRetryDelay(delayMs: number, signal?: AbortSignal): Promise<boole
|
|
|
972
974
|
});
|
|
973
975
|
}
|
|
974
976
|
|
|
977
|
+
interface RetrySettingSnapshot {
|
|
978
|
+
hadRetry: boolean;
|
|
979
|
+
hadEnabled: boolean;
|
|
980
|
+
enabled?: boolean;
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
interface RetryPersistenceGuard {
|
|
984
|
+
depth: number;
|
|
985
|
+
snapshot?: RetrySettingSnapshot;
|
|
986
|
+
restoreTimer?: ReturnType<typeof setTimeout>;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
const retryPersistenceGuards = new Map<string, RetryPersistenceGuard>();
|
|
990
|
+
|
|
991
|
+
/**
|
|
992
|
+
* Pi's RPC set_auto_retry command persists to settings.json even though the
|
|
993
|
+
* child only needs a session-local override. Restore the original value after
|
|
994
|
+
* every concurrently starting child has acknowledged that command.
|
|
995
|
+
*/
|
|
996
|
+
export function acquireRetryPersistenceGuard(settingsPath: string): () => void {
|
|
997
|
+
const existing = retryPersistenceGuards.get(settingsPath);
|
|
998
|
+
if (!existing) {
|
|
999
|
+
const snapshot = readRetrySettingSnapshot(settingsPath);
|
|
1000
|
+
if (snapshot?.hadEnabled && snapshot.enabled === false) return () => {};
|
|
1001
|
+
}
|
|
1002
|
+
if (existing?.restoreTimer) {
|
|
1003
|
+
clearTimeout(existing.restoreTimer);
|
|
1004
|
+
existing.restoreTimer = undefined;
|
|
1005
|
+
}
|
|
1006
|
+
const guard = existing ?? {
|
|
1007
|
+
depth: 0,
|
|
1008
|
+
snapshot: readRetrySettingSnapshot(settingsPath),
|
|
1009
|
+
};
|
|
1010
|
+
guard.depth += 1;
|
|
1011
|
+
retryPersistenceGuards.set(settingsPath, guard);
|
|
1012
|
+
let released = false;
|
|
1013
|
+
return () => {
|
|
1014
|
+
if (released) return;
|
|
1015
|
+
released = true;
|
|
1016
|
+
guard.depth -= 1;
|
|
1017
|
+
if (guard.depth > 0) return;
|
|
1018
|
+
guard.restoreTimer = setTimeout(() => {
|
|
1019
|
+
guard.restoreTimer = undefined;
|
|
1020
|
+
if (guard.depth > 0) return;
|
|
1021
|
+
retryPersistenceGuards.delete(settingsPath);
|
|
1022
|
+
if (guard.snapshot) restoreRetrySettingSnapshot(settingsPath, guard.snapshot);
|
|
1023
|
+
}, 250);
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
function readRetrySettingSnapshot(settingsPath: string): RetrySettingSnapshot | undefined {
|
|
1028
|
+
try {
|
|
1029
|
+
const root = JSON.parse(fs.readFileSync(settingsPath, "utf8")) as unknown;
|
|
1030
|
+
if (!root || typeof root !== "object" || Array.isArray(root)) return undefined;
|
|
1031
|
+
const retry = (root as Record<string, unknown>).retry;
|
|
1032
|
+
const hadRetry = Boolean(retry && typeof retry === "object" && !Array.isArray(retry));
|
|
1033
|
+
const record = hadRetry ? retry as Record<string, unknown> : undefined;
|
|
1034
|
+
return {
|
|
1035
|
+
hadRetry,
|
|
1036
|
+
hadEnabled: typeof record?.enabled === "boolean",
|
|
1037
|
+
...(typeof record?.enabled === "boolean" ? { enabled: record.enabled } : {}),
|
|
1038
|
+
};
|
|
1039
|
+
} catch {
|
|
1040
|
+
return undefined;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
function restoreRetrySettingSnapshot(settingsPath: string, snapshot: RetrySettingSnapshot): void {
|
|
1045
|
+
try {
|
|
1046
|
+
const root = JSON.parse(fs.readFileSync(settingsPath, "utf8")) as Record<string, unknown>;
|
|
1047
|
+
if (!root || typeof root !== "object" || Array.isArray(root)) return;
|
|
1048
|
+
const retry = root.retry && typeof root.retry === "object" && !Array.isArray(root.retry)
|
|
1049
|
+
? { ...root.retry as Record<string, unknown> }
|
|
1050
|
+
: {};
|
|
1051
|
+
const currentHadEnabled = typeof retry.enabled === "boolean";
|
|
1052
|
+
if (
|
|
1053
|
+
currentHadEnabled === snapshot.hadEnabled
|
|
1054
|
+
&& (!snapshot.hadEnabled || retry.enabled === snapshot.enabled)
|
|
1055
|
+
) return;
|
|
1056
|
+
if (snapshot.hadEnabled) retry.enabled = snapshot.enabled;
|
|
1057
|
+
else delete retry.enabled;
|
|
1058
|
+
if (!snapshot.hadRetry && Object.keys(retry).length === 0) delete root.retry;
|
|
1059
|
+
else root.retry = retry;
|
|
1060
|
+
const temporary = `${settingsPath}.${process.pid}.${randomUUID()}.retry.tmp`;
|
|
1061
|
+
fs.writeFileSync(temporary, `${JSON.stringify(root, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
1062
|
+
fs.renameSync(temporary, settingsPath);
|
|
1063
|
+
} catch {
|
|
1064
|
+
// Best effort: a failed restore must not hide the child result.
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
function childSettingsPath(env: NodeJS.ProcessEnv): string {
|
|
1069
|
+
const agentDir = env.PI_CODING_AGENT_DIR?.trim() || path.join(os.homedir(), ".pi", "agent");
|
|
1070
|
+
return path.join(agentDir, "settings.json");
|
|
1071
|
+
}
|
|
1072
|
+
|
|
975
1073
|
// ---------------------------------------------------------------------------
|
|
976
1074
|
// Build pi CLI arguments
|
|
977
1075
|
// ---------------------------------------------------------------------------
|
|
@@ -1605,51 +1703,61 @@ export async function runSingleTeammate(
|
|
|
1605
1703
|
if (acquisition && !acquisition.allowed) continue;
|
|
1606
1704
|
if (modelToUse && !attemptedModels.includes(modelToUse)) attemptedModels.push(modelToUse);
|
|
1607
1705
|
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1706
|
+
let settled = false;
|
|
1707
|
+
try {
|
|
1708
|
+
const maxRetries = acquisition?.state === "HALF_OPEN" ? 0 : NETWORK_RETRY_POLICY.maxRetries;
|
|
1709
|
+
let retryCount = 0;
|
|
1710
|
+
let candidateResult: SingleResult | undefined;
|
|
1611
1711
|
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1712
|
+
while (!options.signal?.aborted) {
|
|
1713
|
+
candidateResult = await runSingleAttempt(
|
|
1714
|
+
params, agentConfig, cwd, correlationId, replyTo, startTime, modelToUse, options,
|
|
1715
|
+
);
|
|
1716
|
+
lastResult = candidateResult;
|
|
1717
|
+
const error = resultFailureMessage(candidateResult.messages);
|
|
1718
|
+
const retryable = candidateResult.exitCode !== 0 && isRetryableProviderError(error);
|
|
1719
|
+
const fallbackEligible = candidateResult.exitCode !== 0 && isFallbackProviderError(error);
|
|
1720
|
+
|
|
1721
|
+
if (candidateResult.exitCode === 0) {
|
|
1722
|
+
if (acquisition?.allowed) { breaker.recordSuccess(acquisition); settled = true; }
|
|
1723
|
+
candidateResult.attemptedModels = attemptedModels.length > 1 ? attemptedModels : undefined;
|
|
1724
|
+
return candidateResult;
|
|
1725
|
+
}
|
|
1726
|
+
if (!retryable || retryCount >= maxRetries) {
|
|
1727
|
+
if (fallbackEligible && acquisition?.allowed) { breaker.recordRetryableFailure(acquisition); settled = true; }
|
|
1728
|
+
break;
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
retryCount += 1;
|
|
1732
|
+
const delayMs = retryDelayMs(retryCount);
|
|
1733
|
+
options.onRetry?.({
|
|
1734
|
+
correlationId,
|
|
1735
|
+
attempt: retryCount,
|
|
1736
|
+
maxRetries,
|
|
1737
|
+
delayMs,
|
|
1738
|
+
nextRetryAt: Date.now() + delayMs,
|
|
1739
|
+
error,
|
|
1740
|
+
});
|
|
1741
|
+
const ready = await (options.waitForRetry?.(delayMs, options.signal) ?? waitForRetryDelay(delayMs, options.signal));
|
|
1742
|
+
if (!ready) break;
|
|
1743
|
+
}
|
|
1620
1744
|
|
|
1621
|
-
if (candidateResult
|
|
1622
|
-
if (acquisition?.allowed) breaker.
|
|
1745
|
+
if (!candidateResult) {
|
|
1746
|
+
if (acquisition?.allowed) { breaker.releaseCandidate(acquisition); settled = true; }
|
|
1747
|
+
continue;
|
|
1748
|
+
}
|
|
1749
|
+
if (!isFallbackModelError(candidateResult.messages)) {
|
|
1750
|
+
if (acquisition?.allowed) { breaker.releaseCandidate(acquisition); settled = true; }
|
|
1623
1751
|
candidateResult.attemptedModels = attemptedModels.length > 1 ? attemptedModels : undefined;
|
|
1624
1752
|
return candidateResult;
|
|
1625
1753
|
}
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1754
|
+
} finally {
|
|
1755
|
+
// Safety net: if a HALF_OPEN trial was acquired but never settled
|
|
1756
|
+
// (e.g. runSingleAttempt threw, or error-classification diverged),
|
|
1757
|
+
// release it so the circuit does not stay stuck in HALF_OPEN forever.
|
|
1758
|
+
if (!settled && acquisition?.allowed && acquisition.state === "HALF_OPEN") {
|
|
1759
|
+
breaker.releaseCandidate(acquisition);
|
|
1629
1760
|
}
|
|
1630
|
-
|
|
1631
|
-
retryCount += 1;
|
|
1632
|
-
const delayMs = retryDelayMs(retryCount);
|
|
1633
|
-
options.onRetry?.({
|
|
1634
|
-
correlationId,
|
|
1635
|
-
attempt: retryCount,
|
|
1636
|
-
maxRetries,
|
|
1637
|
-
delayMs,
|
|
1638
|
-
nextRetryAt: Date.now() + delayMs,
|
|
1639
|
-
error,
|
|
1640
|
-
});
|
|
1641
|
-
const ready = await (options.waitForRetry?.(delayMs, options.signal) ?? waitForRetryDelay(delayMs, options.signal));
|
|
1642
|
-
if (!ready) break;
|
|
1643
|
-
}
|
|
1644
|
-
|
|
1645
|
-
if (!candidateResult) {
|
|
1646
|
-
if (acquisition?.allowed) breaker.releaseCandidate(acquisition);
|
|
1647
|
-
continue;
|
|
1648
|
-
}
|
|
1649
|
-
if (!isFallbackModelError(candidateResult.messages)) {
|
|
1650
|
-
if (acquisition?.allowed) breaker.releaseCandidate(acquisition);
|
|
1651
|
-
candidateResult.attemptedModels = attemptedModels.length > 1 ? attemptedModels : undefined;
|
|
1652
|
-
return candidateResult;
|
|
1653
1761
|
}
|
|
1654
1762
|
}
|
|
1655
1763
|
|
|
@@ -1879,6 +1987,7 @@ async function runSingleAttempt(
|
|
|
1879
1987
|
|
|
1880
1988
|
return new Promise<SingleResult>((resolve) => {
|
|
1881
1989
|
let child: ChildProcess;
|
|
1990
|
+
let releaseRetryPersistenceGuard = () => {};
|
|
1882
1991
|
|
|
1883
1992
|
const spawnEnv = buildChildSpawnEnv(correlationId, replyTo, options, schemaFile, outputFile);
|
|
1884
1993
|
|
|
@@ -1891,6 +2000,7 @@ async function runSingleAttempt(
|
|
|
1891
2000
|
stdio: useIpc ? ["pipe", "pipe", "pipe", "ipc"] : ["pipe", "pipe", "pipe"],
|
|
1892
2001
|
env: spawnEnv,
|
|
1893
2002
|
shell: spawnSpec.shell,
|
|
2003
|
+
windowsHide: true,
|
|
1894
2004
|
};
|
|
1895
2005
|
child = (options.spawnChildProcess ?? crossSpawn)(spawnSpec.command, spawnSpec.args, spawnOpts);
|
|
1896
2006
|
} catch (error) {
|
|
@@ -1921,6 +2031,7 @@ async function runSingleAttempt(
|
|
|
1921
2031
|
// Teammate owns retry and model failover for child runs. Disable Pi's inner
|
|
1922
2032
|
// retry loop first so one provider failure cannot multiply both budgets.
|
|
1923
2033
|
if (child.stdin) {
|
|
2034
|
+
releaseRetryPersistenceGuard = acquireRetryPersistenceGuard(childSettingsPath(spawnEnv));
|
|
1924
2035
|
writeChildStdinLine(child.stdin, JSON.stringify({ type: "set_auto_retry", enabled: false }));
|
|
1925
2036
|
}
|
|
1926
2037
|
|
|
@@ -2013,6 +2124,7 @@ async function runSingleAttempt(
|
|
|
2013
2124
|
exitCode = 0,
|
|
2014
2125
|
): void {
|
|
2015
2126
|
if (state.terminal || state.turnLifecycleSettled) return;
|
|
2127
|
+
releaseRetryPersistenceGuard();
|
|
2016
2128
|
state.turnLifecycleSettled = true;
|
|
2017
2129
|
progress.status = exitCode === 0 ? "completed" : "failed";
|
|
2018
2130
|
progress.resultReadyAt = undefined;
|
|
@@ -2374,8 +2486,9 @@ async function runSingleAttempt(
|
|
|
2374
2486
|
["message_end", onAssistantMessage],
|
|
2375
2487
|
["assistant", onAssistantMessage],
|
|
2376
2488
|
["message_update", onMessageUpdate],
|
|
2377
|
-
|
|
2378
|
-
|
|
2489
|
+
["response", (event) => {
|
|
2490
|
+
if (event.command === "set_auto_retry") releaseRetryPersistenceGuard();
|
|
2491
|
+
}],
|
|
2379
2492
|
["tool_execution_start", onToolStart],
|
|
2380
2493
|
["tool_execution_end", onToolCompleted],
|
|
2381
2494
|
["tool_result_end", onToolCompleted],
|
|
@@ -2418,6 +2531,7 @@ async function runSingleAttempt(
|
|
|
2418
2531
|
});
|
|
2419
2532
|
|
|
2420
2533
|
child.on("close", (code, signal) => {
|
|
2534
|
+
releaseRetryPersistenceGuard();
|
|
2421
2535
|
clearAllTimers();
|
|
2422
2536
|
termination.cleanup();
|
|
2423
2537
|
unbindTerminationSignal();
|
|
@@ -2502,6 +2616,7 @@ async function runSingleAttempt(
|
|
|
2502
2616
|
});
|
|
2503
2617
|
|
|
2504
2618
|
child.on("error", (error) => {
|
|
2619
|
+
releaseRetryPersistenceGuard();
|
|
2505
2620
|
clearAllTimers();
|
|
2506
2621
|
unbindTerminationSignal();
|
|
2507
2622
|
|
package/src/runs/retry.ts
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
|
-
export const NETWORK_RETRY_POLICY = Object.freeze({
|
|
2
|
-
maxRetries:
|
|
3
|
-
initialDelayMs: 1_000,
|
|
4
|
-
maxDelayMs:
|
|
5
|
-
});
|
|
6
|
-
|
|
7
|
-
export type RetryErrorKind = "network" | "provider" | "non-retryable";
|
|
8
|
-
|
|
9
|
-
const NON_RETRYABLE_ERROR =
|
|
10
|
-
/usage[_\s-]*limit|multi-auth rotation failed|unauthori[sz]ed|invalid api key|forbidden|invalid model|unknown model|context[_\s-]*length[_\s-]*exceeded|input exceeds the context window|schema[-\s]*valid|validation (?:failed|error)/i;
|
|
11
|
-
|
|
12
|
-
const NETWORK_ERROR =
|
|
13
|
-
/\b(?:econnreset|econnrefused|ehostunreach|enotfound|eai_again|etimedout|socket hang up|fetch failed|network(?: error| request)?|websocket closed|sse response headers timed out|headers timed out|tls|certificate)\b/i;
|
|
14
|
-
|
|
15
|
-
const PROVIDER_ERROR =
|
|
16
|
-
/\b(?:429|5\d\d|rate limit(?:ed)?|capacity|overloaded|unavailable|provider returned error)\b/i;
|
|
17
|
-
|
|
18
|
-
const FALLBACK_ONLY_ERROR =
|
|
19
|
-
/\b(?:402|insufficient (?:balance|credits?)|credits? exhausted|billing quota)\b/i;
|
|
20
|
-
|
|
21
|
-
export function classifyRetryError(message: string | undefined): RetryErrorKind {
|
|
22
|
-
if (!message || NON_RETRYABLE_ERROR.test(message)) return "non-retryable";
|
|
23
|
-
if (NETWORK_ERROR.test(message)) return "network";
|
|
24
|
-
if (PROVIDER_ERROR.test(message)) return "provider";
|
|
25
|
-
return "non-retryable";
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export function isRetryableProviderError(message: string | undefined): boolean {
|
|
29
|
-
return classifyRetryError(message) !== "non-retryable";
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
export function isFallbackProviderError(message: string | undefined): boolean {
|
|
33
|
-
if (!message || NON_RETRYABLE_ERROR.test(message)) return false;
|
|
34
|
-
return isRetryableProviderError(message) || FALLBACK_ONLY_ERROR.test(message);
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
export function retryDelayMs(retry: number): number {
|
|
38
|
-
const normalizedRetry = Math.max(1, Math.floor(retry));
|
|
39
|
-
return Math.min(
|
|
40
|
-
NETWORK_RETRY_POLICY.initialDelayMs * 2 ** (normalizedRetry - 1),
|
|
41
|
-
NETWORK_RETRY_POLICY.maxDelayMs,
|
|
42
|
-
);
|
|
43
|
-
}
|
|
1
|
+
export const NETWORK_RETRY_POLICY = Object.freeze({
|
|
2
|
+
maxRetries: 12,
|
|
3
|
+
initialDelayMs: 1_000,
|
|
4
|
+
maxDelayMs: 10 * 60_000,
|
|
5
|
+
});
|
|
6
|
+
|
|
7
|
+
export type RetryErrorKind = "network" | "provider" | "non-retryable";
|
|
8
|
+
|
|
9
|
+
const NON_RETRYABLE_ERROR =
|
|
10
|
+
/usage[_\s-]*limit|multi-auth rotation failed|unauthori[sz]ed|invalid api key|forbidden|invalid model|unknown model|context[_\s-]*length[_\s-]*exceeded|input exceeds the context window|schema[-\s]*valid|validation (?:failed|error)/i;
|
|
11
|
+
|
|
12
|
+
const NETWORK_ERROR =
|
|
13
|
+
/\b(?:econnreset|econnrefused|ehostunreach|enotfound|eai_again|etimedout|socket hang up|fetch failed|network(?: error| request)?|connection (?:error|failed|failure|reset|refused|timed out|timeout|closed)|websocket closed|sse response headers timed out|headers timed out|tls|certificate)\b/i;
|
|
14
|
+
|
|
15
|
+
const PROVIDER_ERROR =
|
|
16
|
+
/\b(?:429|5\d\d|rate limit(?:ed)?|capacity|overloaded|unavailable|provider returned error)\b/i;
|
|
17
|
+
|
|
18
|
+
const FALLBACK_ONLY_ERROR =
|
|
19
|
+
/\b(?:402|insufficient (?:balance|credits?)|credits? exhausted|billing quota)\b/i;
|
|
20
|
+
|
|
21
|
+
export function classifyRetryError(message: string | undefined): RetryErrorKind {
|
|
22
|
+
if (!message || NON_RETRYABLE_ERROR.test(message)) return "non-retryable";
|
|
23
|
+
if (NETWORK_ERROR.test(message)) return "network";
|
|
24
|
+
if (PROVIDER_ERROR.test(message)) return "provider";
|
|
25
|
+
return "non-retryable";
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function isRetryableProviderError(message: string | undefined): boolean {
|
|
29
|
+
return classifyRetryError(message) !== "non-retryable";
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function isFallbackProviderError(message: string | undefined): boolean {
|
|
33
|
+
if (!message || NON_RETRYABLE_ERROR.test(message)) return false;
|
|
34
|
+
return isRetryableProviderError(message) || FALLBACK_ONLY_ERROR.test(message);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function retryDelayMs(retry: number): number {
|
|
38
|
+
const normalizedRetry = Math.max(1, Math.floor(retry));
|
|
39
|
+
return Math.min(
|
|
40
|
+
NETWORK_RETRY_POLICY.initialDelayMs * 2 ** (normalizedRetry - 1),
|
|
41
|
+
NETWORK_RETRY_POLICY.maxDelayMs,
|
|
42
|
+
);
|
|
43
|
+
}
|
package/src/tui/render.ts
CHANGED
|
@@ -217,27 +217,13 @@ function renderMultiTaskCall(
|
|
|
217
217
|
return dynamicComponent((w) => [header, ...tree.map((row) => row.text)]
|
|
218
218
|
.map((line) => truncateToWidth(line, Math.max(1, w), "…")));
|
|
219
219
|
}
|
|
220
|
-
//
|
|
220
|
+
// 结果组件(前台流式 progress / 后台 ack 的 progress 快照)紧随调用行渲染,
|
|
221
|
+
// 已包含同一任务拓扑、依赖边与实时状态(后台还带 correlation id)。
|
|
221
222
|
// collapsed 调用仅保留启动摘要,避免相邻组件重复显示任务行。
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
}
|
|
227
|
-
// Collapsed: list every agent with its dependency edges so the DAG stays visible.
|
|
228
|
-
const agentLines = tasks.map((task, index) => {
|
|
229
|
-
const label = task.name
|
|
230
|
-
? `${theme.fg("accent", `@${task.name}`)} ${theme.fg("dim", `(${task.agent})`)}`
|
|
231
|
-
: theme.fg("accent", task.agent);
|
|
232
|
-
const deps = dependenciesByIndex[index];
|
|
233
|
-
const flowMarker = deps.length > 0 ? "→" : "•";
|
|
234
|
-
const dependencyHint = deps.length > 0
|
|
235
|
-
? theme.fg("dim", ` ← result${deps.length === 1 ? "" : "s"} ${deps.map((d) => `#${d + 1}`).join(", ")}`)
|
|
236
|
-
: "";
|
|
237
|
-
return `${theme.fg("dim", flowMarker)} ${theme.fg("accent", String(index + 1))} ${theme.fg("dim", "□")} ${label}${dependencyHint}`;
|
|
238
|
-
});
|
|
239
|
-
return dynamicComponent((w) => [header, ...agentLines]
|
|
240
|
-
.map((line) => truncateToWidth(line, Math.max(1, w), "…")));
|
|
223
|
+
// DAG 细节仍可在 expanded 调用视图与结果 progress 树中查看。
|
|
224
|
+
return dynamicComponent((w) => [
|
|
225
|
+
truncateToWidth(header, Math.max(1, w), "…"),
|
|
226
|
+
]);
|
|
241
227
|
}
|
|
242
228
|
|
|
243
229
|
function isLinearChain(tasks: TaskArg[], taskNames: Set<string>): boolean {
|
|
@@ -155,6 +155,21 @@ export declare function rejectChildProxyRequest(pendingRequests: ChildProxyPendi
|
|
|
155
155
|
/** @internal Exported for lifecycle regression tests. */
|
|
156
156
|
export declare function rejectAllChildProxyRequests(pendingRequests: ChildProxyPendingRequests, error: Error): void;
|
|
157
157
|
/** @internal Exported for lifecycle regression tests. */
|
|
158
|
+
export type IpcSender = (message: Record<string, unknown>, callback: (error: Error | null) => void) => boolean;
|
|
159
|
+
/**
|
|
160
|
+
* Builds the IPC sender the teammate proxy uses to talk to its parent.
|
|
161
|
+
*
|
|
162
|
+
* Node's IPC `send` reads `this.connected` internally, so detaching it from its
|
|
163
|
+
* owner (`const send = proc.send`) leaves `this` undefined in module scope and
|
|
164
|
+
* throws "Cannot read properties of undefined (reading 'connected')" on the
|
|
165
|
+
* first call — which broke every proxied teammate tool in a nested child.
|
|
166
|
+
* Binding the owner keeps the proxied call working. Returns undefined when no
|
|
167
|
+
* live IPC channel exists.
|
|
168
|
+
*/
|
|
169
|
+
export declare function createIpcSender(proc?: {
|
|
170
|
+
connected?: boolean;
|
|
171
|
+
send?: (...args: any[]) => boolean;
|
|
172
|
+
}): IpcSender | undefined;
|
|
158
173
|
export declare function createChildProxyRequest(pendingRequests: ChildProxyPendingRequests, requestId: string, message: Record<string, unknown>, send: (message: Record<string, unknown>, callback: (error: Error | null) => void) => boolean, timeoutMs?: number, signal?: AbortSignal): Promise<unknown>;
|
|
159
174
|
export default function registerTeammateExtension(pi: ExtensionAPI, runtimeOptions?: TeammateRuntimeOptions): void;
|
|
160
175
|
type AgentListView = "active" | "named" | "all";
|
|
@@ -248,6 +248,12 @@ export declare function checkDepthGuard(depth: number): {
|
|
|
248
248
|
current: number;
|
|
249
249
|
max: number;
|
|
250
250
|
};
|
|
251
|
+
/**
|
|
252
|
+
* Pi's RPC set_auto_retry command persists to settings.json even though the
|
|
253
|
+
* child only needs a session-local override. Restore the original value after
|
|
254
|
+
* every concurrently starting child has acknowledged that command.
|
|
255
|
+
*/
|
|
256
|
+
export declare function acquireRetryPersistenceGuard(settingsPath: string): () => void;
|
|
251
257
|
export declare function clampThinkingForModel(thinking: TeammateThinkingLevel, model: string | undefined, modelCapabilities?: readonly TeammateModelCapability[]): TeammateThinkingLevel;
|
|
252
258
|
export declare function validateModelSpecifier(model: string): string;
|
|
253
259
|
export declare function resolveModelSpecifier(model: string, modelCapabilities?: readonly TeammateModelCapability[]): string;
|
package/types/runs/retry.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export declare const NETWORK_RETRY_POLICY: Readonly<{
|
|
2
|
-
maxRetries:
|
|
2
|
+
maxRetries: 12;
|
|
3
3
|
initialDelayMs: 1000;
|
|
4
|
-
maxDelayMs:
|
|
4
|
+
maxDelayMs: number;
|
|
5
5
|
}>;
|
|
6
6
|
export type RetryErrorKind = "network" | "provider" | "non-retryable";
|
|
7
7
|
export declare function classifyRetryError(message: string | undefined): RetryErrorKind;
|