takomi 2.1.29 → 2.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/agents/coder.md +6 -0
- package/.pi/agents/orchestrator.md +16 -0
- package/.pi/extensions/oauth-router/README.md +4 -2
- package/.pi/extensions/oauth-router/commands.ts +37 -9
- package/.pi/extensions/oauth-router/config.ts +7 -1
- package/.pi/extensions/oauth-router/index.ts +1 -1
- package/.pi/extensions/oauth-router/oauth-flow.ts +71 -22
- package/.pi/extensions/oauth-router/provider.ts +112 -12
- package/.pi/extensions/oauth-router/state.ts +12 -1
- package/.pi/extensions/oauth-router/types.ts +15 -2
- package/.pi/extensions/takomi-runtime/context-panel.ts +130 -27
- package/.pi/extensions/takomi-runtime/index.ts +2 -0
- package/.pi/extensions/takomi-runtime/takomi-stats.js +679 -679
- package/.pi/prompts/build-prompt.md +15 -0
- package/.pi/prompts/genesis-prompt.md +8 -0
- package/.pi/prompts/takomi-prompt.md +16 -0
- package/package.json +1 -1
- package/src/takomi-stats.js +749 -679
package/.pi/agents/coder.md
CHANGED
|
@@ -42,6 +42,12 @@ If scope is unclear, ask a focused question instead of guessing.
|
|
|
42
42
|
- handle errors and edge cases deliberately
|
|
43
43
|
- avoid broad refactors unless requested or required
|
|
44
44
|
|
|
45
|
+
## Tool-Use Safety
|
|
46
|
+
- Keep `bash` commands short; use them for filesystem operations, running scripts, inspections, and verification.
|
|
47
|
+
- Do not embed large generated files or long scripts directly in `bash` heredocs.
|
|
48
|
+
- Use `write` for large files, or `write` a small generator script to disk and then run it.
|
|
49
|
+
- If a command fails with `ENAMETOOLONG`, stop using the oversized inline command and switch to file-based writes or a written script.
|
|
50
|
+
|
|
45
51
|
## Phase 4: Verification
|
|
46
52
|
After edits, run the strongest practical checks for the repo.
|
|
47
53
|
For TypeScript/TSX work, prefer `npx tsc --noEmit` or the project equivalent.
|
|
@@ -90,6 +90,22 @@ Task packets should be self-contained enough for a subagent to execute without g
|
|
|
90
90
|
|
|
91
91
|
Keep human-readable markdown meaningful; keep JSON as tracking/continuity metadata.
|
|
92
92
|
|
|
93
|
+
### Tool-Use Safety for Authored Plans
|
|
94
|
+
- Do not pack large markdown plans, task packets, or generator scripts into one oversized `bash` command.
|
|
95
|
+
- Prefer direct markdown authorship through the appropriate file-writing path, then register the same content with `takomi_board`.
|
|
96
|
+
- If repeated files are needed, write a small generator script to disk first and run it with a short shell command.
|
|
97
|
+
- If `ENAMETOOLONG` occurs, switch immediately to file-based writes or a written generator script instead of retrying the inline command.
|
|
98
|
+
|
|
99
|
+
### Stage Expansion Quality Gate
|
|
100
|
+
Before calling `takomi_board expand_stage`:
|
|
101
|
+
- Write or provide full `taskMarkdown` for every expanded task.
|
|
102
|
+
- Ensure each task has real Scope, Definition of Done, Expected Artifacts, Dependencies, and Verification/Review checkpoint.
|
|
103
|
+
- Include exact prime-agent context paths and relevant FR/mockup/design docs.
|
|
104
|
+
- For Build, decompose by implementation slice and FR coverage, not just broad labels like “school workflows.”
|
|
105
|
+
- If a detailed subagent prompt is needed, the task packet should contain the same level of detail before launch.
|
|
106
|
+
- Do not leave generated task files with `Scope: None specified`, `Definition Of Done: None specified`, or `Expected Artifacts: None specified`.
|
|
107
|
+
- If `takomi_board` produced generic task packets, immediately repair them with detailed markdown before dispatching more work.
|
|
108
|
+
|
|
93
109
|
## Phase 4: Delegation
|
|
94
110
|
When delegating:
|
|
95
111
|
- send self-contained task instructions
|
|
@@ -27,6 +27,8 @@ Pi extension that auto-loads and registers an `oauth-router` provider with multi
|
|
|
27
27
|
- weighted round robin
|
|
28
28
|
- 429 cooldowns
|
|
29
29
|
- transient failure penalties
|
|
30
|
+
- client/network transport failure tracking without default account cooldown
|
|
31
|
+
- 5 router-level client/network retries by default before pre-output failover, starting at 5s and doubling
|
|
30
32
|
- auth failure quarantine
|
|
31
33
|
- safe pre-output failover
|
|
32
34
|
|
|
@@ -45,7 +47,7 @@ The extension ships with two default upstream profiles:
|
|
|
45
47
|
- api: `openai-responses`
|
|
46
48
|
- default models: `gpt-4o`, `gpt-4.1`, `o4-mini`
|
|
47
49
|
|
|
48
|
-
Edit `~/.pi/agent/oauth-router/config.json` to add more upstreams, swap endpoints,
|
|
50
|
+
Edit `~/.pi/agent/oauth-router/config.json` to add more upstreams, swap endpoints, change model catalogs, or tune retry behavior. By default client-side transport failures such as `Codex SSE response headers timed out after 10000ms` retry the same account 5 times with exponential backoff (`5s`, `10s`, `20s`, `40s`, then capped at `60s`) before router failover. These failures are recorded but do not cool down an account unless `clientNetworkPenaltyMs` is set above `0`.
|
|
49
51
|
|
|
50
52
|
## Setup
|
|
51
53
|
|
|
@@ -98,7 +100,7 @@ Health and local usage state live separately in:
|
|
|
98
100
|
|
|
99
101
|
- `~/.pi/agent/oauth-router/state.json`
|
|
100
102
|
|
|
101
|
-
The router records successful request usage per account for rolling local 5-hour and weekly windows. These are router-observed counters only. `/router-refresh-usage [id|all]`
|
|
103
|
+
The router records successful request usage per account for rolling local 5-hour and weekly windows. These are router-observed counters only. `/router-refresh-usage [id|all]` probes ChatGPT/Codex provider quota from `/backend-api/wham/usage` first and treats `rate_limit.primary_window.used_percent` as the 5-hour window and `rate_limit.secondary_window.used_percent` as the weekly window. It then falls back to safe token claims such as account id, token expiry, issuer, subject, and available claim keys. `/router-usage` shows a compact visual quota view; `/router-usage-raw [id]` shows detailed/raw provider data. Provider-side quota counters depend on undocumented upstream endpoints and may change; they are not normally present in the OAuth token itself.
|
|
102
104
|
|
|
103
105
|
Most account commands accept an optional account ID. When run in the Pi UI without an ID, the extension opens an account picker instead of dumping the same account list repeatedly.
|
|
104
106
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ExtensionAPI, ExtensionCommandContext } from "@
|
|
1
|
+
import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import { createAccountFromUpstream } from "./oauth-flow.ts";
|
|
3
3
|
import type { RouterStatusRow, RouterUsageSummary, RouterUsageWindowSummary, RoutingPolicyName, StoredRouterAccount } from "./types.ts";
|
|
4
4
|
import { RouterRuntime } from "./provider.ts";
|
|
@@ -98,7 +98,8 @@ export function formatStatusReport(runtime: RouterRuntime): string {
|
|
|
98
98
|
` upstream=${row.upstream} enabled=${row.enabled} weight=${row.weight} state=${health}`,
|
|
99
99
|
` lastUsed=${formatLastUsedSummary(row.lastUsedAt)} | lastStatus=${row.lastStatus ?? "-"} | expires=${formatWhen(row.expires)}`,
|
|
100
100
|
` successes=${row.successCount} failures=${row.failures} 429s=${row.rateLimitCount} authFailures=${row.authFailureCount}`,
|
|
101
|
-
|
|
101
|
+
row.lastError ? ` lastError=${row.lastError}` : undefined,
|
|
102
|
+
].filter((line): line is string => Boolean(line)).join("\n");
|
|
102
103
|
})
|
|
103
104
|
: ["- No accounts configured yet. Use /router-login add."];
|
|
104
105
|
|
|
@@ -118,7 +119,18 @@ export function formatStatusReport(runtime: RouterRuntime): string {
|
|
|
118
119
|
].join("\n");
|
|
119
120
|
}
|
|
120
121
|
|
|
121
|
-
export function
|
|
122
|
+
export async function refreshUsageAfterCredentialChange(runtime: RouterRuntime, accountId: string): Promise<string> {
|
|
123
|
+
try {
|
|
124
|
+
const snapshot = await runtime.refreshUsageSnapshot(accountId);
|
|
125
|
+
const windows = [snapshot.fiveHour ? "5h" : undefined, snapshot.weekly ? "weekly" : undefined].filter(Boolean).join("/");
|
|
126
|
+
return ` Provider usage refreshed (${snapshot.source}${windows ? `, ${windows}` : ""}).`;
|
|
127
|
+
} catch (error) {
|
|
128
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
129
|
+
return ` Provider usage refresh failed: ${message}`;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function formatAccountsReport(runtime: RouterRuntime): string {
|
|
122
134
|
const rows = runtime.getStatusRows();
|
|
123
135
|
const accounts = rows.length
|
|
124
136
|
? [...rows]
|
|
@@ -404,13 +416,15 @@ async function handleRouterLogin(pi: ExtensionAPI, runtime: RouterRuntime, args:
|
|
|
404
416
|
updatedAt: Date.now(),
|
|
405
417
|
});
|
|
406
418
|
runtime.clearAccountHealth(duplicate.id);
|
|
419
|
+
const usageMessage = await refreshUsageAfterCredentialChange(runtime, duplicate.id);
|
|
407
420
|
setFooterStatus(ctx, runtime);
|
|
408
|
-
emitReport(pi, `Updated existing account ${duplicate.id} (${duplicate.label}) with fresh credentials instead of adding a duplicate
|
|
421
|
+
emitReport(pi, `Updated existing account ${duplicate.id} (${duplicate.label}) with fresh credentials instead of adding a duplicate.${usageMessage}`);
|
|
409
422
|
return;
|
|
410
423
|
}
|
|
411
424
|
runtime.addAccount(account);
|
|
425
|
+
const usageMessage = await refreshUsageAfterCredentialChange(runtime, account.id);
|
|
412
426
|
setFooterStatus(ctx, runtime);
|
|
413
|
-
emitReport(pi, `Added account ${account.id} (${account.label}) for upstream ${upstream.id}
|
|
427
|
+
emitReport(pi, `Added account ${account.id} (${account.label}) for upstream ${upstream.id}.${usageMessage}`);
|
|
414
428
|
ctx.ui.notify(`Added ${account.id}`, "info");
|
|
415
429
|
return;
|
|
416
430
|
}
|
|
@@ -459,15 +473,17 @@ async function handleRouterLogin(pi: ExtensionAPI, runtime: RouterRuntime, args:
|
|
|
459
473
|
updatedAt: Date.now(),
|
|
460
474
|
});
|
|
461
475
|
runtime.clearAccountHealth(existing.id);
|
|
476
|
+
const usageMessage = await refreshUsageAfterCredentialChange(runtime, existing.id);
|
|
462
477
|
setFooterStatus(ctx, runtime);
|
|
463
|
-
emitReport(pi, `Re-logged account ${existing.id} (${existing.label}) and cleared auth state
|
|
478
|
+
emitReport(pi, `Re-logged account ${existing.id} (${existing.label}) and cleared auth state.${usageMessage}`);
|
|
464
479
|
return;
|
|
465
480
|
}
|
|
466
481
|
case "refresh": {
|
|
467
482
|
const account = await pickAccount(runtime, ctx, first, "Choose account to refresh");
|
|
468
483
|
const refreshed = await runtime.refreshAccount(account.id);
|
|
484
|
+
const usageMessage = await refreshUsageAfterCredentialChange(runtime, refreshed.id);
|
|
469
485
|
setFooterStatus(ctx, runtime);
|
|
470
|
-
emitReport(pi, `Refreshed account ${refreshed.id} (${refreshed.label})
|
|
486
|
+
emitReport(pi, `Refreshed account ${refreshed.id} (${refreshed.label}).${usageMessage}`);
|
|
471
487
|
return;
|
|
472
488
|
}
|
|
473
489
|
case "help":
|
|
@@ -555,8 +571,20 @@ export function registerRouterCommands(pi: ExtensionAPI, runtime: RouterRuntime)
|
|
|
555
571
|
const [requestedId] = parseArgs(args || "");
|
|
556
572
|
const target = await pickUsageTarget(runtime, ctx, requestedId);
|
|
557
573
|
const ids = target === "all" ? runtime.listAccounts().map((account) => account.id) : [target];
|
|
558
|
-
|
|
559
|
-
|
|
574
|
+
const failures: string[] = [];
|
|
575
|
+
|
|
576
|
+
for (const accountId of ids) {
|
|
577
|
+
try {
|
|
578
|
+
await runtime.refreshUsageSnapshot(accountId);
|
|
579
|
+
} catch (error) {
|
|
580
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
581
|
+
failures.push(`- ${accountId}: ${message}`);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
const report = formatUsageReport(runtime, target === "all" ? undefined : target);
|
|
586
|
+
emitReport(pi, failures.length ? [report, "", "## Refresh failures", ...failures].join("\n") : report);
|
|
587
|
+
if (failures.length) ctx.ui.notify(`oauth-router usage refreshed with ${failures.length} failure(s)`, "error");
|
|
560
588
|
setFooterStatus(ctx, runtime);
|
|
561
589
|
},
|
|
562
590
|
});
|
|
@@ -98,7 +98,7 @@ const DEFAULT_UPSTREAMS: RouterUpstreamConfig[] = [
|
|
|
98
98
|
modelIds: ["gpt-5.1", "gpt-5.4", "gpt-5.4-mini", "gpt-5.5"],
|
|
99
99
|
usageProbe: {
|
|
100
100
|
enabled: true,
|
|
101
|
-
timeoutMs:
|
|
101
|
+
timeoutMs: 8_000,
|
|
102
102
|
endpoints: [
|
|
103
103
|
"/wham/usage",
|
|
104
104
|
"/codex/usage",
|
|
@@ -117,6 +117,12 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
117
117
|
tokenRefreshSkewMs: 60_000,
|
|
118
118
|
rateLimitCooldownMs: 120_000,
|
|
119
119
|
transientPenaltyMs: 30_000,
|
|
120
|
+
upstreamMaxRetries: 0,
|
|
121
|
+
upstreamMaxRetryDelayMs: 60_000,
|
|
122
|
+
clientNetworkMaxRetries: 5,
|
|
123
|
+
clientNetworkRetryBaseDelayMs: 5_000,
|
|
124
|
+
clientNetworkMaxRetryDelayMs: 60_000,
|
|
125
|
+
clientNetworkPenaltyMs: 0,
|
|
120
126
|
models: DEFAULT_MODELS,
|
|
121
127
|
upstreams: DEFAULT_UPSTREAMS,
|
|
122
128
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ExtensionAPI } from "@
|
|
1
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import { registerRouterCommands, formatStatusReport } from "./commands.ts";
|
|
3
3
|
import { registerRouterProvider, RouterRuntime } from "./provider.ts";
|
|
4
4
|
|
|
@@ -3,7 +3,7 @@ import { Buffer } from "node:buffer";
|
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import type { OAuthCredentials, OAuthSelectPrompt } from "@earendil-works/pi-ai/oauth";
|
|
5
5
|
import { getOAuthProvider } from "@earendil-works/pi-ai/oauth";
|
|
6
|
-
import type { ExtensionContext } from "@
|
|
6
|
+
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
7
7
|
import type { RouterProviderQuotaWindow, RouterProviderUsageSnapshot, RouterUpstreamConfig, StoredRouterAccount } from "./types.ts";
|
|
8
8
|
|
|
9
9
|
function now() {
|
|
@@ -197,7 +197,8 @@ export function inspectAccountToken(account: StoredRouterAccount): RouterProvide
|
|
|
197
197
|
getStringClaim(claims, "account_id");
|
|
198
198
|
const planType =
|
|
199
199
|
(typeof account.meta?.planType === "string" ? account.meta.planType : undefined) ??
|
|
200
|
-
(openaiAuth && (getStringClaim(openaiAuth, "plan_type") ?? getStringClaim(openaiAuth, "planType"))) ??
|
|
200
|
+
(openaiAuth && (getStringClaim(openaiAuth, "chatgpt_plan_type") ?? getStringClaim(openaiAuth, "plan_type") ?? getStringClaim(openaiAuth, "planType"))) ??
|
|
201
|
+
getStringClaim(claims, "chatgpt_plan_type") ??
|
|
201
202
|
getStringClaim(claims, "plan_type") ??
|
|
202
203
|
getStringClaim(claims, "planType");
|
|
203
204
|
|
|
@@ -254,12 +255,16 @@ function normalizeResetAt(record: Record<string, unknown>): number | undefined {
|
|
|
254
255
|
return raw < 10_000_000_000 ? raw * 1000 : raw;
|
|
255
256
|
}
|
|
256
257
|
|
|
258
|
+
function clampPercent(value: number): number {
|
|
259
|
+
return Math.max(0, Math.min(100, value));
|
|
260
|
+
}
|
|
261
|
+
|
|
257
262
|
function quotaFromRecord(label: string, record: Record<string, unknown>): RouterProviderQuotaWindow | undefined {
|
|
258
263
|
const limit = firstNumber(record.limit, record.cap, record.total, record.max, record.quota);
|
|
259
264
|
const remaining = firstNumber(record.remaining, record.available, record.left, record.remaining_messages, record.remainingMessages);
|
|
260
265
|
const used = firstNumber(record.used, record.consumed, record.current, record.used_messages, record.usedMessages);
|
|
261
266
|
const usedPercent = firstNumber(record.used_percent, record.usedPercent);
|
|
262
|
-
const percentRemaining = firstNumber(record.percent_remaining, record.percentRemaining, record.remaining_percent, record.remainingPercent) ?? (usedPercent !== undefined ? 100 -
|
|
267
|
+
const percentRemaining = firstNumber(record.percent_remaining, record.percentRemaining, record.remaining_percent, record.remainingPercent) ?? (usedPercent !== undefined ? 100 - clampPercent(usedPercent) : undefined);
|
|
263
268
|
const resetAt = normalizeResetAt(record);
|
|
264
269
|
|
|
265
270
|
if (limit === undefined && remaining === undefined && used === undefined && percentRemaining === undefined && resetAt === undefined) {
|
|
@@ -271,7 +276,24 @@ function quotaFromRecord(label: string, record: Record<string, unknown>): Router
|
|
|
271
276
|
used,
|
|
272
277
|
limit,
|
|
273
278
|
remaining,
|
|
274
|
-
percentRemaining: percentRemaining !== undefined ? percentRemaining : limit && remaining !== undefined ? (remaining / limit) * 100 : undefined,
|
|
279
|
+
percentRemaining: percentRemaining !== undefined ? clampPercent(percentRemaining) : limit && remaining !== undefined ? clampPercent((remaining / limit) * 100) : undefined,
|
|
280
|
+
resetAt,
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function quotaFromWhamWindow(label: string, window: Record<string, unknown> | undefined, limitReached?: boolean): RouterProviderQuotaWindow | undefined {
|
|
285
|
+
if (!window) return undefined;
|
|
286
|
+
|
|
287
|
+
const usedPercent = firstNumber(window.used_percent, window.usedPercent);
|
|
288
|
+
const explicitRemainingPercent = firstNumber(window.percent_remaining, window.percentRemaining, window.remaining_percent, window.remainingPercent);
|
|
289
|
+
const percentRemaining = explicitRemainingPercent ?? (usedPercent !== undefined ? 100 - clampPercent(usedPercent) : limitReached ? 0 : undefined);
|
|
290
|
+
const resetAt = normalizeResetAt(window);
|
|
291
|
+
|
|
292
|
+
if (percentRemaining === undefined && resetAt === undefined) return undefined;
|
|
293
|
+
|
|
294
|
+
return {
|
|
295
|
+
label,
|
|
296
|
+
percentRemaining: percentRemaining !== undefined ? clampPercent(percentRemaining) : undefined,
|
|
275
297
|
resetAt,
|
|
276
298
|
};
|
|
277
299
|
}
|
|
@@ -282,20 +304,43 @@ function mergeQuota(previous: RouterProviderQuotaWindow | undefined, next: Route
|
|
|
282
304
|
return { ...previous, ...next };
|
|
283
305
|
}
|
|
284
306
|
|
|
307
|
+
function pickWindow(record: Record<string, unknown>, snakeKey: string, camelKey: string): Record<string, unknown> | undefined {
|
|
308
|
+
const snake = record[snakeKey];
|
|
309
|
+
if (isRecord(snake)) return snake;
|
|
310
|
+
const camel = record[camelKey];
|
|
311
|
+
return isRecord(camel) ? camel : undefined;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function extractWhamUsage(json: unknown): Pick<RouterProviderUsageSnapshot, "planType" | "fiveHour" | "weekly"> {
|
|
315
|
+
if (!isRecord(json)) return {};
|
|
316
|
+
|
|
317
|
+
const planType = firstString(json.plan_type, json.planType, json.plan, json.subscription_plan, json.subscriptionPlan);
|
|
318
|
+
const rateLimit = isRecord(json.rate_limit) ? json.rate_limit : isRecord(json.rateLimit) ? json.rateLimit : undefined;
|
|
319
|
+
if (!rateLimit) return { planType };
|
|
320
|
+
|
|
321
|
+
const limitReached = Boolean(rateLimit.limit_reached ?? rateLimit.limitReached);
|
|
322
|
+
return {
|
|
323
|
+
planType,
|
|
324
|
+
fiveHour: quotaFromWhamWindow("5h", pickWindow(rateLimit, "primary_window", "primaryWindow"), limitReached),
|
|
325
|
+
weekly: quotaFromWhamWindow("weekly", pickWindow(rateLimit, "secondary_window", "secondaryWindow"), limitReached),
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
285
329
|
function extractProviderUsage(json: unknown): Pick<RouterProviderUsageSnapshot, "planType" | "fiveHour" | "weekly"> {
|
|
286
|
-
|
|
287
|
-
let
|
|
288
|
-
let
|
|
330
|
+
const wham = extractWhamUsage(json);
|
|
331
|
+
let planType = wham.planType;
|
|
332
|
+
let fiveHour = wham.fiveHour;
|
|
333
|
+
let weekly = wham.weekly;
|
|
334
|
+
|
|
335
|
+
if (fiveHour || weekly) return { planType, fiveHour, weekly };
|
|
289
336
|
|
|
290
337
|
walkRecords(json, (record) => {
|
|
291
338
|
planType ??= firstString(record.plan_type, record.planType, record.plan, record.subscription_plan, record.subscriptionPlan, record.account_plan, record.accountPlan);
|
|
292
339
|
|
|
293
|
-
const rateLimit = isRecord(record.rate_limit) ? record.rate_limit : undefined;
|
|
340
|
+
const rateLimit = isRecord(record.rate_limit) ? record.rate_limit : isRecord(record.rateLimit) ? record.rateLimit : undefined;
|
|
294
341
|
if (rateLimit) {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
fiveHour = mergeQuota(fiveHour, primary ? quotaFromRecord("5h", primary) : undefined);
|
|
298
|
-
weekly = mergeQuota(weekly, secondary ? quotaFromRecord("weekly", secondary) : undefined);
|
|
342
|
+
fiveHour = mergeQuota(fiveHour, quotaFromWhamWindow("5h", pickWindow(rateLimit, "primary_window", "primaryWindow"), Boolean(rateLimit.limit_reached ?? rateLimit.limitReached)));
|
|
343
|
+
weekly = mergeQuota(weekly, quotaFromWhamWindow("weekly", pickWindow(rateLimit, "secondary_window", "secondaryWindow"), Boolean(rateLimit.limit_reached ?? rateLimit.limitReached)));
|
|
299
344
|
}
|
|
300
345
|
|
|
301
346
|
const name = [record.name, record.label, record.bucket, record.window, record.period, record.type, record.key, record.id]
|
|
@@ -361,7 +406,7 @@ export async function refreshProviderUsageSnapshot(account: StoredRouterAccount,
|
|
|
361
406
|
|
|
362
407
|
const accountId = base.accountId;
|
|
363
408
|
const endpoints = Array.from(new Set(["/wham/usage", ...(upstream.usageProbe?.endpoints ?? [])]));
|
|
364
|
-
if (
|
|
409
|
+
if (endpoints.length === 0) {
|
|
365
410
|
return { ...base, message: `${base.message ?? "Token inspected."} No provider usage probe endpoints are configured.` };
|
|
366
411
|
}
|
|
367
412
|
|
|
@@ -382,17 +427,21 @@ export async function refreshProviderUsageSnapshot(account: StoredRouterAccount,
|
|
|
382
427
|
const url = resolveProbeUrl(upstream.baseUrl, endpoint);
|
|
383
428
|
lastEndpoint = url;
|
|
384
429
|
try {
|
|
430
|
+
const headers: Record<string, string> = {
|
|
431
|
+
Authorization: `Bearer ${account.access}`,
|
|
432
|
+
Originator: "codex_cli_rs",
|
|
433
|
+
originator: "codex_cli_rs",
|
|
434
|
+
"User-Agent": "codex_cli_rs/0.133.0 (Windows; x86_64) pi/oauth-router",
|
|
435
|
+
accept: "application/json",
|
|
436
|
+
};
|
|
437
|
+
if (accountId) {
|
|
438
|
+
headers["ChatGPT-Account-Id"] = accountId;
|
|
439
|
+
headers["chatgpt-account-id"] = accountId;
|
|
440
|
+
}
|
|
441
|
+
|
|
385
442
|
const { response, text } = await fetchJsonWithTimeout(url, {
|
|
386
443
|
method: "GET",
|
|
387
|
-
headers
|
|
388
|
-
Authorization: `Bearer ${account.access}`,
|
|
389
|
-
"ChatGPT-Account-Id": accountId,
|
|
390
|
-
"chatgpt-account-id": accountId,
|
|
391
|
-
Originator: "codex_cli_rs",
|
|
392
|
-
originator: "codex_cli_rs",
|
|
393
|
-
"User-Agent": "codex_cli_rs/0.133.0 (Windows; x86_64) pi/oauth-router",
|
|
394
|
-
accept: "application/json",
|
|
395
|
-
},
|
|
444
|
+
headers,
|
|
396
445
|
}, Math.max(1, remainingMs));
|
|
397
446
|
lastStatus = response.status;
|
|
398
447
|
lastHeaders = collectRateLimitHeaders(response.headers);
|
|
@@ -10,8 +10,8 @@ import {
|
|
|
10
10
|
streamSimpleOpenAICodexResponses,
|
|
11
11
|
streamSimpleOpenAICompletions,
|
|
12
12
|
streamSimpleOpenAIResponses,
|
|
13
|
-
} from "@
|
|
14
|
-
import type { ExtensionAPI } from "@
|
|
13
|
+
} from "@earendil-works/pi-ai";
|
|
14
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
15
15
|
import { loadRouterConfig } from "./config.ts";
|
|
16
16
|
import { refreshAccountCredentials, getApiKeyForAccount, refreshProviderUsageSnapshot } from "./oauth-flow.ts";
|
|
17
17
|
import { RouterAccountStore } from "./oauth-store.ts";
|
|
@@ -43,6 +43,21 @@ function parseRetryAfterMs(value: string | undefined): number | undefined {
|
|
|
43
43
|
return undefined;
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
+
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
if (signal?.aborted) {
|
|
49
|
+
reject(new Error("Request was aborted"));
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const timeout = setTimeout(resolve, ms);
|
|
54
|
+
signal?.addEventListener("abort", () => {
|
|
55
|
+
clearTimeout(timeout);
|
|
56
|
+
reject(new Error("Request was aborted"));
|
|
57
|
+
}, { once: true });
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
46
61
|
function isMeaningfulEvent(event: AssistantMessageEvent): boolean {
|
|
47
62
|
switch (event.type) {
|
|
48
63
|
case "text_delta":
|
|
@@ -88,6 +103,25 @@ function isAbortLike(message?: string, signal?: AbortSignal, stopReason?: string
|
|
|
88
103
|
return lower.includes("abort") || lower.includes("cancelled") || lower.includes("canceled");
|
|
89
104
|
}
|
|
90
105
|
|
|
106
|
+
function isClientNetworkFailure(lower: string): boolean {
|
|
107
|
+
return [
|
|
108
|
+
"codex sse response headers timed out",
|
|
109
|
+
"response headers timed out",
|
|
110
|
+
"headers timeout",
|
|
111
|
+
"headers timed out",
|
|
112
|
+
"fetch failed",
|
|
113
|
+
"network",
|
|
114
|
+
"econnreset",
|
|
115
|
+
"etimedout",
|
|
116
|
+
"enotfound",
|
|
117
|
+
"eai_again",
|
|
118
|
+
"socket hang up",
|
|
119
|
+
"connection reset",
|
|
120
|
+
"connection terminated",
|
|
121
|
+
"und_err_headers_timeout",
|
|
122
|
+
].some((token) => lower.includes(token));
|
|
123
|
+
}
|
|
124
|
+
|
|
91
125
|
function classifyFailure(
|
|
92
126
|
status: number | undefined,
|
|
93
127
|
headers: Record<string, string>,
|
|
@@ -105,13 +139,15 @@ function classifyFailure(
|
|
|
105
139
|
return { kind: "auth", status: status ?? 401, message };
|
|
106
140
|
}
|
|
107
141
|
|
|
108
|
-
if (
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
142
|
+
if (isClientNetworkFailure(lower)) {
|
|
143
|
+
return { kind: "client-network", status, message };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (status !== undefined && status >= 500) {
|
|
147
|
+
return { kind: "transient", status, message };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
if (lower.includes("timeout") || lower.includes("service unavailable") || lower.includes("overloaded")) {
|
|
115
151
|
return { kind: "transient", status: status ?? 500, message };
|
|
116
152
|
}
|
|
117
153
|
|
|
@@ -218,11 +254,35 @@ export class RouterRuntime {
|
|
|
218
254
|
|
|
219
255
|
async refreshUsageSnapshot(id: string) {
|
|
220
256
|
this.reloadConfig();
|
|
221
|
-
|
|
257
|
+
let account = this.accounts.get(id);
|
|
222
258
|
if (!account) throw new Error(`Unknown account: ${id}`);
|
|
223
259
|
const upstream = this.getUpstream(account.upstreamId);
|
|
224
260
|
if (!upstream) throw new Error(`Unknown upstream for account ${id}: ${account.upstreamId}`);
|
|
225
|
-
|
|
261
|
+
|
|
262
|
+
const refreshIfPossible = async (reason: string): Promise<boolean> => {
|
|
263
|
+
if (!account || account.provider === "api-key") return false;
|
|
264
|
+
try {
|
|
265
|
+
account = await refreshAccountCredentials(account);
|
|
266
|
+
this.accounts.update(account);
|
|
267
|
+
this.state.clearHealth(id);
|
|
268
|
+
return true;
|
|
269
|
+
} catch (error) {
|
|
270
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
271
|
+
this.state.markAuthFailure(id, 401, `Unable to refresh OAuth credentials for usage probe (${reason}): ${message}`);
|
|
272
|
+
throw error;
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
|
|
276
|
+
if (account.provider !== "api-key" && account.expires - this.config.tokenRefreshSkewMs <= now()) {
|
|
277
|
+
await refreshIfPossible("expired access token");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
let snapshot = await refreshProviderUsageSnapshot(account, upstream);
|
|
281
|
+
if (account.provider !== "api-key" && (snapshot.status === 401 || snapshot.status === 403)) {
|
|
282
|
+
const refreshed = await refreshIfPossible(`provider quota probe returned HTTP ${snapshot.status}`);
|
|
283
|
+
if (refreshed) snapshot = await refreshProviderUsageSnapshot(account, upstream);
|
|
284
|
+
}
|
|
285
|
+
|
|
226
286
|
this.state.setProviderUsage(id, snapshot);
|
|
227
287
|
return snapshot;
|
|
228
288
|
}
|
|
@@ -283,6 +343,7 @@ export class RouterRuntime {
|
|
|
283
343
|
rateLimitCount: state.rateLimitCount,
|
|
284
344
|
authFailureCount: state.authFailureCount,
|
|
285
345
|
successCount: state.successCount,
|
|
346
|
+
lastError: state.lastError,
|
|
286
347
|
expires: account.expires,
|
|
287
348
|
};
|
|
288
349
|
});
|
|
@@ -378,12 +439,49 @@ export class RouterRuntime {
|
|
|
378
439
|
failure.message,
|
|
379
440
|
);
|
|
380
441
|
break;
|
|
442
|
+
case "client-network":
|
|
443
|
+
if (this.config.clientNetworkPenaltyMs > 0) {
|
|
444
|
+
this.state.markTransientFailure(accountId, this.config.clientNetworkPenaltyMs, failure.status ?? 500, failure.message);
|
|
445
|
+
} else {
|
|
446
|
+
this.state.recordClientNetworkFailure(accountId, failure.status, failure.message);
|
|
447
|
+
}
|
|
448
|
+
break;
|
|
381
449
|
case "fatal":
|
|
382
450
|
this.state.markTransientFailure(accountId, this.config.transientPenaltyMs, failure.status ?? 500, failure.message);
|
|
383
451
|
break;
|
|
384
452
|
}
|
|
385
453
|
}
|
|
386
454
|
|
|
455
|
+
private getClientNetworkRetryDelayMs(retryIndex: number): number {
|
|
456
|
+
const baseDelayMs = Math.max(0, this.config.clientNetworkRetryBaseDelayMs);
|
|
457
|
+
const uncappedDelayMs = baseDelayMs * 2 ** retryIndex;
|
|
458
|
+
const maxDelayMs = this.config.clientNetworkMaxRetryDelayMs;
|
|
459
|
+
return maxDelayMs > 0 ? Math.min(uncappedDelayMs, maxDelayMs) : uncappedDelayMs;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
private async tryAccountWithClientNetworkRetries(
|
|
463
|
+
selection: DelegateSelection,
|
|
464
|
+
outerStream: ReturnType<typeof createAssistantMessageEventStream>,
|
|
465
|
+
context: Context,
|
|
466
|
+
modelId: string,
|
|
467
|
+
options?: SimpleStreamOptions,
|
|
468
|
+
): Promise<{ completed: boolean; emittedMeaningfulOutput: boolean; failure?: FailureClassification }> {
|
|
469
|
+
for (let retryIndex = 0; ; retryIndex += 1) {
|
|
470
|
+
const result = await this.trySingleAccount(selection, outerStream, context, options);
|
|
471
|
+
const canRetry =
|
|
472
|
+
!result.completed &&
|
|
473
|
+
!result.emittedMeaningfulOutput &&
|
|
474
|
+
result.failure?.kind === "client-network" &&
|
|
475
|
+
retryIndex < this.config.clientNetworkMaxRetries;
|
|
476
|
+
|
|
477
|
+
if (!canRetry) return result;
|
|
478
|
+
|
|
479
|
+
const delayMs = this.getClientNetworkRetryDelayMs(retryIndex);
|
|
480
|
+
await sleep(delayMs, options?.signal);
|
|
481
|
+
this.state.markAttempt(selection.account.id, modelId);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
387
485
|
private async trySingleAccount(
|
|
388
486
|
selection: DelegateSelection,
|
|
389
487
|
outerStream: ReturnType<typeof createAssistantMessageEventStream>,
|
|
@@ -402,6 +500,8 @@ export class RouterRuntime {
|
|
|
402
500
|
...(options?.headers ?? {}),
|
|
403
501
|
...selection.headers,
|
|
404
502
|
},
|
|
503
|
+
maxRetries: options?.maxRetries ?? this.config.upstreamMaxRetries,
|
|
504
|
+
maxRetryDelayMs: options?.maxRetryDelayMs ?? this.config.upstreamMaxRetryDelayMs,
|
|
405
505
|
onResponse: async (response, responseModel) => {
|
|
406
506
|
responseStatus = response.status;
|
|
407
507
|
responseHeaders = response.headers;
|
|
@@ -520,7 +620,7 @@ export class RouterRuntime {
|
|
|
520
620
|
continue;
|
|
521
621
|
}
|
|
522
622
|
|
|
523
|
-
const result = await this.
|
|
623
|
+
const result = await this.tryAccountWithClientNetworkRetries(selection, outer, context, model.id, options);
|
|
524
624
|
if (result.completed) {
|
|
525
625
|
outer.end();
|
|
526
626
|
return;
|
|
@@ -9,7 +9,7 @@ import type {
|
|
|
9
9
|
RouterUsageWindowSummary,
|
|
10
10
|
RoutingPolicyName,
|
|
11
11
|
} from "./types.ts";
|
|
12
|
-
import type { AssistantMessage } from "@
|
|
12
|
+
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
|
13
13
|
|
|
14
14
|
const FIVE_HOURS_MS = 5 * 60 * 60 * 1000;
|
|
15
15
|
const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
|
|
@@ -335,6 +335,17 @@ export class RouterStateStore {
|
|
|
335
335
|
});
|
|
336
336
|
}
|
|
337
337
|
|
|
338
|
+
recordClientNetworkFailure(accountId: string, status: number | undefined, message = "Client/network transport failure") {
|
|
339
|
+
this.mutate(() => {
|
|
340
|
+
const account = this.ensureAccountInMemory(accountId);
|
|
341
|
+
account.lastStatus = status;
|
|
342
|
+
account.lastError = message;
|
|
343
|
+
account.failures += 1;
|
|
344
|
+
account.penaltyUntil = undefined;
|
|
345
|
+
account.cooldownUntil = undefined;
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
|
|
338
349
|
clearHealth(accountId: string) {
|
|
339
350
|
this.mutate(() => {
|
|
340
351
|
const account = this.ensureAccountInMemory(accountId);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Api, AssistantMessage, Context, Model, SimpleStreamOptions } from "@
|
|
1
|
+
import type { Api, AssistantMessage, Context, Model, SimpleStreamOptions } from "@earendil-works/pi-ai";
|
|
2
2
|
|
|
3
3
|
export type RouterUpstreamApi = "openai-completions" | "openai-responses" | "openai-codex-responses";
|
|
4
4
|
export type RouterAuthMode = "oauth" | "api-key";
|
|
@@ -52,6 +52,18 @@ export interface RouterConfig {
|
|
|
52
52
|
tokenRefreshSkewMs: number;
|
|
53
53
|
rateLimitCooldownMs: number;
|
|
54
54
|
transientPenaltyMs: number;
|
|
55
|
+
/** Default retry attempts passed to upstream providers when the caller does not specify one. */
|
|
56
|
+
upstreamMaxRetries: number;
|
|
57
|
+
/** Maximum provider-request retry delay when supported by the upstream provider. */
|
|
58
|
+
upstreamMaxRetryDelayMs: number;
|
|
59
|
+
/** Router-level same-account retries for local/client transport failures before account failover. */
|
|
60
|
+
clientNetworkMaxRetries: number;
|
|
61
|
+
/** First router-level retry delay for local/client transport failures. Later retries double this delay. */
|
|
62
|
+
clientNetworkRetryBaseDelayMs: number;
|
|
63
|
+
/** Maximum router-level retry delay for local/client transport failures. */
|
|
64
|
+
clientNetworkMaxRetryDelayMs: number;
|
|
65
|
+
/** Penalty for local/client transport failures. Zero records the failure without cooling down the account. */
|
|
66
|
+
clientNetworkPenaltyMs: number;
|
|
55
67
|
models: RouterModelConfig[];
|
|
56
68
|
upstreams: RouterUpstreamConfig[];
|
|
57
69
|
}
|
|
@@ -180,6 +192,7 @@ export interface RouterStatusRow {
|
|
|
180
192
|
rateLimitCount: number;
|
|
181
193
|
authFailureCount: number;
|
|
182
194
|
successCount: number;
|
|
195
|
+
lastError?: string;
|
|
183
196
|
expires: number;
|
|
184
197
|
}
|
|
185
198
|
|
|
@@ -200,7 +213,7 @@ export interface DelegateSelection {
|
|
|
200
213
|
}
|
|
201
214
|
|
|
202
215
|
export interface FailureClassification {
|
|
203
|
-
kind: "rate-limit" | "auth" | "transient" | "fatal";
|
|
216
|
+
kind: "rate-limit" | "auth" | "transient" | "client-network" | "fatal";
|
|
204
217
|
status?: number;
|
|
205
218
|
retryAfterMs?: number;
|
|
206
219
|
message: string;
|