pi-ccswitch-auto-switch 0.3.10 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/README.zh-CN.md +8 -4
- package/candidates.ts +7 -3
- package/classify.ts +4 -1
- package/health.ts +179 -83
- package/index.ts +101 -40
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -13,7 +13,7 @@ The extension observes real Pi requests, records sanitized health signals, and
|
|
|
13
13
|
- Endpoint circuit breakers for DNS, connection, server, and streaming failures.
|
|
14
14
|
- Endpoint platform isolation: when several models on the same endpoint (same BaseURL/provider) fail within one round, the whole endpoint is isolated so sibling models from the same platform are not tried one by one.
|
|
15
15
|
- Model-only isolation for missing models and incompatible parameters.
|
|
16
|
-
- Learns model-family content-policy constraints from real failures (`glm-4.5` and `GLM-4.6`, for example, are both `glm`) and avoids the entire family during the current content-policy failover chain. Evidence is
|
|
16
|
+
- Learns model-family content-policy constraints from real failures (`glm-4.5` and `GLM-4.6`, for example, are both `glm`) and avoids the entire family during the current content-policy failover chain. Evidence lives only in the owning extension instance, is never written to shared health state, and is cleared at that instance’s `session_start`, because a new session may handle tasks that are not content-sensitive, so the family becomes eligible again until this session observes its own content-policy rejection.
|
|
17
17
|
- Applies hard health, failure-domain, policy-family, input-modality, and context filters before ranking candidates by provider diversity, model equivalence, capability compatibility, and historical reliability.
|
|
18
18
|
- Independent-provider semantics: a copied vendor (e.g. `my-provider-copy`) shares the BaseURL but is a distinct provider with its own endpoint key, so isolating one never blocks the other.
|
|
19
19
|
- 90-second first-response and 120-second streaming-idle watchdogs.
|
|
@@ -39,6 +39,7 @@ All knobs below are optional and have sensible defaults:
|
|
|
39
39
|
| --- | --- | --- | --- |
|
|
40
40
|
| `PI_CODING_AGENT_DIR` | Extension state | `~/.pi/agent` | Where health state, logs, and failure reports are stored. Only relevant if you relocated your Pi agent directory; the extension follows Pi's own convention. |
|
|
41
41
|
| `PI_BIN` | Headless runner only | `pi` on `PATH` | Must point to the real `pi` executable. Legacy self-references to `pi-ccswitch-run` or `ccswitch-run` are ignored with a warning and fall back to `pi`; other invalid commands remain configuration errors. |
|
|
42
|
+
| `CCSWITCH_ROUND_LIMIT_MS` | Failover | `480000` (8 minutes) | Time limit for one failover round, in milliseconds. The window restarts on each successful switch; when exceeded, switching stops and the report lists every tried model with its last error. |
|
|
42
43
|
| Model scope (`/model` etc.) | Failover candidates | Full registry | When Pi has an active model scope, failover only considers models inside that scope; otherwise the full registry is used. The status bar shows which source is active. |
|
|
43
44
|
| `baseUrl` metadata | Endpoint isolation | provider key | Endpoint-level platform isolation groups models by `baseUrl` (provided by CC Switch `3.20+`). Without it, isolation degrades to provider-level grouping, which still works. |
|
|
44
45
|
|
|
@@ -159,6 +160,7 @@ CCSwitch therefore prechecks **before the request is sent**, instead of waiting
|
|
|
159
160
|
|
|
160
161
|
- When user input carries images (`input` event with `images`), CCSwitch immediately switches to a healthy multimodal candidate (`input` explicitly includes `image`) before the request is processed, so the images are preserved;
|
|
161
162
|
- When a tool execution returns images (e.g. the `read` tool loading an image file, whose result content includes `image` parts), CCSwitch also switches to a multimodal model before the next LLM call, so the tool-result images are not stripped;
|
|
163
|
+
- Image requirements remain active for the entire failover round, including images returned by tools during retry verification and later fallback attempts;
|
|
162
164
|
- Only models that **explicitly** support images (`input` includes `image`) are considered; models with missing metadata are never assumed to support images;
|
|
163
165
|
- The switch reason is recorded as `modality` and counted in both the session and lifetime switch counters;
|
|
164
166
|
- If no multimodal candidate is available, CCSwitch notifies the user and keeps the current model (Pi will still strip the image and add its notice).
|
|
@@ -167,7 +169,9 @@ The modality precheck applies only in interactive TUI/RPC sessions (the same sco
|
|
|
167
169
|
|
|
168
170
|
## Data and privacy
|
|
169
171
|
|
|
170
|
-
Health state is stored in Pi's agent directory as `ccswitch-auto-switch-state.json`. The extension stores counters, timestamps, cooldowns,
|
|
172
|
+
Health state is stored in Pi's agent directory as `ccswitch-auto-switch-state.json`. The extension stores counters, timestamps, cooldowns, and redacted/truncated error summaries. Model-family policy constraints stay in session memory. It does not access credentials, authorization headers, CC Switch's database, or Pi's `auth.json`.
|
|
173
|
+
|
|
174
|
+
State schema 3 loads schema 2 health records and preserves their existing totals as a shared baseline. Per-writer counters merge independent increments without counting repeated merges twice. Older releases cannot read schema 3 and cannot be made safe by code in a newer process. **Do not run schema-2 and schema-3 releases together against the same directory.** Before upgrading: (1) stop every old Pi process and scheduled runner using the same `PI_CODING_AGENT_DIR`; (2) back up `ccswitch-auto-switch-state.json` while they are stopped; (3) upgrade every installation that uses that directory; (4) restart only the upgraded processes. A running current-version store refuses writes when it encounters an unsupported schema or observes a downgrade from 3 to 2, preserves the file, and logs the reason in `ccswitch-auto-switch.log`. This cannot undo damage already caused by old code; the stop/upgrade/restart sequence is required. Manual enable/disable operations use monotonic local timestamps; merge picks the newer operation, with disable winning an exact timestamp tie (including conflicting legacy values without timestamps). Reset barriers prevent stale snapshots from restoring cleared records. Counter metadata grows with the number of writer instances that record failures or switches.
|
|
171
175
|
|
|
172
176
|
## Development
|
|
173
177
|
|
|
@@ -177,7 +181,7 @@ npm run typecheck
|
|
|
177
181
|
npm test
|
|
178
182
|
```
|
|
179
183
|
|
|
180
|
-
Tests use Node's built-in test runner and cover the failure matrix, family-level policy avoidance, provider-first selection, input compatibility, cooldowns, state persistence, and Windows-compatible paths.
|
|
184
|
+
TypeScript is an explicit development dependency. `npm test` enables Node’s native TypeScript stripping and runs all `health`, `integration`, `logic`, and `runner` suites on Node 22.19+. Tests use Node's built-in test runner and cover the failure matrix, family-level policy avoidance, provider-first selection, input compatibility, cooldowns, state persistence, and Windows-compatible paths.
|
|
181
185
|
|
|
182
186
|
## License
|
|
183
187
|
|
package/README.zh-CN.md
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
- 对认证、额度、账单和限流实施 Provider 优先熔断。
|
|
13
13
|
- 对 DNS、连接、服务端和流中断实施端点熔断。
|
|
14
14
|
- 模型不存在或参数不兼容时仅隔离该模型。
|
|
15
|
-
- 内容审查失败会学习模型系列约束(例如 `glm-4.5`、`GLM-4.6` 同属 `glm
|
|
15
|
+
- 内容审查失败会学习模型系列约束(例如 `glm-4.5`、`GLM-4.6` 同属 `glm`),在本轮内容审查故障转移中避开整个受限系列。该约束仅保存在当前扩展实例的内存中,不写入共享健康状态;当前实例的 `session_start` 只清除自身约束,因为新 session 处理的任务不一定涉及审查内容,模型系列恢复可用,直到本 session 再次观察到内容审查拒绝。
|
|
16
16
|
- 候选模型先经过健康、故障域、审查系列、输入模态和上下文窗口硬过滤,再按跨 Provider、模型等价性、能力兼容和历史稳定性排序。
|
|
17
17
|
- 首响应 90 秒、流式停滞 120 秒看门狗。
|
|
18
18
|
- 指数退避、`Retry-After` 支持和跨进程 half-open 单探针租约。
|
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
| --- | --- | --- | --- |
|
|
38
38
|
| `PI_CODING_AGENT_DIR` | 扩展状态 | `~/.pi/agent` | 健康状态、日志与失败报告的存放目录;仅在你迁移了 Pi agent 目录时需要,插件跟随 Pi 自身约定。 |
|
|
39
39
|
| `PI_BIN` | 仅 headless runner | `PATH` 中的 `pi` | 必须指向真实 `pi` 可执行文件。历史自引用值 `pi-ccswitch-run` 或 `ccswitch-run` 会警告并降级到 `pi`;其他非法命令仍按配置错误处理。 |
|
|
40
|
+
| `CCSWITCH_ROUND_LIMIT_MS` | 故障转移 | `480000`(8 分钟) | 本轮故障转移的时间限制(毫秒)。窗口从本轮开始或最后一次成功切换起算;超过后停止切换并报告已试模型及最后错误。 |
|
|
40
41
|
| 模型 scope(`/model` 等) | 故障转移候选 | 全量注册表 | Pi 启用了 model scope 时,只在 scope 内模型间切换;否则使用全量注册表。状态栏会显示当前数据源。 |
|
|
41
42
|
| `baseUrl` 元数据 | 端点隔离 | provider key | 端点级平台隔离按 `baseUrl` 分组模型(由 CC Switch `3.20+` 提供);缺失时退化为 provider 级分组,仍然可用。 |
|
|
42
43
|
|
|
@@ -137,7 +138,7 @@ CCS v0.3.5 ✓70/131 · ⏳61 · ⛔0 · 🔄3 · provider/model-id
|
|
|
137
138
|
| 内容审查/敏感拦截 | 当前模型 + 模型系列约束 | 模型冷却 2 分钟;系列约束按 session 隔离(`session_start` 时清除) |
|
|
138
139
|
| 上下文溢出 | 仅本轮 | 无 |
|
|
139
140
|
|
|
140
|
-
|
|
141
|
+
冷却时间会指数增长但有上限。发生内容审查时,插件只在审查故障转移链中避开已标记系列,普通限流、网络或模型配置故障仍可选择这些模型。上下文溢出时只会选择上下文窗口更大的模型;带图片的请求只选择显式声明支持 image 的模型。无法归类的异常按单模型故障处理并正常切换。用户主动取消是唯一不会触发故障转移的异常终止;看门狗取消会记录为超时。
|
|
141
142
|
|
|
142
143
|
### 模态预检(OCR / 图片请求自动切换多模态模型)
|
|
143
144
|
|
|
@@ -148,6 +149,7 @@ Pi 在把请求发给 Provider 时,会根据当前模型的 `input` 能力静
|
|
|
148
149
|
- **用户输入带图片时**(`input` 事件含 `images`),如果当前模型不支持图片,立即切换到健康的多模态候选(`input` 明确含 `image`)再处理,图片被保留;
|
|
149
150
|
- **工具执行返回图片时**(如 `read` 工具读取图片文件,工具结果含 `image` content),同样在下一轮 LLM 调用前切到多模态模型,确保图片不被剥除;
|
|
150
151
|
- 候选只选**明确声明**支持图片的模型(`input` 含 `image`);元数据缺失的模型不冒险选择;
|
|
152
|
+
- 图片能力要求贯穿整个故障转移轮次,包括重试验证阶段工具返回图片以及此后的再次切换;
|
|
151
153
|
- 切换原因记为 `modality`,计入本 session 切换数与累计切换;
|
|
152
154
|
- 如果没有可用的多模态候选,则通知用户并保持原模型继续(Pi 会照常剥图并附带提示)。
|
|
153
155
|
|
|
@@ -155,17 +157,19 @@ Pi 在把请求发给 Provider 时,会根据当前模型的 `input` 能力静
|
|
|
155
157
|
|
|
156
158
|
## 数据与隐私
|
|
157
159
|
|
|
158
|
-
健康状态保存在 Pi agent 目录的 `ccswitch-auto-switch-state.json
|
|
160
|
+
健康状态保存在 Pi agent 目录的 `ccswitch-auto-switch-state.json`。其中只有计数、时间、冷却信息和脱敏/截断的错误摘要;模型系列审查约束只保存在 session 内存中;插件不会访问凭据、Authorization 请求头、CC Switch 数据库或 Pi 的 `auth.json`。
|
|
159
161
|
|
|
160
162
|
## 开发
|
|
161
163
|
|
|
164
|
+
状态 schema 3 兼容加载 schema 2 的健康记录,将原有累计数保留为共同基线;每个写入实例独立计数,合并时不丢增量、不重复累计。旧版本无法读取 schema 3,新进程中的代码无法修复仍在运行的旧代码。**禁止 schema-2 与 schema-3 版本共用同一目录混跑。**升级时必须依次:① 退出所有使用同一 `PI_CODING_AGENT_DIR` 的旧 Pi 进程并停止相关定时 runner;② 在进程停止后备份 `ccswitch-auto-switch-state.json`;③ 升级使用该目录的所有安装;④ 仅启动升级后的进程。当前版本遇到不支持的 schema,或已观察到 schema 3 后又读到 schema 2 时,会拒绝写入、保留原文件,并在 `ccswitch-auto-switch.log` 记录原因。这不能撤销旧代码已经造成的破坏,因此必须执行上述停机升级步骤。人工启用/禁用按本地单调时间戳选择较新操作;时间戳完全相同时保守保留禁用(旧记录缺字段时也如此)。reset 标记阻止旧快照恢复已清除的记录。计数元数据随实际记录失败或切换的实例数增长。
|
|
165
|
+
|
|
162
166
|
```bash
|
|
163
167
|
npm install
|
|
164
168
|
npm run typecheck
|
|
165
169
|
npm test
|
|
166
170
|
```
|
|
167
171
|
|
|
168
|
-
|
|
172
|
+
TypeScript 已显式列为开发依赖。`npm test` 开启 Node 原生 TypeScript 类型剥离,在 Node 22.19+ 上运行 `health`、`integration`、`logic`、`runner` 全部测试。测试使用 Node 内置测试运行器,覆盖失败分类矩阵、系列级审查避让、Provider 优先选择、输入兼容、冷却、状态持久化以及 Windows 路径。
|
|
169
173
|
|
|
170
174
|
## 许可证
|
|
171
175
|
|
package/candidates.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { FailureClass, ModelRef, ScopedModel } from './types.ts'
|
|
2
|
-
import { endpointKey, modelKey, type HealthState } from './health.ts'
|
|
2
|
+
import { endpointKey, modelKey, platformKey, type HealthState } from './health.ts'
|
|
3
3
|
|
|
4
4
|
export interface CandidateOptions {
|
|
5
5
|
current?: ModelRef
|
|
@@ -90,7 +90,9 @@ export function chooseCandidate(models: ModelRef[], options: CandidateOptions):
|
|
|
90
90
|
const current = options.current
|
|
91
91
|
const candidates = models.filter(model => {
|
|
92
92
|
if (options.tried.has(modelKey(model)) || blocked(model, options.health)) return false
|
|
93
|
-
|
|
93
|
+
// avoidEndpoints 同时支持 endpointKey(含 provider,旧语义)与 platformKey(仅 BaseURL)两种 key:
|
|
94
|
+
// 本轮隔离按平台(platformKey)归组,同平台的 provider 副本一并跳过
|
|
95
|
+
if (options.avoidEndpoints?.has(endpointKey(model)) || options.avoidEndpoints?.has(platformKey(model))) return false
|
|
94
96
|
if (options.avoidFamilies?.has(modelFamily(model))) return false
|
|
95
97
|
if (!supportsRequiredInputs(model, options.requiredInputs)) return false
|
|
96
98
|
if (options.failureKind === 'context_overflow' && current && (model.contextWindow ?? 0) <= (current.contextWindow ?? 0)) return false
|
|
@@ -129,7 +131,9 @@ export function modelFamily(model: Pick<ModelRef, 'id' | 'name'>): string {
|
|
|
129
131
|
}
|
|
130
132
|
|
|
131
133
|
function supportsRequiredInputs(candidate: ModelRef, required: readonly string[] | undefined): boolean {
|
|
132
|
-
if (!required?.length
|
|
134
|
+
if (!required?.length) return true
|
|
135
|
+
if (required.includes('image') && !candidate.input?.includes('image')) return false
|
|
136
|
+
if (!candidate.input?.length) return true
|
|
133
137
|
return required.every(input => candidate.input!.includes(input))
|
|
134
138
|
}
|
|
135
139
|
|
package/classify.ts
CHANGED
|
@@ -8,7 +8,9 @@ export interface Classification {
|
|
|
8
8
|
}
|
|
9
9
|
|
|
10
10
|
const content = /content[ _-]?(?:filter|policy|blocked|moderation)|sensitive|guardrail|policy[_ -]?violation|responsibleai|safety[_ -]?(?:policy|filter|violation)|security[_ -]?policy|moderation[_ -]?(?:blocked|failed)|blocked by.{0,20}(?:policy|safety|moderation)|data[_ -]?inspection[_ -]?failed|内容(?:审查|审核|安全|违规|被拦截)|安全审核|安全(?:策略|风控).{0,12}(?:阻断|拦截|拒绝)|审核(?:未通过|不通过|失败)|敏感(?:内容|词|信息)?|政治敏感|涉政|不合规|违反.{0,12}(?:安全|政策|规定)/i
|
|
11
|
-
|
|
11
|
+
// 配额/余额类失败(scope provider)。含中文供应商报错:用户积分不足、余额不足、欠费等,
|
|
12
|
+
// 这些通常不携带英文关键词,漏分类会落入 unknown(仅 2min 模型级冷却),导致死 provider 每轮重试。
|
|
13
|
+
const quota = /quota|billing|insufficient.?balance|out of budget|usage.?limit|credit.?balance|balance.{0,12}(?:exhausted|insufficient|depleted)|credits|monthly.?limit|积分不足|余额不足|欠费|请充值|额度.{0,6}(?:用完|耗尽|不足|已用)|无可用额度/i
|
|
12
14
|
const context = /context.?window|context.?length|too many tokens|prompt is too long|input is too long|token limit/i
|
|
13
15
|
const transport = /dns|enotfound|eai_again|econn(?:reset|refused|aborted)|epipe|socket|connection|network|fetch failed|tls|certificate|timed? ?out|timeout|stream ended|terminated|websocket|und_err/i
|
|
14
16
|
const overloaded = /overload|server busy|service unavailable|temporarily unavailable|capacity|upstream|bad gateway/i
|
|
@@ -25,6 +27,7 @@ export function classifyFailure(observation: FailureObservation): Classification
|
|
|
25
27
|
if (status === 413) return { kind: 'context_overflow', roundOnly: true }
|
|
26
28
|
if (status === 429) return { kind: 'rate_limit', scope: 'provider', roundOnly: false, retryAfterMs: observation.retryAfterMs }
|
|
27
29
|
if (status === 402) return { kind: 'quota', scope: 'provider', roundOnly: false }
|
|
30
|
+
if (status === 403 && quota.test(text)) return { kind: 'quota', scope: 'provider', roundOnly: false }
|
|
28
31
|
if (status === 401 || status === 403) return { kind: 'auth', scope: 'provider', roundOnly: false }
|
|
29
32
|
if (quota.test(text)) return { kind: 'quota', scope: 'provider', roundOnly: false }
|
|
30
33
|
if (status === 408) return { kind: 'timeout', scope: 'endpoint', roundOnly: false }
|
package/health.ts
CHANGED
|
@@ -15,12 +15,15 @@ const MAX_LOG_BYTES = 512 * 1024
|
|
|
15
15
|
export interface HealthRecord {
|
|
16
16
|
consecutiveFailures: number
|
|
17
17
|
totalFailures: number
|
|
18
|
+
/** Per-writer grow-only counters; legacy totals form one shared baseline. */
|
|
19
|
+
failureCounts?: Record<string, number>
|
|
18
20
|
lastFailureAt?: number
|
|
19
21
|
lastSuccessAt?: number
|
|
20
22
|
cooldownUntil?: number
|
|
21
23
|
lastClass?: FailureClass
|
|
22
24
|
lastError?: string
|
|
23
25
|
disabled?: boolean
|
|
26
|
+
disabledUpdatedAt?: number
|
|
24
27
|
leaseUntil?: number
|
|
25
28
|
}
|
|
26
29
|
|
|
@@ -33,18 +36,22 @@ export interface ContentPolicyConstraint {
|
|
|
33
36
|
}
|
|
34
37
|
|
|
35
38
|
export interface HealthState {
|
|
36
|
-
schemaVersion: 2
|
|
39
|
+
schemaVersion: 2 | 3
|
|
37
40
|
updatedAt: number
|
|
41
|
+
/** Reset barriers prevent older snapshots from resurrecting cleared history. */
|
|
42
|
+
resetAt?: number
|
|
43
|
+
modelResetAt?: Record<string, number>
|
|
38
44
|
models: Record<string, HealthRecord>
|
|
39
45
|
providers: Record<string, HealthRecord>
|
|
40
46
|
endpoints: Record<string, HealthRecord>
|
|
41
47
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
48
|
+
* Legacy schema-2 field: ignored on load and removed on the next write.
|
|
49
|
+
* Active policy constraints live only in the owning HealthStore instance.
|
|
44
50
|
*/
|
|
45
51
|
contentPolicyFamilies?: Record<string, ContentPolicyConstraint>
|
|
46
52
|
/** 累计成功切换次数(跨 session 持久化,用于衡量扩展有效程度) */
|
|
47
53
|
switches?: number
|
|
54
|
+
switchCounts?: Record<string, number>
|
|
48
55
|
/** 最近成功切换日志(有限条,环形保留) */
|
|
49
56
|
switchLog?: Array<{
|
|
50
57
|
at: number
|
|
@@ -67,6 +74,16 @@ export function endpointKey(model: ModelRef): string {
|
|
|
67
74
|
return createHash('sha256').update(`${model.provider}\0${source}`).digest('hex').slice(0, 20)
|
|
68
75
|
}
|
|
69
76
|
|
|
77
|
+
/**
|
|
78
|
+
* 同平台标识:仅按 BaseURL 归组(不含 provider 名)。b-ai / b-ai-copy / b-ai-copy-copy 这类
|
|
79
|
+
* 同一上游平台的多个 provider 副本会得到相同 key,用于本轮内同平台隔离。
|
|
80
|
+
* 注意:与 endpointKey(含 provider,用于跨轮健康台账)刻意不同——不同的 API key
|
|
81
|
+
* 有独立的配额/限流,跨轮冷却不应互相波及。
|
|
82
|
+
*/
|
|
83
|
+
export function platformKey(model: ModelRef): string {
|
|
84
|
+
return model.baseUrl ? safeEndpoint(model.baseUrl) : `provider:${model.provider}`
|
|
85
|
+
}
|
|
86
|
+
|
|
70
87
|
function safeEndpoint(value: string): string {
|
|
71
88
|
try {
|
|
72
89
|
const url = new URL(value)
|
|
@@ -77,7 +94,7 @@ function safeEndpoint(value: string): string {
|
|
|
77
94
|
}
|
|
78
95
|
|
|
79
96
|
function blank(): HealthState {
|
|
80
|
-
return { schemaVersion:
|
|
97
|
+
return { schemaVersion: 3, updatedAt: Date.now(), models: {}, providers: {}, endpoints: {}, switches: 0, switchLog: [] }
|
|
81
98
|
}
|
|
82
99
|
|
|
83
100
|
function redact(text: string | undefined): string | undefined {
|
|
@@ -103,30 +120,29 @@ export class HealthStore {
|
|
|
103
120
|
readonly dir: string
|
|
104
121
|
private state: HealthState = blank()
|
|
105
122
|
private dirty = false
|
|
106
|
-
private
|
|
107
|
-
private
|
|
108
|
-
|
|
109
|
-
private
|
|
123
|
+
private readonly writer = randomUUID()
|
|
124
|
+
private manualClock = 0
|
|
125
|
+
private writeBlocked = false
|
|
126
|
+
private observedSchema3 = false
|
|
127
|
+
private constraints: Record<string, ContentPolicyConstraint> = {}
|
|
110
128
|
|
|
111
129
|
constructor(dir = agentDir()) { this.dir = dir }
|
|
112
130
|
get file(): string { return join(this.dir, STATE_FILE) }
|
|
113
131
|
get snapshot(): HealthState { return structuredClone(this.state) }
|
|
132
|
+
get policyConstraints(): Record<string, ContentPolicyConstraint> { return structuredClone(this.constraints) }
|
|
114
133
|
|
|
115
134
|
async load(): Promise<void> {
|
|
116
135
|
await mkdir(this.dir, { recursive: true })
|
|
117
136
|
try {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
this.state = blank()
|
|
128
|
-
this.dirty = true
|
|
129
|
-
await this.flush()
|
|
137
|
+
this.state = await this.readDisk()
|
|
138
|
+
this.writeBlocked = false
|
|
139
|
+
delete this.state.contentPolicyFamilies
|
|
140
|
+
if (!existsSync(this.file)) await this.migrateLegacy()
|
|
141
|
+
} catch (error) {
|
|
142
|
+
// Unknown/newer formats are not corruption. Keep the original in place
|
|
143
|
+
// and fail closed until an explicit successful reload repairs the state.
|
|
144
|
+
this.writeBlocked = true
|
|
145
|
+
await this.log(`health state load refused: ${String(error)}`)
|
|
130
146
|
}
|
|
131
147
|
}
|
|
132
148
|
|
|
@@ -153,6 +169,7 @@ export class HealthStore {
|
|
|
153
169
|
const failures = previous.consecutiveFailures + 1
|
|
154
170
|
bucket[key] = {
|
|
155
171
|
...previous, consecutiveFailures: failures, totalFailures: previous.totalFailures + 1,
|
|
172
|
+
failureCounts: incrementCounts(previous.failureCounts, previous.totalFailures, this.writer),
|
|
156
173
|
lastFailureAt: Date.now(), cooldownUntil: Date.now() + cooldownMs(kind, failures, retryAfterMs),
|
|
157
174
|
lastClass: kind, lastError: redact(message), leaseUntil: undefined,
|
|
158
175
|
}
|
|
@@ -165,7 +182,7 @@ export class HealthStore {
|
|
|
165
182
|
* 本 session 内再次观察到同系列审查会增加证据计数并刷新时间戳。
|
|
166
183
|
*/
|
|
167
184
|
recordContentPolicyConstraint(family: string, model: ModelRef, message?: string): void {
|
|
168
|
-
const constraints = this.
|
|
185
|
+
const constraints = this.constraints
|
|
169
186
|
const previous = constraints[family]
|
|
170
187
|
const now = Date.now()
|
|
171
188
|
constraints[family] = {
|
|
@@ -175,7 +192,6 @@ export class HealthStore {
|
|
|
175
192
|
lastModel: modelKey(model),
|
|
176
193
|
lastError: redact(message),
|
|
177
194
|
}
|
|
178
|
-
this.touch()
|
|
179
195
|
}
|
|
180
196
|
|
|
181
197
|
recordSuccess(model: ModelRef): void {
|
|
@@ -190,6 +206,7 @@ export class HealthStore {
|
|
|
190
206
|
* 用于衡量扩展有效程度,并在 /ccswitch status 面板展示。
|
|
191
207
|
*/
|
|
192
208
|
recordSwitch(from: string, to: string, reason?: string): void {
|
|
209
|
+
this.state.switchCounts = incrementCounts(this.state.switchCounts, this.state.switches ?? 0, this.writer)
|
|
193
210
|
this.state.switches = (this.state.switches ?? 0) + 1
|
|
194
211
|
const log = this.state.switchLog ?? []
|
|
195
212
|
log.push({ at: Date.now(), from, to, reason })
|
|
@@ -205,35 +222,53 @@ export class HealthStore {
|
|
|
205
222
|
|
|
206
223
|
disable(key: string, disabled: boolean): void {
|
|
207
224
|
const old = this.state.models[key] ?? { consecutiveFailures: 0, totalFailures: 0 }
|
|
208
|
-
this.state.models[key] = { ...old, disabled }
|
|
225
|
+
this.state.models[key] = { ...old, disabled, disabledUpdatedAt: this.manualVersion() }
|
|
209
226
|
this.touch()
|
|
210
227
|
}
|
|
211
228
|
|
|
212
|
-
reactivate(model: ModelRef):
|
|
213
|
-
|
|
214
|
-
for (const [scope, id] of [['model', key], ['provider', model.provider], ['endpoint', endpointKey(model)]] as const) {
|
|
215
|
-
const old = this.bucket(scope)[id]
|
|
216
|
-
if (old) this.bucket(scope)[id] = { ...old, disabled: false, cooldownUntil: undefined, leaseUntil: undefined, lastSuccessAt: Date.now(), consecutiveFailures: 0 }
|
|
217
|
-
}
|
|
218
|
-
this.touch()
|
|
229
|
+
async reactivate(model: ModelRef): Promise<boolean> {
|
|
230
|
+
return this.reactivateAtomically(model)
|
|
219
231
|
}
|
|
220
232
|
|
|
221
|
-
reactivateAll():
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
233
|
+
async reactivateAll(): Promise<boolean> {
|
|
234
|
+
return this.reactivateAtomically()
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
private async reactivateAtomically(model?: ModelRef): Promise<boolean> {
|
|
238
|
+
try {
|
|
239
|
+
if (this.writeBlocked) throw new Error('health state writes are blocked; reload a supported state first')
|
|
240
|
+
return await this.withLock(async () => {
|
|
241
|
+
this.state = mergeState(await this.readDisk(), this.state)
|
|
242
|
+
const disabledUpdatedAt = this.manualVersion()
|
|
243
|
+
const targets: Array<[HealthScope, string]> = model
|
|
244
|
+
? [['model', modelKey(model)], ['provider', model.provider], ['endpoint', endpointKey(model)]]
|
|
245
|
+
: (['model', 'provider', 'endpoint'] as const).flatMap(scope => Object.keys(this.bucket(scope)).map(key => [scope, key] as [HealthScope, string]))
|
|
246
|
+
for (const [scope, key] of targets) {
|
|
247
|
+
const old = this.bucket(scope)[key]
|
|
248
|
+
if (!old) continue
|
|
249
|
+
this.bucket(scope)[key] = { ...old, disabled: false, disabledUpdatedAt,
|
|
250
|
+
cooldownUntil: undefined, leaseUntil: undefined, consecutiveFailures: 0,
|
|
251
|
+
lastSuccessAt: Math.max(Date.now(), (old.lastFailureAt ?? 0) + 1) }
|
|
252
|
+
}
|
|
253
|
+
this.touch()
|
|
254
|
+
await this.commit()
|
|
255
|
+
return true
|
|
256
|
+
})
|
|
257
|
+
} catch (error) {
|
|
258
|
+
await this.log(`reactivation refused: ${String(error)}`)
|
|
259
|
+
return false
|
|
225
260
|
}
|
|
226
|
-
this.touch()
|
|
227
261
|
}
|
|
228
262
|
|
|
229
263
|
reset(target: string | 'all'): void {
|
|
264
|
+
const resetAt = this.manualVersion()
|
|
230
265
|
if (target === 'all') {
|
|
231
|
-
this.
|
|
232
|
-
this.
|
|
233
|
-
this.resetModels.clear()
|
|
266
|
+
this.clearContentPolicyConstraints()
|
|
267
|
+
this.state = { ...blank(), resetAt }
|
|
234
268
|
} else {
|
|
235
269
|
delete this.state.models[target]
|
|
236
|
-
this.
|
|
270
|
+
const resets = this.state.modelResetAt ??= {}
|
|
271
|
+
resets[target] = resetAt
|
|
237
272
|
}
|
|
238
273
|
this.touch()
|
|
239
274
|
}
|
|
@@ -243,40 +278,44 @@ export class HealthStore {
|
|
|
243
278
|
* 对审查内容敏感,因此不在 session 间继承审查约束。
|
|
244
279
|
*/
|
|
245
280
|
clearContentPolicyConstraints(): void {
|
|
246
|
-
this.
|
|
247
|
-
this.clearContentPolicyOnFlush = true
|
|
248
|
-
this.touch()
|
|
281
|
+
this.constraints = {}
|
|
249
282
|
}
|
|
250
283
|
|
|
251
284
|
isBlocked(model: ModelRef, now = Date.now()): boolean {
|
|
285
|
+
if (this.writeBlocked) return true
|
|
252
286
|
return [this.state.models[modelKey(model)], this.state.providers[model.provider], this.state.endpoints[endpointKey(model)]]
|
|
253
287
|
.some(record => Boolean(record?.disabled || (record?.cooldownUntil && record.cooldownUntil > now) || (record?.leaseUntil && record.leaseUntil > now)))
|
|
254
288
|
}
|
|
255
289
|
|
|
256
|
-
async
|
|
257
|
-
|
|
258
|
-
let claimed =
|
|
290
|
+
async claimCandidate(model: ModelRef): Promise<boolean> {
|
|
291
|
+
if (this.writeBlocked) return false
|
|
292
|
+
let claimed = false
|
|
259
293
|
await this.withLock(async () => {
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
this.
|
|
266
|
-
|
|
267
|
-
|
|
294
|
+
const disk = await this.readDisk()
|
|
295
|
+
this.state = mergeState(disk, this.state)
|
|
296
|
+
const records = (state: HealthState) => [state.models[modelKey(model)], state.providers[model.provider], state.endpoints[endpointKey(model)]]
|
|
297
|
+
const now = Date.now()
|
|
298
|
+
// Check disk as well: a stale local disabled flag must never override a
|
|
299
|
+
// disable written by another process before this atomic claim.
|
|
300
|
+
if ([...records(disk), ...records(this.state)].some(record => record?.disabled || (record?.cooldownUntil ?? 0) > now || (record?.leaseUntil ?? 0) > now)) return
|
|
301
|
+
const expired = records(this.state).filter(record => record?.cooldownUntil !== undefined && record.cooldownUntil <= now)
|
|
302
|
+
if (expired.length) {
|
|
303
|
+
for (const record of expired) record!.leaseUntil = now + 2 * 60_000
|
|
304
|
+
this.touch()
|
|
305
|
+
await this.commit()
|
|
306
|
+
}
|
|
307
|
+
claimed = true
|
|
308
|
+
}).catch(async error => { claimed = false; await this.log(`candidate claim refused: ${String(error)}`) })
|
|
268
309
|
return claimed
|
|
269
310
|
}
|
|
270
311
|
|
|
271
312
|
async flush(): Promise<void> {
|
|
272
|
-
if (!this.dirty) return
|
|
313
|
+
if (!this.dirty || this.writeBlocked) return
|
|
273
314
|
await this.withLock(async () => {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
// 必须在 merge 之后再清空:磁盘上残留的旧 session 约束不能因合并而重新出现
|
|
277
|
-
if (this.clearContentPolicyOnFlush) this.state.contentPolicyFamilies = {}
|
|
315
|
+
// Even reset-all must validate the on-disk schema before writing.
|
|
316
|
+
this.state = mergeState(await this.readDisk(), this.state)
|
|
278
317
|
await this.commit()
|
|
279
|
-
}).catch(
|
|
318
|
+
}).catch(async error => { await this.log(`health state flush refused: ${String(error)}`) })
|
|
280
319
|
}
|
|
281
320
|
|
|
282
321
|
async log(line: string): Promise<void> {
|
|
@@ -300,20 +339,39 @@ export class HealthStore {
|
|
|
300
339
|
private touch(): void { this.dirty = true }
|
|
301
340
|
private async readDisk(): Promise<HealthState> {
|
|
302
341
|
try {
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
342
|
+
let raw: string
|
|
343
|
+
try { raw = await readFile(this.file, 'utf8') }
|
|
344
|
+
catch (error: any) { if (error?.code === 'ENOENT') return blank(); throw error }
|
|
345
|
+
const parsed = JSON.parse(raw) as HealthState
|
|
346
|
+
if (!parsed || (parsed.schemaVersion !== 2 && parsed.schemaVersion !== 3)) throw new Error(`unsupported state schema: ${parsed?.schemaVersion}`)
|
|
347
|
+
if (this.observedSchema3 && parsed.schemaVersion === 2) throw new Error('unsupported state schema downgrade: 3 -> 2; stop all old Pi processes')
|
|
348
|
+
if (!parsed.models || !parsed.providers || !parsed.endpoints) throw new Error('invalid health state buckets')
|
|
349
|
+
this.observedSchema3 ||= parsed.schemaVersion === 3
|
|
350
|
+
return parsed
|
|
351
|
+
} catch (error) {
|
|
352
|
+
this.writeBlocked = true
|
|
353
|
+
throw error
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
private manualVersion(): number {
|
|
358
|
+
const records = [...Object.values(this.state.models), ...Object.values(this.state.providers), ...Object.values(this.state.endpoints)]
|
|
359
|
+
this.manualClock = Math.max(Date.now(), this.manualClock + 1, (this.state.resetAt ?? 0) + 1,
|
|
360
|
+
...Object.values(this.state.modelResetAt ?? {}).map(at => at + 1),
|
|
361
|
+
...records.map(record => (record.disabledUpdatedAt ?? 0) + 1))
|
|
362
|
+
return this.manualClock
|
|
307
363
|
}
|
|
364
|
+
|
|
308
365
|
private async commit(): Promise<void> {
|
|
366
|
+
if (this.writeBlocked) throw new Error("health state writes are blocked; reload a supported state first")
|
|
367
|
+
this.state.schemaVersion = 3
|
|
368
|
+
delete this.state.contentPolicyFamilies
|
|
309
369
|
this.state.updatedAt = Date.now()
|
|
310
370
|
const temp = `${this.file}.tmp-${process.pid}-${randomUUID()}`
|
|
311
371
|
await writeFile(temp, JSON.stringify(this.state, null, 2), 'utf8')
|
|
312
372
|
await rename(temp, this.file)
|
|
373
|
+
this.observedSchema3 = true
|
|
313
374
|
this.dirty = false
|
|
314
|
-
this.replaceOnFlush = false
|
|
315
|
-
this.resetModels.clear()
|
|
316
|
-
this.clearContentPolicyOnFlush = false
|
|
317
375
|
}
|
|
318
376
|
private bucket(scope: HealthScope): Record<string, HealthRecord> {
|
|
319
377
|
return scope === 'model' ? this.state.models : scope === 'provider' ? this.state.providers : this.state.endpoints
|
|
@@ -341,10 +399,15 @@ function mergeRecord(a: HealthRecord | undefined, b: HealthRecord | undefined):
|
|
|
341
399
|
const lastFailureAt = Math.max(a.lastFailureAt ?? 0, b.lastFailureAt ?? 0)
|
|
342
400
|
const lastSuccessAt = Math.max(a.lastSuccessAt ?? 0, b.lastSuccessAt ?? 0)
|
|
343
401
|
const successWins = lastSuccessAt > lastFailureAt
|
|
402
|
+
const failureCounts = mergeCounts(a.failureCounts, a.totalFailures, b.failureCounts, b.totalFailures)
|
|
344
403
|
return {
|
|
345
404
|
...newest,
|
|
346
|
-
totalFailures:
|
|
347
|
-
|
|
405
|
+
totalFailures: sumCounts(failureCounts),
|
|
406
|
+
failureCounts,
|
|
407
|
+
disabled: (a.disabledUpdatedAt ?? 0) === (b.disabledUpdatedAt ?? 0)
|
|
408
|
+
? (a.disabled === true || b.disabled === true ? true : b.disabled ?? a.disabled)
|
|
409
|
+
: (a.disabledUpdatedAt ?? 0) > (b.disabledUpdatedAt ?? 0) ? a.disabled : b.disabled,
|
|
410
|
+
disabledUpdatedAt: Math.max(a.disabledUpdatedAt ?? 0, b.disabledUpdatedAt ?? 0) || undefined,
|
|
348
411
|
lastFailureAt: lastFailureAt || undefined,
|
|
349
412
|
lastSuccessAt: lastSuccessAt || undefined,
|
|
350
413
|
consecutiveFailures: successWins ? 0 : Math.max(a.consecutiveFailures, b.consecutiveFailures),
|
|
@@ -363,26 +426,59 @@ function mergeBucket(a: Record<string, HealthRecord>, b: Record<string, HealthRe
|
|
|
363
426
|
}
|
|
364
427
|
|
|
365
428
|
function mergeState(a: HealthState, b: HealthState): HealthState {
|
|
366
|
-
const
|
|
429
|
+
const resetAt = Math.max(a.resetAt ?? 0, b.resetAt ?? 0)
|
|
430
|
+
a = afterGlobalReset(a, resetAt)
|
|
431
|
+
b = afterGlobalReset(b, resetAt)
|
|
432
|
+
const modelResetAt = { ...a.modelResetAt }
|
|
433
|
+
for (const [key, at] of Object.entries(b.modelResetAt ?? {})) modelResetAt[key] = Math.max(modelResetAt[key] ?? 0, at)
|
|
434
|
+
const models = mergeBucket(a.models, b.models)
|
|
435
|
+
for (const [key, at] of Object.entries(modelResetAt)) {
|
|
436
|
+
const record = mergeRecord(afterReset(a.models[key], a.modelResetAt?.[key] ?? 0, at), afterReset(b.models[key], b.modelResetAt?.[key] ?? 0, at))
|
|
437
|
+
if (record) models[key] = record
|
|
438
|
+
else delete models[key]
|
|
439
|
+
}
|
|
440
|
+
const switchCounts = mergeCounts(a.switchCounts, a.switches ?? 0, b.switchCounts, b.switches ?? 0)
|
|
441
|
+
const switches = sumCounts(switchCounts)
|
|
367
442
|
// 取最近更新的 switchLog(按 at 去倒序合并,保留最新 20 条)
|
|
368
443
|
const log = [...(a.switchLog ?? []), ...(b.switchLog ?? [])]
|
|
369
444
|
.sort((x, y) => y.at - x.at)
|
|
370
445
|
.filter((entry, index, all) => index === 0 || all[index - 1].at !== entry.at || all[index - 1].from !== entry.from || all[index - 1].to !== entry.to)
|
|
371
446
|
.slice(0, 20)
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
447
|
+
return { schemaVersion: 3, resetAt: resetAt || undefined, modelResetAt, updatedAt: Math.max(a.updatedAt ?? 0, b.updatedAt ?? 0), models, providers: mergeBucket(a.providers ?? {}, b.providers ?? {}), endpoints: mergeBucket(a.endpoints ?? {}, b.endpoints ?? {}), switchCounts, switches, switchLog: log }
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// Each store increments only its own component. Merging the same state twice is
|
|
451
|
+
// idempotent; independent increments add without inflating the backoff streak.
|
|
452
|
+
function mergeCounts(a: Record<string, number> | undefined, aTotal: number, b: Record<string, number> | undefined, bTotal: number): Record<string, number> {
|
|
453
|
+
const left = a ?? { legacy: aTotal }, right = b ?? { legacy: bTotal }
|
|
454
|
+
const result = { ...left }
|
|
455
|
+
for (const [writer, count] of Object.entries(right)) result[writer] = Math.max(result[writer] ?? 0, count)
|
|
456
|
+
return result
|
|
457
|
+
}
|
|
458
|
+
function incrementCounts(counts: Record<string, number> | undefined, total: number, writer: string): Record<string, number> {
|
|
459
|
+
const next = { ...(counts ?? { legacy: total }) }
|
|
460
|
+
next[writer] = (next[writer] ?? 0) + 1
|
|
461
|
+
return next
|
|
462
|
+
}
|
|
463
|
+
function sumCounts(counts: Record<string, number>): number {
|
|
464
|
+
return Object.values(counts).reduce((sum, count) => sum + count, 0)
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
function afterReset(record: HealthRecord | undefined, sourceResetAt: number, resetAt: number): HealthRecord | undefined {
|
|
468
|
+
if (sourceResetAt >= resetAt || !record) return record
|
|
469
|
+
// A manual operation newer than the reset survives, but old health history
|
|
470
|
+
// does not. Legacy records have timestamp zero and cannot beat a reset.
|
|
471
|
+
if ((record.disabledUpdatedAt ?? 0) > resetAt) return { consecutiveFailures: 0, totalFailures: 0, disabled: record.disabled, disabledUpdatedAt: record.disabledUpdatedAt }
|
|
472
|
+
return undefined
|
|
473
|
+
}
|
|
474
|
+
function afterGlobalReset(state: HealthState, resetAt: number): HealthState {
|
|
475
|
+
if ((state.resetAt ?? 0) >= resetAt) return state
|
|
476
|
+
const clean = { ...blank(), resetAt }
|
|
477
|
+
for (const scope of ['models', 'providers', 'endpoints'] as const) {
|
|
478
|
+
for (const [key, record] of Object.entries(state[scope])) {
|
|
479
|
+
const kept = afterReset(record, state.resetAt ?? 0, resetAt)
|
|
480
|
+
if (kept) clean[scope][key] = kept
|
|
385
481
|
}
|
|
386
482
|
}
|
|
387
|
-
return
|
|
483
|
+
return clean
|
|
388
484
|
}
|
package/index.ts
CHANGED
|
@@ -6,13 +6,18 @@ import { fileURLToPath } from 'node:url'
|
|
|
6
6
|
import type { ExtensionAPI, ExtensionContext, FailureObservation, ModelRef } from './types.ts'
|
|
7
7
|
import { classifyFailure, parseRetryAfter } from './classify.ts'
|
|
8
8
|
import { candidateSnapshot, effectiveCandidates, chooseCandidate, multimodalCandidates, modelFamily, summarizeCandidateHealth } from './candidates.ts'
|
|
9
|
-
import { HealthStore, endpointKey, modelKey, type
|
|
9
|
+
import { HealthStore, endpointKey, modelKey, platformKey, type ContentPolicyConstraint } from './health.ts'
|
|
10
10
|
|
|
11
11
|
const FIRST_RESPONSE_TIMEOUT = 90_000
|
|
12
12
|
const STREAM_IDLE_TIMEOUT = 120_000
|
|
13
|
-
|
|
13
|
+
/** 本轮时间限制默认值;可用 CCSWITCH_ROUND_LIMIT_MS 覆盖(毫秒)。 */
|
|
14
|
+
const ROUND_LIMIT_DEFAULT = 8 * 60_000
|
|
15
|
+
const roundLimitMs = (): number => {
|
|
16
|
+
const override = Number(process.env.CCSWITCH_ROUND_LIMIT_MS)
|
|
17
|
+
return Number.isFinite(override) && override > 0 ? override : ROUND_LIMIT_DEFAULT
|
|
18
|
+
}
|
|
14
19
|
const RPC_PROTOCOL_VERSION = 1
|
|
15
|
-
const EXTENSION_VERSION = '0.3.
|
|
20
|
+
const EXTENSION_VERSION = '0.3.12'
|
|
16
21
|
// 同端点(BaseURL 相同)连续失败达到该次数即隔离该端点,避免同一个平台的多个模型逐个试错耗尽本轮切换
|
|
17
22
|
const ENDPOINT_FAIL_THRESHOLD = 3
|
|
18
23
|
|
|
@@ -28,6 +33,8 @@ interface Round {
|
|
|
28
33
|
startedAt: number
|
|
29
34
|
text: string
|
|
30
35
|
images?: Array<Record<string, unknown>>
|
|
36
|
+
needsImage: boolean
|
|
37
|
+
modalitySwitch?: Promise<void>
|
|
31
38
|
tried: Set<string>
|
|
32
39
|
attempts: number
|
|
33
40
|
hadTool: boolean
|
|
@@ -38,16 +45,18 @@ interface Round {
|
|
|
38
45
|
observation?: FailureObservation
|
|
39
46
|
model?: ModelRef
|
|
40
47
|
endpointFails?: EndpointFailTracker
|
|
41
|
-
/**
|
|
48
|
+
/** 本轮内容审查故障转移中必须避开的模型系列(包含本 session 学到的约束)。 */
|
|
42
49
|
avoidFamilies?: Set<string>
|
|
43
50
|
/** 本轮内最后一次成功切换的时间;用于刷新 ROUND_LIMIT 窗口,避免供应商内部重试耗时导致误判“超过本轮时间限制” */
|
|
44
51
|
lastSwitchAt?: number
|
|
52
|
+
/** 本轮内每个已试模型的最后错误消息(key 为 modelKey),用于 exhaust 报告 */
|
|
53
|
+
errors: Map<string, string>
|
|
45
54
|
}
|
|
46
55
|
|
|
47
56
|
function key(model: ModelRef | undefined): string | undefined { return model && modelKey(model) }
|
|
48
57
|
function canRetry(ctx: ExtensionContext): boolean { return ctx.mode === 'tui' || ctx.mode === 'rpc' }
|
|
49
|
-
function activePolicyFamilies(
|
|
50
|
-
return new Set(Object.entries(
|
|
58
|
+
function activePolicyFamilies(constraints: Record<string, ContentPolicyConstraint>, now = Date.now()): Set<string> {
|
|
59
|
+
return new Set(Object.entries(constraints)
|
|
51
60
|
.filter(([, record]) => record.avoidUntil > now)
|
|
52
61
|
.map(([family]) => family))
|
|
53
62
|
}
|
|
@@ -79,6 +88,15 @@ export default function (pi: ExtensionAPI) {
|
|
|
79
88
|
let lastStatus: { status?: number, retryAfterMs?: number } = {}
|
|
80
89
|
// 本次 pi session 内成功切换的模型数量(用于衡量插件的有效程度)
|
|
81
90
|
let sessionSwitches = 0
|
|
91
|
+
// pi.setModel() 内部会 emit model_select 事件(pi core 源码 _emitModelSelect(model, previous, "set"))。
|
|
92
|
+
// 扩展自己触发的切换必须标记 selfSwitching,否则 model_select 处理器会清空本轮故障转移状态
|
|
93
|
+
// (tried/attempts/endpointFails),导致:a) exhaust 报告“已尝试”为空;b) 端点隔离被清零后
|
|
94
|
+
// 同平台候选被逐个重试,烧光 ROUND_LIMIT → “超过本轮时间限制”。
|
|
95
|
+
let selfSwitching = false
|
|
96
|
+
const setModelInternally = async (model: ModelRef): Promise<boolean> => {
|
|
97
|
+
selfSwitching = true
|
|
98
|
+
try { return await pi.setModel(model) } catch { return false } finally { selfSwitching = false }
|
|
99
|
+
}
|
|
82
100
|
|
|
83
101
|
const clearWatchdog = () => { if (timer) clearTimeout(timer); timer = undefined }
|
|
84
102
|
const armWatchdog = (ctx: ExtensionContext, ms: number, roundId: number) => {
|
|
@@ -138,7 +156,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
138
156
|
const candidates = snapshot.models
|
|
139
157
|
const counts = summarizeCandidateHealth(candidates, state)
|
|
140
158
|
const now = Date.now()
|
|
141
|
-
const policyFamilies = activePolicyFamilies(
|
|
159
|
+
const policyFamilies = activePolicyFamilies(health.policyConstraints, now)
|
|
142
160
|
const rows = candidates.slice(0, 10).map(model => {
|
|
143
161
|
const modelRecord = state.models[modelKey(model)]
|
|
144
162
|
const records = [modelRecord, state.providers[model.provider], state.endpoints[endpointKey(model)]]
|
|
@@ -163,7 +181,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
163
181
|
const policySummary = policyFamilies.size ? [...policyFamilies].join(', ') : '无'
|
|
164
182
|
const action = await ctx.ui.select(`CCSwitch 健康面板\n当前:${key(ctx.model) ?? '无'}\n${scopeLabel} · 唯一模型:${counts.total} · 健康:${counts.healthy} · 自动冷却:${counts.cooling} · 手动禁用:${counts.disabled}\n熔断记录:${counts.breakerRecords} · 审查约束系列:${policySummary}\n${switchSummary}${switchPanel}\n${rows.join('\n') || '没有可用模型'}`, ['刷新', '重新激活当前模型', '禁用当前模型', '重置当前模型历史', '关闭'])
|
|
165
183
|
if (action === '刷新') await refresh(ctx)
|
|
166
|
-
if (action === '重新激活当前模型' && ctx.model) {
|
|
184
|
+
if (action === '重新激活当前模型' && ctx.model) {
|
|
185
|
+
const restored = await health.reactivate(ctx.model)
|
|
186
|
+
status(ctx)
|
|
187
|
+
notify(ctx, restored ? '已重新激活当前模型' : '重新激活失败,健康状态未确认保存;请查看 ccswitch-auto-switch.log', restored ? 'info' : 'error')
|
|
188
|
+
}
|
|
167
189
|
if (action === '禁用当前模型' && ctx.model) { health.disable(modelKey(ctx.model), true); await health.flush(); status(ctx); notify(ctx, '已禁用当前模型', 'warning') }
|
|
168
190
|
if (action === '重置当前模型历史' && ctx.model) {
|
|
169
191
|
const ok = !ctx.ui.confirm || await ctx.ui.confirm('重置健康历史', `删除 ${modelKey(ctx.model)} 的记录?`)
|
|
@@ -175,10 +197,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
175
197
|
round.phase = 'exhausted'
|
|
176
198
|
clearWatchdog()
|
|
177
199
|
ctx.ui.setWorkingMessage()
|
|
178
|
-
|
|
200
|
+
// 报告列出本轮每个已试模型及其最后错误,便于直接定位是哪个 provider/模型、什么原因
|
|
201
|
+
const triedLines = [...round.tried].map(item => {
|
|
202
|
+
const error = round!.errors.get(item)
|
|
203
|
+
return error ? `- ${item} — ${error}` : `- ${item}`
|
|
204
|
+
})
|
|
205
|
+
await health.report(`# CCSwitch 自动故障转移失败\n\n时间:${new Date().toISOString()}\n原因:${reason}\n\n已尝试:\n${triedLines.join('\n')}\n\n可使用 /ccswitch status 查看状态,/ccswitch reactivate <provider/model|all> 重新激活。`)
|
|
179
206
|
await health.log(`round exhausted: ${reason}; tried=${[...round.tried].join(',')}`)
|
|
180
|
-
// RPC runner relies on an explicit terminal signal instead of guessing from
|
|
181
|
-
// agent_settled. Keep this RPC-only so ordinary TUI transcripts stay quiet.
|
|
182
207
|
if (ctx.mode === 'rpc') {
|
|
183
208
|
const lastFailure = round.observation ? classifyFailure(round.observation).kind : undefined
|
|
184
209
|
try {
|
|
@@ -190,23 +215,30 @@ export default function (pi: ExtensionAPI) {
|
|
|
190
215
|
model: key(round.model),
|
|
191
216
|
attempts: round.attempts,
|
|
192
217
|
tried: [...round.tried],
|
|
218
|
+
errors: Object.fromEntries(round.errors),
|
|
193
219
|
sessionSwitches,
|
|
194
220
|
})
|
|
195
221
|
} catch { /* reporting must not prevent the regular exhausted path */ }
|
|
196
222
|
}
|
|
197
|
-
|
|
223
|
+
const currentKey = key(round.model)
|
|
224
|
+
const currentError = currentKey ? round.errors.get(currentKey) : undefined
|
|
225
|
+
notify(ctx, `CCSwitch:自动切换停止(${reason})${currentError ? `;当前 ${currentKey}:${currentError}` : ''},请用 /ccswitch 查看详情`, 'error')
|
|
198
226
|
status(ctx)
|
|
199
227
|
}
|
|
200
228
|
/**
|
|
201
229
|
* 模态预检主动切换:当前模型不支持图片但本轮输入/工具结果带图时,
|
|
202
230
|
* 主动切到一个健康的多模态候选(不等失败)。无候选时通知并保持原模型。
|
|
203
231
|
*/
|
|
204
|
-
const
|
|
232
|
+
const performModalitySwitch = async (ctx: ExtensionContext, round: Round): Promise<void> => {
|
|
205
233
|
const current = round.model ?? ctx.model
|
|
206
|
-
if (!current || !canRetry(ctx) || round.phase
|
|
234
|
+
if (!current || !canRetry(ctx) || !['monitoring', 'verifying'].includes(round.phase)) return
|
|
207
235
|
if (current.input?.includes('image')) return
|
|
208
236
|
const candidates = effectiveCandidates(ctx.scopedModels, ctx.modelRegistry.getAvailable())
|
|
209
|
-
|
|
237
|
+
let next: ModelRef | undefined
|
|
238
|
+
for (const candidate of multimodalCandidates(candidates, current, health.snapshot)) {
|
|
239
|
+
if (round.tried.has(modelKey(candidate)) || round.avoidFamilies?.has(modelFamily(candidate)) || round.endpointFails?.isolated.has(platformKey(candidate))) continue
|
|
240
|
+
if (await health.claimCandidate(candidate)) { next = candidate; break }
|
|
241
|
+
}
|
|
210
242
|
if (!next) {
|
|
211
243
|
await health.log(`modality precheck: no multimodal candidate, staying on ${modelKey(current)}`)
|
|
212
244
|
notify(ctx, 'CCSwitch:当前模型不支持图片,且没有可用的多模态候选模型,已保持原模型', 'warning')
|
|
@@ -214,7 +246,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
214
246
|
return
|
|
215
247
|
}
|
|
216
248
|
const previousModel = current
|
|
217
|
-
const set = await
|
|
249
|
+
const set = await setModelInternally(next)
|
|
218
250
|
if (!set) {
|
|
219
251
|
await health.log(`modality precheck: Pi refused model selection ${modelKey(next)}`)
|
|
220
252
|
notify(ctx, `CCSwitch:多模态候选 ${modelKey(next)} 切换失败,已保持原模型`, 'warning')
|
|
@@ -241,32 +273,45 @@ export default function (pi: ExtensionAPI) {
|
|
|
241
273
|
status(ctx)
|
|
242
274
|
}
|
|
243
275
|
|
|
276
|
+
const proactiveModalitySwitch = (ctx: ExtensionContext, target: Round): Promise<void> => {
|
|
277
|
+
// Concurrent tool completions await the same switch before returning to Pi.
|
|
278
|
+
if (!target.modalitySwitch) {
|
|
279
|
+
target.modalitySwitch = performModalitySwitch(ctx, target).finally(() => { target.modalitySwitch = undefined })
|
|
280
|
+
}
|
|
281
|
+
return target.modalitySwitch
|
|
282
|
+
}
|
|
283
|
+
|
|
244
284
|
const failover = async (ctx: ExtensionContext) => {
|
|
245
285
|
if (!round || !round.observation || !round.model || !canRetry(ctx)) return
|
|
246
|
-
// 窗口从上一次成功切换(或本轮开始)起算:供应商内部重试耗时不应消耗整轮限额
|
|
247
|
-
const windowStart = Math.max(round.startedAt, round.lastSwitchAt ?? 0)
|
|
248
|
-
if (Date.now() - windowStart >= ROUND_LIMIT) return exhaust(ctx, '超过本轮时间限制')
|
|
249
286
|
const classification = classifyFailure(round.observation)
|
|
250
287
|
if (round.observation.aborted && !round.observation.watchdog) { round.phase = 'idle'; clearWatchdog(); status(ctx); return }
|
|
251
288
|
round.phase = 'switching'
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
289
|
+
const failedKey = modelKey(round.model)
|
|
290
|
+
round.tried.add(failedKey)
|
|
291
|
+
round.errors.set(failedKey, round.observation.message ?? classification.kind)
|
|
292
|
+
// 跟踪同平台失败:按 BaseURL 归组(platformKey 不含 provider 名),同一平台下的多个
|
|
293
|
+
// provider/模型逐个试错会共享同一故障根源,连续失败到阈值后隔离整个平台,
|
|
294
|
+
// 避免它们烧光本轮切换时间(每个候选最多占用 FIRST_RESPONSE_TIMEOUT)
|
|
255
295
|
const failTracker = round.endpointFails ??= { failed: new Map(), isolated: new Set() }
|
|
256
|
-
const
|
|
257
|
-
const
|
|
258
|
-
failTracker.failed.set(
|
|
259
|
-
if (
|
|
260
|
-
failTracker.isolated.add(
|
|
261
|
-
await health.log(`
|
|
296
|
+
const platform = platformKey(round.model)
|
|
297
|
+
const platformFails = (failTracker.failed.get(platform) ?? 0) + 1
|
|
298
|
+
failTracker.failed.set(platform, platformFails)
|
|
299
|
+
if (platformFails >= ENDPOINT_FAIL_THRESHOLD) {
|
|
300
|
+
failTracker.isolated.add(platform)
|
|
301
|
+
await health.log(`platform ${platform} failed ${platformFails} times this round, isolating platform`)
|
|
262
302
|
}
|
|
263
303
|
if (classification.kind === 'content_policy') {
|
|
264
304
|
health.recordContentPolicyConstraint(modelFamily(round.model), round.model, round.observation.message)
|
|
265
305
|
round.avoidFamilies ??= new Set()
|
|
266
|
-
for (const family of activePolicyFamilies(health.
|
|
306
|
+
for (const family of activePolicyFamilies(health.policyConstraints)) round.avoidFamilies.add(family)
|
|
267
307
|
}
|
|
268
|
-
|
|
308
|
+
// 失败必须先落台账再做时限判断:否则本轮最后一次失败不会进健康状态(无冷却),
|
|
309
|
+
// 下一轮还会重试同一个刚失败的模型/provider
|
|
310
|
+
if (!classification.roundOnly && classification.scope) health.recordFailure(classification.scope, classification.scope === 'model' ? failedKey : classification.scope === 'provider' ? round.model.provider : endpointKey(round.model), classification.kind, round.observation.message, classification.retryAfterMs)
|
|
269
311
|
await health.flush()
|
|
312
|
+
// 窗口从上一次成功切换(或本轮开始)起算:供应商内部重试耗时不应消耗整轮限额
|
|
313
|
+
const windowStart = Math.max(round.startedAt, round.lastSwitchAt ?? 0)
|
|
314
|
+
if (Date.now() - windowStart >= roundLimitMs()) return exhaust(ctx, '超过本轮时间限制')
|
|
270
315
|
await refresh(ctx)
|
|
271
316
|
const candidates = effectiveCandidates(ctx.scopedModels, ctx.modelRegistry.getAvailable())
|
|
272
317
|
const candidateOptions = () => ({
|
|
@@ -276,15 +321,15 @@ export default function (pi: ExtensionAPI) {
|
|
|
276
321
|
health: health.snapshot,
|
|
277
322
|
avoidEndpoints: failTracker.isolated,
|
|
278
323
|
avoidFamilies: round!.avoidFamilies,
|
|
279
|
-
requiredInputs: round!.
|
|
324
|
+
requiredInputs: round!.needsImage ? ['text', 'image'] : ['text'],
|
|
280
325
|
})
|
|
281
326
|
let next = chooseCandidate(candidates, candidateOptions())
|
|
282
327
|
while (next) {
|
|
283
|
-
if (!await health.
|
|
328
|
+
if (!await health.claimCandidate(next)) { round.tried.add(modelKey(next)); next = chooseCandidate(candidates, candidateOptions()); continue }
|
|
284
329
|
round.attempts++
|
|
285
330
|
ctx.ui.setWorkingMessage(`模型异常,正在切换到 ${modelKey(next)}…`)
|
|
286
331
|
const previousModel = round.model
|
|
287
|
-
const set = await
|
|
332
|
+
const set = await setModelInternally(next)
|
|
288
333
|
if (!set) {
|
|
289
334
|
health.recordFailure('model', modelKey(next), 'model_config', 'Pi refused model selection')
|
|
290
335
|
round.tried.add(modelKey(next)); await health.flush()
|
|
@@ -344,14 +389,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
344
389
|
if (event.source === 'extension') return { action: 'continue' }
|
|
345
390
|
clearWatchdog()
|
|
346
391
|
lastStatus = {}
|
|
347
|
-
round = { id: (round?.id ?? 0) + 1, phase: 'monitoring', startedAt: Date.now(), text: event.text, images: event.images, tried: new Set(), attempts: 0, hadTool: false, inTool: false, hadOutput: false, watchdog: false, cleanRetry: false, model: ctx.model, endpointFails: undefined, avoidFamilies: undefined }
|
|
392
|
+
round = { id: (round?.id ?? 0) + 1, phase: 'monitoring', startedAt: Date.now(), text: event.text, images: event.images, needsImage: Boolean(event.images?.length), tried: new Set(), attempts: 0, hadTool: false, inTool: false, hadOutput: false, watchdog: false, cleanRetry: false, model: ctx.model, endpointFails: undefined, avoidFamilies: undefined, errors: new Map() }
|
|
348
393
|
status(ctx)
|
|
349
394
|
// 模态预检:输入带图片但当前模型不支持图片(非多模态)→ 主动切换到多模态模型,
|
|
350
395
|
// 避免 Pi 静默剥图后模型只回答“看不到图片”(此类情况不会触发 failover)。
|
|
351
396
|
if (event.images?.length && round.phase === 'monitoring') await proactiveModalitySwitch(ctx, round)
|
|
352
397
|
return { action: 'continue' }
|
|
353
398
|
})
|
|
354
|
-
pi.on('before_provider_request', (_event, ctx) => {
|
|
399
|
+
pi.on('before_provider_request', (_event, ctx) => {
|
|
400
|
+
lastStatus = {}
|
|
401
|
+
if (round) {
|
|
402
|
+
round.inTool = false
|
|
403
|
+
armWatchdog(ctx, FIRST_RESPONSE_TIMEOUT, round.id)
|
|
404
|
+
}
|
|
405
|
+
})
|
|
355
406
|
pi.on('after_provider_response', (event, ctx) => {
|
|
356
407
|
lastStatus = { status: event.status, retryAfterMs: parseRetryAfter(event.headers?.['retry-after']) }
|
|
357
408
|
if (round && event.status >= 200 && event.status < 300) armWatchdog(ctx, STREAM_IDLE_TIMEOUT, round.id)
|
|
@@ -374,7 +425,10 @@ export default function (pi: ExtensionAPI) {
|
|
|
374
425
|
round.inTool = false
|
|
375
426
|
// 工具结果含图片(如 read 图片文件)且当前模型不支持 → 主动切换多模态模型,
|
|
376
427
|
// 确保下一轮 LLM 调用使用新模型,工具结果中的图片不被 Pi 静默剥除。
|
|
377
|
-
if (
|
|
428
|
+
if (resultContainsImage(_event.result)) {
|
|
429
|
+
round.needsImage = true
|
|
430
|
+
await proactiveModalitySwitch(ctx, round)
|
|
431
|
+
}
|
|
378
432
|
})
|
|
379
433
|
pi.on('turn_end', async (event, ctx) => {
|
|
380
434
|
const message = event.message
|
|
@@ -423,6 +477,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
423
477
|
await failover(ctx)
|
|
424
478
|
})
|
|
425
479
|
pi.on('model_select', (event, ctx) => {
|
|
480
|
+
// pi.setModel() 内部也会发出 model_select(pi core _emitModelSelect(source="set"))。
|
|
481
|
+
// 本扩展自己触发的切换绝不能清空本轮故障转移状态(tried/attempts/endpointFails/errors),
|
|
482
|
+
// 否则:a) exhaust 报告“已尝试”为空,用户看不到哪些 provider/model 失败过;
|
|
483
|
+
// b) 同平台隔离被清零后,同平台候选被逐个重试烧光时限 → “超过本轮时间限制”。
|
|
484
|
+
if (selfSwitching) return
|
|
426
485
|
// 用户手动切换模型(TUI 模型选择器 / /model / 方向键循环)。让 ccs 跟随:
|
|
427
486
|
// 更新“当前模型”并重置本轮故障转移状态,否则 round.model 停留在上一轮使用的模型,
|
|
428
487
|
// 导致状态栏与后续 failover 都滞后于实际选择,直到下次 input 才偶尔纠正。
|
|
@@ -451,9 +510,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
451
510
|
if (!target && verb !== 'reset') { notify(ctx, '请指定 provider/model 或 all', 'warning'); return }
|
|
452
511
|
if (verb === 'disable' && target) { health.disable(target, true); await health.flush(); status(ctx); return }
|
|
453
512
|
if (verb === 'reactivate' && target) {
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
513
|
+
const [provider, ...rest] = target.split('/')
|
|
514
|
+
const restored = await (target === 'all' ? health.reactivateAll() : health.reactivate({ provider, id: rest.join('/') }))
|
|
515
|
+
status(ctx)
|
|
516
|
+
notify(ctx, restored ? '已重新激活' : '重新激活失败,健康状态未确认保存;请查看 ccswitch-auto-switch.log', restored ? 'info' : 'error')
|
|
517
|
+
return
|
|
457
518
|
}
|
|
458
519
|
if (verb === 'reset' && target) {
|
|
459
520
|
const ok = !ctx.ui.confirm || await ctx.ui.confirm('重置健康历史', `删除 ${target} 的健康记录?`)
|
|
@@ -467,7 +528,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
467
528
|
const snapshot = candidateSnapshot(ctx.scopedModels, ctx.modelRegistry.getAvailable())
|
|
468
529
|
const counts = summarizeCandidateHealth(snapshot.models, health.snapshot)
|
|
469
530
|
const state = health.snapshot
|
|
470
|
-
const policyFamilies = activePolicyFamilies(
|
|
531
|
+
const policyFamilies = activePolicyFamilies(health.policyConstraints)
|
|
471
532
|
const source = snapshot.source === 'scoped' ? `Pi scope ${snapshot.sourceEntries} 条` : `Pi 注册表 ${snapshot.sourceEntries} 条`
|
|
472
533
|
notify(ctx, `CCSwitch 自检:${source} · 唯一模型 ${counts.total} · 健康 ${counts.healthy} · 自动冷却 ${counts.cooling} · 手动禁用 ${counts.disabled} · 熔断记录 ${counts.breakerRecords} · 审查约束系列 ${policyFamilies.size} · 本session切换 ${sessionSwitches} · 累计切换 ${state.switches ?? 0}`, counts.total ? 'info' : 'warning')
|
|
473
534
|
}})
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-ccswitch-auto-switch",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.12",
|
|
4
4
|
"description": "Provider-first automatic model failover extension for Pi and CC Switch",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -39,10 +39,11 @@
|
|
|
39
39
|
"node": ">=22.19.0"
|
|
40
40
|
},
|
|
41
41
|
"scripts": {
|
|
42
|
-
"test": "node --test test/*.test.ts test/*.test.mjs",
|
|
42
|
+
"test": "node --experimental-strip-types --test test/*.test.ts test/*.test.mjs",
|
|
43
43
|
"typecheck": "tsc --noEmit --allowImportingTsExtensions --module nodenext --moduleResolution nodenext --target es2022 --skipLibCheck *.ts"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
|
-
"@types/node": "^22.19.19"
|
|
46
|
+
"@types/node": "^22.19.19",
|
|
47
|
+
"typescript": "~5.9.3"
|
|
47
48
|
}
|
|
48
49
|
}
|