pi-ccswitch-auto-switch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +109 -0
- package/README.zh-CN.md +101 -0
- package/candidates.ts +49 -0
- package/classify.ts +37 -0
- package/health.ts +291 -0
- package/index.ts +236 -0
- package/package.json +44 -0
- package/types.ts +71 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Juny Wuuuu91
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
package/README.md
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# Pi CCSwitch Auto Switch
|
|
2
|
+
|
|
3
|
+
> 中文说明:[README.zh-CN.md](README.zh-CN.md)
|
|
4
|
+
|
|
5
|
+
Resilient model failover for [Pi](https://github.com/badlogic/pi-mono) when its providers are managed by [CC Switch](https://ccswitch.io/).
|
|
6
|
+
|
|
7
|
+
The extension observes real Pi requests, records sanitized health signals, and—only in interactive TUI/RPC sessions—moves a failed request to a healthy model. It treats provider failures as provider failures: an exhausted balance, invalid credentials, or rate limit will not cause a rapid series of retries against sibling models from the same provider.
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
- Uses Pi's effective model registry; never reads CC Switch databases, Pi auth files, or API keys.
|
|
12
|
+
- Provider-first circuit breakers for authentication, quota, billing, and rate-limit failures.
|
|
13
|
+
- Endpoint circuit breakers for DNS, connection, server, and streaming failures.
|
|
14
|
+
- Model-only isolation for missing models and incompatible parameters.
|
|
15
|
+
- Prefers a healthy model from another provider before considering a sibling model.
|
|
16
|
+
- 90-second first-response and 120-second streaming-idle watchdogs.
|
|
17
|
+
- Exponential cooldowns, `Retry-After` support, and one persisted half-open probe lease per provider.
|
|
18
|
+
- Persistent, atomic, cross-process health state with error redaction and bounded logs.
|
|
19
|
+
- Compact Pi status bar and interactive health panel.
|
|
20
|
+
- Non-interactive `--print` / JSON runs monitor and record failures but do not inject a competing retry.
|
|
21
|
+
|
|
22
|
+
## Requirements
|
|
23
|
+
|
|
24
|
+
- Pi `0.84.4` or newer.
|
|
25
|
+
- CC Switch `3.20+` is recommended for its native Pi model configuration support.
|
|
26
|
+
- Node.js `22.19+` only for local development and tests. Pi supplies the runtime for the extension itself.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
### Pi package install (recommended)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pi install git:github.com/JunyWuuuu91/pi-ccswitch-auto-switch
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
This works on macOS, Linux, and Windows when Git is available. After the npm release, this will also work:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pi install npm:pi-ccswitch-auto-switch
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Restart Pi or run `/reload` after installing or updating. To update the Git installation later, run `pi update --extensions`.
|
|
43
|
+
|
|
44
|
+
CC Switch should be configured normally. This extension deliberately does not write Pi model settings or CC Switch data.
|
|
45
|
+
|
|
46
|
+
## Commands
|
|
47
|
+
|
|
48
|
+
| Command | What it does |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| `/ccswitch` or `/ccswitch status` | Open the health panel. |
|
|
51
|
+
| `/ccswitch help` | Show the command reference in Pi. |
|
|
52
|
+
| `/ccswitch refresh` | Refresh Pi's model registry and the status display. |
|
|
53
|
+
| `/ccswitch reactivate <provider/model\|all>` | Clear a breaker and let the next real request verify recovery, while preserving history. |
|
|
54
|
+
| `/ccswitch disable <provider/model>` | Manually exclude a model from failover. |
|
|
55
|
+
| `/ccswitch reset <provider/model\|all>` | Delete selected health history after confirmation. |
|
|
56
|
+
| `/ccswitch-test` | Inspect candidate discovery without changing models. |
|
|
57
|
+
|
|
58
|
+
Examples:
|
|
59
|
+
|
|
60
|
+
```text
|
|
61
|
+
/ccswitch reactivate all
|
|
62
|
+
/ccswitch disable openai/gpt-4.1
|
|
63
|
+
/ccswitch reset anthropic/claude-sonnet-4
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Status bar
|
|
67
|
+
|
|
68
|
+
```text
|
|
69
|
+
CCS ✓130 · ⏳1 · ⛔0
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
- `✓130`: effective models currently eligible for selection.
|
|
73
|
+
- `⏳1`: one model, provider, or endpoint health record is cooling down.
|
|
74
|
+
- `⛔0`: no manually disabled models.
|
|
75
|
+
- During a switch, `CCS ↻2/5 provider/model` means the second of at most five attempts is being made.
|
|
76
|
+
|
|
77
|
+
Use `/ccswitch` to inspect the exact record behind a cooldown.
|
|
78
|
+
|
|
79
|
+
## Failover behavior
|
|
80
|
+
|
|
81
|
+
The state machine waits for Pi's native retry cycle to settle before switching. A user message invalidates any pending switch from an older round, avoiding duplicate dispatches.
|
|
82
|
+
|
|
83
|
+
| Failure | Scope | Initial cooldown |
|
|
84
|
+
| --- | --- | --- |
|
|
85
|
+
| `401`, `403`, quota, billing | Provider | 30 minutes |
|
|
86
|
+
| `429` | Provider | `Retry-After` when present; otherwise 5 minutes |
|
|
87
|
+
| DNS, network, `5xx`, interrupted stream | Endpoint | 2 minutes |
|
|
88
|
+
| `404`, invalid model, incompatible parameters | Model | 15 minutes |
|
|
89
|
+
| Content filtering or context overflow | Current round only | None |
|
|
90
|
+
|
|
91
|
+
Cooldowns grow exponentially within bounded limits. Context-overflow retries only consider models with a larger context window. User cancellations are not recorded as failures; watchdog cancellations are recorded as timeouts.
|
|
92
|
+
|
|
93
|
+
## Data and privacy
|
|
94
|
+
|
|
95
|
+
Health state is stored in Pi's agent directory as `ccswitch-auto-switch-state.json`. The extension stores counters, timestamps, cooldowns, and redacted/truncated error summaries. It does not access credentials, authorization headers, CC Switch's database, or Pi's `auth.json`.
|
|
96
|
+
|
|
97
|
+
## Development
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
npm install
|
|
101
|
+
npm run typecheck
|
|
102
|
+
npm test
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Tests use Node's built-in test runner and cover failure classification, provider-first selection, cooldowns, state persistence, and Windows-compatible paths.
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
[MIT](LICENSE)
|
package/README.zh-CN.md
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# Pi CCSwitch 自动故障转移
|
|
2
|
+
|
|
3
|
+
> English: [README.md](README.md)
|
|
4
|
+
|
|
5
|
+
当 [CC Switch](https://ccswitch.io/) 管理 [Pi](https://github.com/badlogic/pi-mono) 的 Provider 时,为 Pi 提供更稳健的模型自动故障转移。
|
|
6
|
+
|
|
7
|
+
它只观察 Pi 的真实请求,保存脱敏健康记录;仅在 TUI/RPC 交互会话中,才会把失败请求平滑切到健康模型。额度耗尽、凭据失效或限流等 Provider 级问题会直接熔断整个 Provider,不会连续尝试同一 Provider 下的多个兄弟模型。
|
|
8
|
+
|
|
9
|
+
## 功能
|
|
10
|
+
|
|
11
|
+
- 以 Pi 的有效模型注册表为准,不读取 CC Switch 数据库、Pi 认证文件或 API Key。
|
|
12
|
+
- 对认证、额度、账单和限流实施 Provider 优先熔断。
|
|
13
|
+
- 对 DNS、连接、服务端和流中断实施端点熔断。
|
|
14
|
+
- 模型不存在或参数不兼容时仅隔离该模型。
|
|
15
|
+
- 优先选择不同 Provider 的健康模型;同 Provider 的模型排在后面。
|
|
16
|
+
- 首响应 90 秒、流式停滞 120 秒看门狗。
|
|
17
|
+
- 指数退避、`Retry-After` 支持和跨进程 half-open 单探针租约。
|
|
18
|
+
- 原子持久化状态、错误脱敏和有上限的日志。
|
|
19
|
+
- Pi 紧凑状态栏与健康管理面板。
|
|
20
|
+
- `--print` / JSON 非交互运行只监控和记录,不会注入可能与进程退出竞争的新请求。
|
|
21
|
+
|
|
22
|
+
## 要求
|
|
23
|
+
|
|
24
|
+
- Pi `0.84.4` 或更高版本。
|
|
25
|
+
- 建议使用 CC Switch `3.20+`,其已支持原生维护 Pi 模型配置。
|
|
26
|
+
- 本地开发和测试需要 Node.js `22.19+`;Pi 运行插件本身不需要额外安装 Node。
|
|
27
|
+
|
|
28
|
+
## 安装
|
|
29
|
+
|
|
30
|
+
### 通过 Pi Package 安装(推荐)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pi install git:github.com/JunyWuuuu91/pi-ccswitch-auto-switch
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
该命令在 macOS、Linux 和安装了 Git 的 Windows 上均可使用。npm 版本发布后,也可以执行:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pi install npm:pi-ccswitch-auto-switch
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
安装或更新后重启 Pi,或执行 `/reload`。后续更新 Git 版本可执行 `pi update --extensions`。
|
|
43
|
+
|
|
44
|
+
正常使用 CC Switch 配置 Provider 即可;本插件不会修改 Pi 模型设置或 CC Switch 数据。
|
|
45
|
+
|
|
46
|
+
## 命令
|
|
47
|
+
|
|
48
|
+
| 命令 | 作用 |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| `/ccswitch` 或 `/ccswitch status` | 打开健康面板。 |
|
|
51
|
+
| `/ccswitch help` | 在 Pi 内显示命令帮助。 |
|
|
52
|
+
| `/ccswitch refresh` | 刷新 Pi 模型注册表和状态栏。 |
|
|
53
|
+
| `/ccswitch reactivate <provider/model\|all>` | 解除熔断,下一次真实请求验证恢复情况;保留历史。 |
|
|
54
|
+
| `/ccswitch disable <provider/model>` | 手动排除模型。 |
|
|
55
|
+
| `/ccswitch reset <provider/model\|all>` | 确认后删除相应健康历史。 |
|
|
56
|
+
| `/ccswitch-test` | 仅检查候选发现,不切换模型。 |
|
|
57
|
+
|
|
58
|
+
## 状态栏
|
|
59
|
+
|
|
60
|
+
```text
|
|
61
|
+
CCS ✓130 · ⏳1 · ⛔0
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
- `✓130`:当前可参与选择的有效模型数量。
|
|
65
|
+
- `⏳1`:有一条模型、Provider 或端点健康记录仍在冷却。
|
|
66
|
+
- `⛔0`:没有被手动禁用的模型。
|
|
67
|
+
- 切换中出现 `CCS ↻2/5 provider/model`,表示正在进行最多 5 次尝试中的第 2 次。
|
|
68
|
+
|
|
69
|
+
通过 `/ccswitch` 查看具体哪个记录正在冷却。
|
|
70
|
+
|
|
71
|
+
## 故障转移逻辑
|
|
72
|
+
|
|
73
|
+
插件会先等待 Pi 内置重试结束和会话恢复 idle,再进行切换。新的用户输入会使旧轮次的待切换任务失效,从而避免重复发送。
|
|
74
|
+
|
|
75
|
+
| 失败类型 | 熔断范围 | 初始冷却 |
|
|
76
|
+
| --- | --- | --- |
|
|
77
|
+
| `401`、`403`、额度或账单问题 | Provider | 30 分钟 |
|
|
78
|
+
| `429` | Provider | 优先使用 `Retry-After`,否则 5 分钟 |
|
|
79
|
+
| DNS、网络、`5xx`、流中断 | 端点 | 2 分钟 |
|
|
80
|
+
| `404`、模型不存在、参数不兼容 | 模型 | 15 分钟 |
|
|
81
|
+
| 内容过滤、上下文溢出 | 仅本轮 | 无 |
|
|
82
|
+
|
|
83
|
+
冷却时间会指数增长但有上限。上下文溢出时只会选择上下文窗口更大的模型。用户主动取消不计失败;看门狗取消会记录为超时。
|
|
84
|
+
|
|
85
|
+
## 数据与隐私
|
|
86
|
+
|
|
87
|
+
健康状态保存在 Pi agent 目录的 `ccswitch-auto-switch-state.json`。其中只有计数、时间、冷却信息和脱敏/截断的错误摘要;插件不会访问凭据、Authorization 请求头、CC Switch 数据库或 Pi 的 `auth.json`。
|
|
88
|
+
|
|
89
|
+
## 开发
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
npm install
|
|
93
|
+
npm run typecheck
|
|
94
|
+
npm test
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
测试使用 Node 内置测试运行器,覆盖失败分类、Provider 优先选择、冷却、状态持久化以及 Windows 路径。
|
|
98
|
+
|
|
99
|
+
## 许可证
|
|
100
|
+
|
|
101
|
+
[MIT](LICENSE)
|
package/candidates.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { ModelRef, ScopedModel } from './types.ts'
|
|
2
|
+
import { endpointKey, modelKey, type HealthState } from './health.ts'
|
|
3
|
+
|
|
4
|
+
export interface CandidateOptions {
|
|
5
|
+
current?: ModelRef
|
|
6
|
+
tried: Set<string>
|
|
7
|
+
failureKind?: string
|
|
8
|
+
health: HealthState
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function blocked(model: ModelRef, health: HealthState, now = Date.now()): boolean {
|
|
12
|
+
const records = [health.models[modelKey(model)], health.providers[model.provider], health.endpoints[endpointKey(model)]]
|
|
13
|
+
return records.some(record => Boolean(record?.disabled || (record?.cooldownUntil && record.cooldownUntil > now) || (record?.leaseUntil && record.leaseUntil > now)))
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function effectiveCandidates(scoped: readonly ScopedModel[], available: ModelRef[]): ModelRef[] {
|
|
17
|
+
const source = scoped.length > 0 ? scoped.map(item => item.model) : available
|
|
18
|
+
const seen = new Set<string>()
|
|
19
|
+
return source.filter(model => {
|
|
20
|
+
const key = modelKey(model)
|
|
21
|
+
if (seen.has(key)) return false
|
|
22
|
+
seen.add(key)
|
|
23
|
+
return true
|
|
24
|
+
})
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function chooseCandidate(models: ModelRef[], options: CandidateOptions): ModelRef | undefined {
|
|
28
|
+
const current = options.current
|
|
29
|
+
const candidates = models.filter(model => {
|
|
30
|
+
if (options.tried.has(modelKey(model)) || blocked(model, options.health)) return false
|
|
31
|
+
if (options.failureKind === 'context_overflow' && current && (model.contextWindow ?? 0) <= (current.contextWindow ?? 0)) return false
|
|
32
|
+
return true
|
|
33
|
+
})
|
|
34
|
+
return candidates.sort((a, b) => score(a, current) - score(b, current))[0]
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function score(candidate: ModelRef, current?: ModelRef): number {
|
|
38
|
+
if (!current) return 0
|
|
39
|
+
const sameProvider = candidate.provider === current.provider ? 1000 : 0
|
|
40
|
+
const differentId = candidate.id === current.id ? 0 : 100
|
|
41
|
+
const inputMismatch = sameInputs(candidate, current) ? 0 : 20
|
|
42
|
+
const reasoningMismatch = candidate.reasoning === current.reasoning ? 0 : 10
|
|
43
|
+
const contextPenalty = (candidate.contextWindow ?? 0) < (current.contextWindow ?? 0) ? 5 : 0
|
|
44
|
+
return sameProvider + differentId + inputMismatch + reasoningMismatch + contextPenalty
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function sameInputs(a: ModelRef, b: ModelRef): boolean {
|
|
48
|
+
return (a.input ?? []).join(',') === (b.input ?? []).join(',')
|
|
49
|
+
}
|
package/classify.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { FailureClass, FailureObservation, HealthScope } from './types.ts'
|
|
2
|
+
|
|
3
|
+
export interface Classification {
|
|
4
|
+
kind: FailureClass
|
|
5
|
+
scope?: HealthScope
|
|
6
|
+
roundOnly: boolean
|
|
7
|
+
retryAfterMs?: number
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
const content = /content[ _-]?filter|sensitive|guardrail|policy[_ ]?violation|content[_ ]?blocked/i
|
|
11
|
+
const quota = /quota|billing|insufficient.?balance|out of budget|usage.?limit|credit.?balance|monthly.?limit/i
|
|
12
|
+
const context = /context.?window|context.?length|too many tokens|prompt is too long|input is too long|token limit/i
|
|
13
|
+
const transport = /dns|enotfound|eai_again|socket|connection|network|fetch failed|timed? ?out|timeout|stream ended|terminated|websocket/i
|
|
14
|
+
|
|
15
|
+
export function classifyFailure(observation: FailureObservation): Classification {
|
|
16
|
+
const text = `${observation.rawStopReason ?? ''} ${observation.message ?? ''}`
|
|
17
|
+
const status = observation.status
|
|
18
|
+
if (observation.aborted && !observation.watchdog) return { kind: 'unknown', roundOnly: true }
|
|
19
|
+
if (observation.watchdog) return { kind: 'timeout', scope: 'endpoint', roundOnly: false }
|
|
20
|
+
if (content.test(text)) return { kind: 'content_policy', roundOnly: true }
|
|
21
|
+
if (context.test(text)) return { kind: 'context_overflow', roundOnly: true }
|
|
22
|
+
if (status === 429) return { kind: 'rate_limit', scope: 'provider', roundOnly: false, retryAfterMs: observation.retryAfterMs }
|
|
23
|
+
if (status === 401 || status === 403) return { kind: 'auth', scope: 'provider', roundOnly: false }
|
|
24
|
+
if (quota.test(text)) return { kind: 'quota', scope: 'provider', roundOnly: false }
|
|
25
|
+
if (status !== undefined && status >= 500) return { kind: 'endpoint', scope: 'endpoint', roundOnly: false }
|
|
26
|
+
if (status === 400 || status === 404 || status === 422) return { kind: 'model_config', scope: 'model', roundOnly: false }
|
|
27
|
+
if (transport.test(text)) return { kind: 'endpoint', scope: 'endpoint', roundOnly: false }
|
|
28
|
+
return { kind: 'unknown', scope: 'model', roundOnly: false }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function parseRetryAfter(value: string | undefined, now = Date.now()): number | undefined {
|
|
32
|
+
if (!value) return undefined
|
|
33
|
+
const seconds = Number(value)
|
|
34
|
+
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000
|
|
35
|
+
const at = Date.parse(value)
|
|
36
|
+
return Number.isNaN(at) ? undefined : Math.max(0, at - now)
|
|
37
|
+
}
|
package/health.ts
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
2
|
+
import { appendFile, mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
3
|
+
import { existsSync } from 'node:fs'
|
|
4
|
+
import { homedir } from 'node:os'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import type { FailureClass, HealthScope, ModelRef } from './types.ts'
|
|
7
|
+
|
|
8
|
+
const STATE_FILE = 'ccswitch-auto-switch-state.json'
|
|
9
|
+
const LEGACY_FILE = 'ccswitch-cooldown.json'
|
|
10
|
+
const LOCK_DIR = '.ccswitch-auto-switch.lock'
|
|
11
|
+
const LOG_FILE = 'ccswitch-auto-switch.log'
|
|
12
|
+
const REPORT_FILE = 'ccswitch-failure-report.md'
|
|
13
|
+
const MAX_LOG_BYTES = 512 * 1024
|
|
14
|
+
|
|
15
|
+
export interface HealthRecord {
|
|
16
|
+
consecutiveFailures: number
|
|
17
|
+
totalFailures: number
|
|
18
|
+
lastFailureAt?: number
|
|
19
|
+
lastSuccessAt?: number
|
|
20
|
+
cooldownUntil?: number
|
|
21
|
+
lastClass?: FailureClass
|
|
22
|
+
lastError?: string
|
|
23
|
+
disabled?: boolean
|
|
24
|
+
leaseUntil?: number
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface HealthState {
|
|
28
|
+
schemaVersion: 2
|
|
29
|
+
updatedAt: number
|
|
30
|
+
models: Record<string, HealthRecord>
|
|
31
|
+
providers: Record<string, HealthRecord>
|
|
32
|
+
endpoints: Record<string, HealthRecord>
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function agentDir(env = process.env, home = homedir()): string {
|
|
36
|
+
return env.PI_CODING_AGENT_DIR?.trim() || join(home, '.pi', 'agent')
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function modelKey(model: Pick<ModelRef, 'provider' | 'id'>): string {
|
|
40
|
+
return `${model.provider}/${model.id}`
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function endpointKey(model: ModelRef): string {
|
|
44
|
+
const source = model.baseUrl ? safeEndpoint(model.baseUrl) : `provider:${model.provider}`
|
|
45
|
+
return createHash('sha256').update(`${model.provider}\0${source}`).digest('hex').slice(0, 20)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function safeEndpoint(value: string): string {
|
|
49
|
+
try {
|
|
50
|
+
const url = new URL(value)
|
|
51
|
+
return `${url.protocol}//${url.host}${url.pathname}`
|
|
52
|
+
} catch {
|
|
53
|
+
return value.slice(0, 256)
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function blank(): HealthState {
|
|
58
|
+
return { schemaVersion: 2, updatedAt: Date.now(), models: {}, providers: {}, endpoints: {} }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function redact(text: string | undefined): string | undefined {
|
|
62
|
+
if (!text) return undefined
|
|
63
|
+
return text
|
|
64
|
+
.replace(/(authorization\s*[:=]\s*)(\S+)/gi, '$1[redacted]')
|
|
65
|
+
.replace(/(bearer\s+)(\S+)/gi, '$1[redacted]')
|
|
66
|
+
.replace(/([?&](?:key|token|api[_-]?key|signature)=)[^&\s]+/gi, '$1[redacted]')
|
|
67
|
+
.replace(/sk-[A-Za-z0-9_-]{12,}/g, '[redacted]')
|
|
68
|
+
.replace(/[\x00-\x1f\x7f]/g, ' ')
|
|
69
|
+
.slice(0, 240)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function cooldownMs(kind: FailureClass, failures: number, retryAfterMs?: number): number {
|
|
73
|
+
const factor = 2 ** Math.max(0, failures - 1)
|
|
74
|
+
if (kind === 'rate_limit') return Math.min(24 * 60 * 60_000, Math.max(retryAfterMs ?? 0, 5 * 60_000 * factor))
|
|
75
|
+
if (kind === 'auth' || kind === 'quota') return Math.min(6 * 60 * 60_000, 30 * 60_000 * factor)
|
|
76
|
+
if (kind === 'model_config') return Math.min(2 * 60 * 60_000, 15 * 60_000 * factor)
|
|
77
|
+
return Math.min(30 * 60_000, 2 * 60_000 * factor)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export class HealthStore {
|
|
81
|
+
readonly dir: string
|
|
82
|
+
private state: HealthState = blank()
|
|
83
|
+
private dirty = false
|
|
84
|
+
private replaceOnFlush = false
|
|
85
|
+
|
|
86
|
+
constructor(dir = agentDir()) { this.dir = dir }
|
|
87
|
+
get file(): string { return join(this.dir, STATE_FILE) }
|
|
88
|
+
get snapshot(): HealthState { return structuredClone(this.state) }
|
|
89
|
+
|
|
90
|
+
async load(): Promise<void> {
|
|
91
|
+
await mkdir(this.dir, { recursive: true })
|
|
92
|
+
try {
|
|
93
|
+
if (existsSync(this.file)) {
|
|
94
|
+
const parsed = JSON.parse(await readFile(this.file, 'utf8')) as HealthState
|
|
95
|
+
if (parsed.schemaVersion !== 2 || !parsed.models || !parsed.providers || !parsed.endpoints) throw new Error('unsupported state schema')
|
|
96
|
+
this.state = parsed
|
|
97
|
+
} else {
|
|
98
|
+
await this.migrateLegacy()
|
|
99
|
+
}
|
|
100
|
+
} catch {
|
|
101
|
+
if (existsSync(this.file)) await rename(this.file, `${this.file}.corrupt-${Date.now()}`).catch(() => {})
|
|
102
|
+
this.state = blank()
|
|
103
|
+
this.dirty = true
|
|
104
|
+
await this.flush()
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private async migrateLegacy(): Promise<void> {
|
|
109
|
+
const legacy = join(this.dir, LEGACY_FILE)
|
|
110
|
+
if (!existsSync(legacy)) return
|
|
111
|
+
try {
|
|
112
|
+
const entries = JSON.parse(await readFile(legacy, 'utf8')) as Record<string, { failedAt?: number, reason?: string, errorMessage?: string }>
|
|
113
|
+
for (const [key, entry] of Object.entries(entries)) {
|
|
114
|
+
this.state.models[key] = {
|
|
115
|
+
consecutiveFailures: 1, totalFailures: 1, lastFailureAt: entry.failedAt,
|
|
116
|
+
cooldownUntil: (entry.failedAt ?? Date.now()) + 60 * 60_000,
|
|
117
|
+
lastClass: entry.reason === 'sensitive' ? 'content_policy' : 'unknown', lastError: redact(entry.errorMessage),
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
this.dirty = true
|
|
121
|
+
await this.flush()
|
|
122
|
+
} catch { /* legacy data is optional */ }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
recordFailure(scope: HealthScope, key: string, kind: FailureClass, message?: string, retryAfterMs?: number): void {
|
|
126
|
+
const bucket = this.bucket(scope)
|
|
127
|
+
const previous = bucket[key] ?? { consecutiveFailures: 0, totalFailures: 0 }
|
|
128
|
+
const failures = previous.consecutiveFailures + 1
|
|
129
|
+
bucket[key] = {
|
|
130
|
+
...previous, consecutiveFailures: failures, totalFailures: previous.totalFailures + 1,
|
|
131
|
+
lastFailureAt: Date.now(), cooldownUntil: Date.now() + cooldownMs(kind, failures, retryAfterMs),
|
|
132
|
+
lastClass: kind, lastError: redact(message), leaseUntil: undefined,
|
|
133
|
+
}
|
|
134
|
+
this.touch()
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
recordSuccess(model: ModelRef): void {
|
|
138
|
+
this.close('model', modelKey(model))
|
|
139
|
+
this.close('provider', model.provider)
|
|
140
|
+
this.close('endpoint', endpointKey(model))
|
|
141
|
+
this.touch()
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private close(scope: HealthScope, key: string): void {
|
|
145
|
+
const current = this.bucket(scope)[key]
|
|
146
|
+
if (!current) return
|
|
147
|
+
this.bucket(scope)[key] = { ...current, consecutiveFailures: 0, cooldownUntil: undefined, leaseUntil: undefined, lastSuccessAt: Date.now() }
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
disable(key: string, disabled: boolean): void {
|
|
151
|
+
const old = this.state.models[key] ?? { consecutiveFailures: 0, totalFailures: 0 }
|
|
152
|
+
this.state.models[key] = { ...old, disabled }
|
|
153
|
+
this.touch()
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
reactivate(model: ModelRef): void {
|
|
157
|
+
const key = modelKey(model)
|
|
158
|
+
for (const [scope, id] of [['model', key], ['provider', model.provider], ['endpoint', endpointKey(model)]] as const) {
|
|
159
|
+
const old = this.bucket(scope)[id]
|
|
160
|
+
if (old) this.bucket(scope)[id] = { ...old, disabled: false, cooldownUntil: undefined, leaseUntil: undefined, lastSuccessAt: Date.now(), consecutiveFailures: 0 }
|
|
161
|
+
}
|
|
162
|
+
this.touch()
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
reactivateAll(): void {
|
|
166
|
+
const now = Date.now()
|
|
167
|
+
for (const bucket of [this.state.models, this.state.providers, this.state.endpoints]) {
|
|
168
|
+
for (const [key, old] of Object.entries(bucket)) bucket[key] = { ...old, disabled: false, cooldownUntil: undefined, leaseUntil: undefined, lastSuccessAt: now, consecutiveFailures: 0 }
|
|
169
|
+
}
|
|
170
|
+
this.touch()
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
reset(target: string | 'all'): void {
|
|
174
|
+
if (target === 'all') this.state = blank()
|
|
175
|
+
else delete this.state.models[target]
|
|
176
|
+
this.replaceOnFlush = true
|
|
177
|
+
this.touch()
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
isBlocked(model: ModelRef, now = Date.now()): boolean {
|
|
181
|
+
return [this.state.models[modelKey(model)], this.state.providers[model.provider], this.state.endpoints[endpointKey(model)]]
|
|
182
|
+
.some(record => Boolean(record?.disabled || (record?.cooldownUntil && record.cooldownUntil > now) || (record?.leaseUntil && record.leaseUntil > now)))
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async claimProvider(model: ModelRef): Promise<boolean> {
|
|
186
|
+
const key = model.provider
|
|
187
|
+
let claimed = true
|
|
188
|
+
await this.withLock(async () => {
|
|
189
|
+
if (!this.replaceOnFlush) this.state = mergeState(await this.readDisk(), this.state)
|
|
190
|
+
const record = this.state.providers[key]
|
|
191
|
+
if (!record || !(record.cooldownUntil && record.cooldownUntil <= Date.now())) return
|
|
192
|
+
if (record.leaseUntil && record.leaseUntil > Date.now()) { claimed = false; return }
|
|
193
|
+
record.leaseUntil = Date.now() + 2 * 60_000
|
|
194
|
+
this.touch()
|
|
195
|
+
await this.commit()
|
|
196
|
+
}).catch(() => { claimed = false })
|
|
197
|
+
return claimed
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
async flush(): Promise<void> {
|
|
201
|
+
if (!this.dirty) return
|
|
202
|
+
await this.withLock(async () => {
|
|
203
|
+
this.state = mergeState(await this.readDisk(), this.state)
|
|
204
|
+
await this.commit()
|
|
205
|
+
}).catch(() => {})
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async log(line: string): Promise<void> {
|
|
209
|
+
const path = join(this.dir, LOG_FILE)
|
|
210
|
+
try {
|
|
211
|
+
if (existsSync(path) && (await stat(path)).size > MAX_LOG_BYTES) {
|
|
212
|
+
await rm(`${path}.3`, { force: true })
|
|
213
|
+
await rename(`${path}.2`, `${path}.3`).catch(() => {})
|
|
214
|
+
await rename(`${path}.1`, `${path}.2`).catch(() => {})
|
|
215
|
+
await rename(path, `${path}.1`)
|
|
216
|
+
}
|
|
217
|
+
await appendFile(path, `[${new Date().toISOString()}] ${redact(line) ?? ''}\n`, 'utf8')
|
|
218
|
+
} catch { /* logging must never affect Pi */ }
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
async report(markdown: string): Promise<void> {
|
|
222
|
+
const safe = markdown.split('\n').map(line => redact(line) ?? '').join('\n')
|
|
223
|
+
await writeFile(join(this.dir, REPORT_FILE), safe, 'utf8').catch(() => {})
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
private touch(): void { this.dirty = true }
|
|
227
|
+
private async readDisk(): Promise<HealthState> {
|
|
228
|
+
try {
|
|
229
|
+
if (!existsSync(this.file)) return blank()
|
|
230
|
+
const parsed = JSON.parse(await readFile(this.file, 'utf8')) as HealthState
|
|
231
|
+
return parsed.schemaVersion === 2 ? parsed : blank()
|
|
232
|
+
} catch { return blank() }
|
|
233
|
+
}
|
|
234
|
+
private async commit(): Promise<void> {
|
|
235
|
+
this.state.updatedAt = Date.now()
|
|
236
|
+
const temp = `${this.file}.tmp-${process.pid}-${randomUUID()}`
|
|
237
|
+
await writeFile(temp, JSON.stringify(this.state, null, 2), 'utf8')
|
|
238
|
+
await rename(temp, this.file)
|
|
239
|
+
this.dirty = false
|
|
240
|
+
this.replaceOnFlush = false
|
|
241
|
+
}
|
|
242
|
+
private bucket(scope: HealthScope): Record<string, HealthRecord> {
|
|
243
|
+
return scope === 'model' ? this.state.models : scope === 'provider' ? this.state.providers : this.state.endpoints
|
|
244
|
+
}
|
|
245
|
+
private async withLock<T>(work: () => Promise<T>): Promise<T> {
|
|
246
|
+
const lock = join(this.dir, LOCK_DIR)
|
|
247
|
+
const deadline = Date.now() + 750
|
|
248
|
+
while (true) {
|
|
249
|
+
try {
|
|
250
|
+
await mkdir(lock)
|
|
251
|
+
try { return await work() } finally { await rm(lock, { recursive: true, force: true }) }
|
|
252
|
+
} catch (error: any) {
|
|
253
|
+
if (error?.code !== 'EEXIST' || Date.now() >= deadline) throw error
|
|
254
|
+
try { if (Date.now() - (await stat(lock)).mtimeMs > 30_000) await rm(lock, { recursive: true, force: true }) } catch { /* another process won */ }
|
|
255
|
+
await new Promise(resolve => setTimeout(resolve, 25 + Math.floor(Math.random() * 50)))
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function mergeRecord(a: HealthRecord | undefined, b: HealthRecord | undefined): HealthRecord | undefined {
|
|
262
|
+
if (!a) return b
|
|
263
|
+
if (!b) return a
|
|
264
|
+
const newest = (b.lastFailureAt ?? 0) >= (a.lastFailureAt ?? 0) ? b : a
|
|
265
|
+
const lastFailureAt = Math.max(a.lastFailureAt ?? 0, b.lastFailureAt ?? 0)
|
|
266
|
+
const lastSuccessAt = Math.max(a.lastSuccessAt ?? 0, b.lastSuccessAt ?? 0)
|
|
267
|
+
const successWins = lastSuccessAt > lastFailureAt
|
|
268
|
+
return {
|
|
269
|
+
...newest,
|
|
270
|
+
totalFailures: Math.max(a.totalFailures, b.totalFailures),
|
|
271
|
+
disabled: b.disabled ?? a.disabled,
|
|
272
|
+
lastFailureAt: lastFailureAt || undefined,
|
|
273
|
+
lastSuccessAt: lastSuccessAt || undefined,
|
|
274
|
+
consecutiveFailures: successWins ? 0 : Math.max(a.consecutiveFailures, b.consecutiveFailures),
|
|
275
|
+
cooldownUntil: successWins ? undefined : Math.max(a.cooldownUntil ?? 0, b.cooldownUntil ?? 0) || undefined,
|
|
276
|
+
leaseUntil: successWins ? undefined : Math.max(a.leaseUntil ?? 0, b.leaseUntil ?? 0) || undefined,
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function mergeBucket(a: Record<string, HealthRecord>, b: Record<string, HealthRecord>): Record<string, HealthRecord> {
|
|
281
|
+
const output: Record<string, HealthRecord> = {}
|
|
282
|
+
for (const key of new Set([...Object.keys(a), ...Object.keys(b)])) {
|
|
283
|
+
const record = mergeRecord(a[key], b[key])
|
|
284
|
+
if (record) output[key] = record
|
|
285
|
+
}
|
|
286
|
+
return output
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function mergeState(a: HealthState, b: HealthState): HealthState {
|
|
290
|
+
return { schemaVersion: 2, updatedAt: Math.max(a.updatedAt ?? 0, b.updatedAt ?? 0), models: mergeBucket(a.models ?? {}, b.models ?? {}), providers: mergeBucket(a.providers ?? {}, b.providers ?? {}), endpoints: mergeBucket(a.endpoints ?? {}, b.endpoints ?? {}) }
|
|
291
|
+
}
|
package/index.ts
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import type { ExtensionAPI, ExtensionContext, FailureObservation, ModelRef } from './types.ts'
|
|
2
|
+
import { classifyFailure, parseRetryAfter } from './classify.ts'
|
|
3
|
+
import { effectiveCandidates, chooseCandidate } from './candidates.ts'
|
|
4
|
+
import { HealthStore, endpointKey, modelKey } from './health.ts'
|
|
5
|
+
|
|
6
|
+
const FIRST_RESPONSE_TIMEOUT = 90_000
|
|
7
|
+
const STREAM_IDLE_TIMEOUT = 120_000
|
|
8
|
+
const MAX_ATTEMPTS = 5
|
|
9
|
+
const ROUND_LIMIT = 8 * 60_000
|
|
10
|
+
|
|
11
|
+
type Phase = 'idle' | 'monitoring' | 'settled-error' | 'switching' | 'redispatching' | 'verifying' | 'exhausted'
|
|
12
|
+
interface Round {
|
|
13
|
+
id: number
|
|
14
|
+
phase: Phase
|
|
15
|
+
startedAt: number
|
|
16
|
+
text: string
|
|
17
|
+
images?: Array<Record<string, unknown>>
|
|
18
|
+
tried: Set<string>
|
|
19
|
+
attempts: number
|
|
20
|
+
hadTool: boolean
|
|
21
|
+
inTool: boolean
|
|
22
|
+
hadOutput: boolean
|
|
23
|
+
watchdog: boolean
|
|
24
|
+
cleanRetry: boolean
|
|
25
|
+
observation?: FailureObservation
|
|
26
|
+
model?: ModelRef
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function key(model: ModelRef | undefined): string | undefined { return model && modelKey(model) }
|
|
30
|
+
function canRetry(ctx: ExtensionContext): boolean { return ctx.mode === 'tui' || ctx.mode === 'rpc' }
|
|
31
|
+
|
|
32
|
+
export default function (pi: ExtensionAPI) {
|
|
33
|
+
const health = new HealthStore()
|
|
34
|
+
let round: Round | undefined
|
|
35
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
36
|
+
let lastStatus: { status?: number, retryAfterMs?: number } = {}
|
|
37
|
+
|
|
38
|
+
const clearWatchdog = () => { if (timer) clearTimeout(timer); timer = undefined }
|
|
39
|
+
const armWatchdog = (ctx: ExtensionContext, ms: number, roundId: number) => {
|
|
40
|
+
clearWatchdog()
|
|
41
|
+
timer = setTimeout(() => {
|
|
42
|
+
if (!round || round.id !== roundId || (!ctx.isIdle() && !round.inTool)) {
|
|
43
|
+
round && (round.watchdog = true)
|
|
44
|
+
ctx.abort()
|
|
45
|
+
}
|
|
46
|
+
}, ms)
|
|
47
|
+
timer.unref?.()
|
|
48
|
+
}
|
|
49
|
+
const status = (ctx: ExtensionContext) => {
|
|
50
|
+
const state = health.snapshot
|
|
51
|
+
const models = effectiveCandidates(ctx.scopedModels, ctx.modelRegistry.getAvailable())
|
|
52
|
+
const now = Date.now()
|
|
53
|
+
const disabled = Object.values(state.models).filter(item => item.disabled).length
|
|
54
|
+
const cooling = [...Object.values(state.models), ...Object.values(state.providers), ...Object.values(state.endpoints)]
|
|
55
|
+
.filter(item => item.cooldownUntil && item.cooldownUntil > now).length
|
|
56
|
+
const unhealthy = models.filter(model => health.isBlocked(model)).length
|
|
57
|
+
const healthy = Math.max(0, models.length - unhealthy)
|
|
58
|
+
const plain = round?.phase === 'switching' ? `CCS ↻${round.attempts}/${MAX_ATTEMPTS} ${round.model ? modelKey(round.model) : ''}` :
|
|
59
|
+
cooling || disabled ? `CCS ✓${healthy} · ⏳${cooling} · ⛔${disabled}` : `CCS ✓${models.length}`
|
|
60
|
+
const theme = ctx.ui.theme
|
|
61
|
+
ctx.ui.setStatus('ccswitch-ha', theme ? theme.fg(cooling || disabled ? 'warning' : 'success', plain) : plain)
|
|
62
|
+
}
|
|
63
|
+
const notify = (ctx: ExtensionContext, message: string, type: 'info' | 'warning' | 'error' = 'info') => {
|
|
64
|
+
if (ctx.hasUI) ctx.ui.notify(message, type)
|
|
65
|
+
}
|
|
66
|
+
const showHelp = (ctx: ExtensionContext) => {
|
|
67
|
+
notify(ctx, [
|
|
68
|
+
'CCSwitch 命令:',
|
|
69
|
+
'/ccswitch 或 /ccswitch status — 查看健康面板',
|
|
70
|
+
'/ccswitch help — 显示此帮助',
|
|
71
|
+
'/ccswitch refresh — 刷新 Pi 模型注册表和状态',
|
|
72
|
+
'/ccswitch reactivate <provider/model|all> — 解除熔断,保留历史',
|
|
73
|
+
'/ccswitch disable <provider/model> — 手动禁用模型',
|
|
74
|
+
'/ccswitch reset <provider/model|all> — 清除健康历史(需确认)',
|
|
75
|
+
'/ccswitch-test — 自检候选模型,不实际切换',
|
|
76
|
+
].join('\n'), 'info')
|
|
77
|
+
}
|
|
78
|
+
const refresh = async (ctx: ExtensionContext) => {
|
|
79
|
+
try { await ctx.modelRegistry.refresh() } catch (error) { await health.log(`registry refresh failed: ${String(error)}`); notify(ctx, 'CCSwitch:Pi 模型注册表刷新失败,继续使用上次快照', 'warning') }
|
|
80
|
+
status(ctx)
|
|
81
|
+
}
|
|
82
|
+
const showPanel = async (ctx: ExtensionContext) => {
|
|
83
|
+
const state = health.snapshot
|
|
84
|
+
const candidates = effectiveCandidates(ctx.scopedModels, ctx.modelRegistry.getAvailable())
|
|
85
|
+
const rows = candidates.slice(0, 10).map(model => {
|
|
86
|
+
const record = state.models[modelKey(model)] ?? state.providers[model.provider]
|
|
87
|
+
const suffix = record?.disabled ? '禁用' : record?.cooldownUntil && record.cooldownUntil > Date.now() ? `冷却 ${Math.ceil((record.cooldownUntil - Date.now()) / 60_000)}m` : '健康'
|
|
88
|
+
return `${modelKey(model)} ${suffix}`
|
|
89
|
+
})
|
|
90
|
+
if (!ctx.ui.select) { notify(ctx, `CCSwitch:${candidates.length} 个候选,${rows.filter(row => row.includes('冷却')).length} 个冷却`, 'info'); return }
|
|
91
|
+
const action = await ctx.ui.select(`CCSwitch 健康面板\n当前:${key(ctx.model) ?? '无'}\n${rows.join('\n') || '没有可用模型'}`, ['刷新', '重新激活当前模型', '禁用当前模型', '重置当前模型历史', '关闭'])
|
|
92
|
+
if (action === '刷新') await refresh(ctx)
|
|
93
|
+
if (action === '重新激活当前模型' && ctx.model) { health.reactivate(ctx.model); await health.flush(); status(ctx); notify(ctx, '已重新激活当前模型') }
|
|
94
|
+
if (action === '禁用当前模型' && ctx.model) { health.disable(modelKey(ctx.model), true); await health.flush(); status(ctx); notify(ctx, '已禁用当前模型', 'warning') }
|
|
95
|
+
if (action === '重置当前模型历史' && ctx.model) {
|
|
96
|
+
const ok = !ctx.ui.confirm || await ctx.ui.confirm('重置健康历史', `删除 ${modelKey(ctx.model)} 的记录?`)
|
|
97
|
+
if (ok) { health.reset(modelKey(ctx.model)); await health.flush(); status(ctx) }
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const exhaust = async (ctx: ExtensionContext, reason: string) => {
|
|
101
|
+
if (!round) return
|
|
102
|
+
round.phase = 'exhausted'
|
|
103
|
+
clearWatchdog()
|
|
104
|
+
ctx.ui.setWorkingMessage()
|
|
105
|
+
await health.report(`# CCSwitch 自动故障转移失败\n\n时间:${new Date().toISOString()}\n原因:${reason}\n\n已尝试:\n${[...round.tried].map(item => `- ${item}`).join('\n')}\n\n可使用 /ccswitch status 查看状态,/ccswitch reactivate <provider/model> 重新激活。`)
|
|
106
|
+
await health.log(`round exhausted: ${reason}; tried=${[...round.tried].join(',')}`)
|
|
107
|
+
notify(ctx, `CCSwitch:自动切换停止(${reason}),请用 /ccswitch 查看详情`, 'error')
|
|
108
|
+
status(ctx)
|
|
109
|
+
}
|
|
110
|
+
const failover = async (ctx: ExtensionContext) => {
|
|
111
|
+
if (!round || !round.observation || !round.model || !canRetry(ctx)) return
|
|
112
|
+
if (round.attempts >= MAX_ATTEMPTS || Date.now() - round.startedAt >= ROUND_LIMIT) return exhaust(ctx, '达到本轮切换上限')
|
|
113
|
+
const classification = classifyFailure(round.observation)
|
|
114
|
+
if (round.observation.aborted && !round.observation.watchdog) { round.phase = 'idle'; clearWatchdog(); status(ctx); return }
|
|
115
|
+
round.phase = 'switching'
|
|
116
|
+
round.tried.add(modelKey(round.model))
|
|
117
|
+
if (!classification.roundOnly && classification.scope) health.recordFailure(classification.scope, classification.scope === 'model' ? modelKey(round.model) : classification.scope === 'provider' ? round.model.provider : endpointKey(round.model), classification.kind, round.observation.message, classification.retryAfterMs)
|
|
118
|
+
await health.flush()
|
|
119
|
+
await refresh(ctx)
|
|
120
|
+
const candidates = effectiveCandidates(ctx.scopedModels, ctx.modelRegistry.getAvailable())
|
|
121
|
+
let next = chooseCandidate(candidates, { current: round.model, tried: round.tried, failureKind: classification.kind, health: health.snapshot })
|
|
122
|
+
while (next) {
|
|
123
|
+
if (!await health.claimProvider(next)) { round.tried.add(modelKey(next)); next = chooseCandidate(candidates, { current: round.model, tried: round.tried, failureKind: classification.kind, health: health.snapshot }); continue }
|
|
124
|
+
round.attempts++
|
|
125
|
+
ctx.ui.setWorkingMessage(`模型异常,正在切换到 ${modelKey(next)}…`)
|
|
126
|
+
const set = await pi.setModel(next).catch(() => false)
|
|
127
|
+
if (!set) {
|
|
128
|
+
health.recordFailure('model', modelKey(next), 'model_config', 'Pi refused model selection')
|
|
129
|
+
round.tried.add(modelKey(next)); await health.flush()
|
|
130
|
+
next = chooseCandidate(candidates, { current: round.model, tried: round.tried, failureKind: classification.kind, health: health.snapshot })
|
|
131
|
+
continue
|
|
132
|
+
}
|
|
133
|
+
round.model = next
|
|
134
|
+
round.phase = 'redispatching'
|
|
135
|
+
notify(ctx, `CCSwitch:已切换至 ${modelKey(next)}(${round.attempts}/${MAX_ATTEMPTS})`, 'info')
|
|
136
|
+
const continuation = round.hadTool || round.hadOutput
|
|
137
|
+
? '请从当前会话状态继续完成上一条请求;不要重复已经完成的工具操作。'
|
|
138
|
+
: round.images?.length ? [{ type: 'text', text: round.text }, ...round.images] : round.text
|
|
139
|
+
round.cleanRetry = !round.hadTool && !round.hadOutput
|
|
140
|
+
round.observation = undefined
|
|
141
|
+
round.watchdog = false
|
|
142
|
+
round.phase = 'verifying'
|
|
143
|
+
pi.sendUserMessage(continuation as any)
|
|
144
|
+
status(ctx)
|
|
145
|
+
return
|
|
146
|
+
}
|
|
147
|
+
await exhaust(ctx, '没有健康的候选模型')
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
pi.on('session_start', async (_event, ctx) => { await health.load(); await refresh(ctx); await health.log('extension started') })
|
|
151
|
+
pi.on('session_shutdown', async (_event, ctx) => { clearWatchdog(); ctx.ui.setWorkingMessage(); await health.flush() })
|
|
152
|
+
pi.on('input', (event, ctx) => {
|
|
153
|
+
if (event.source === 'extension') return { action: 'continue' }
|
|
154
|
+
clearWatchdog()
|
|
155
|
+
lastStatus = {}
|
|
156
|
+
round = { id: (round?.id ?? 0) + 1, phase: 'monitoring', startedAt: Date.now(), text: event.text, images: event.images, tried: new Set(), attempts: 0, hadTool: false, inTool: false, hadOutput: false, watchdog: false, cleanRetry: false, model: ctx.model }
|
|
157
|
+
status(ctx)
|
|
158
|
+
return { action: 'continue' }
|
|
159
|
+
})
|
|
160
|
+
pi.on('before_provider_request', (_event, ctx) => { lastStatus = {}; if (round) { round.inTool = false; armWatchdog(ctx, FIRST_RESPONSE_TIMEOUT, round.id) } })
|
|
161
|
+
pi.on('after_provider_response', (event, ctx) => {
|
|
162
|
+
lastStatus = { status: event.status, retryAfterMs: parseRetryAfter(event.headers?.['retry-after']) }
|
|
163
|
+
if (round && event.status >= 200 && event.status < 300) armWatchdog(ctx, STREAM_IDLE_TIMEOUT, round.id)
|
|
164
|
+
})
|
|
165
|
+
pi.on('message_update', (_event, ctx) => { if (round) armWatchdog(ctx, STREAM_IDLE_TIMEOUT, round.id) })
|
|
166
|
+
pi.on('context', (event) => {
|
|
167
|
+
if (!round?.cleanRetry) return
|
|
168
|
+
round.cleanRetry = false
|
|
169
|
+
let removed = false
|
|
170
|
+
return {
|
|
171
|
+
messages: event.messages.filter((message: any) => {
|
|
172
|
+
if (!removed && message?.role === 'assistant' && (message.stopReason === 'error' || message.stopReason === 'aborted')) { removed = true; return false }
|
|
173
|
+
return true
|
|
174
|
+
}),
|
|
175
|
+
}
|
|
176
|
+
})
|
|
177
|
+
pi.on('tool_execution_start', () => { if (round) { round.hadTool = true; round.inTool = true; clearWatchdog() } })
|
|
178
|
+
pi.on('tool_execution_end', () => { if (round) round.inTool = false })
|
|
179
|
+
pi.on('turn_end', async (event, ctx) => {
|
|
180
|
+
const message = event.message
|
|
181
|
+
if (message?.role !== 'assistant' || !round) return
|
|
182
|
+
clearWatchdog()
|
|
183
|
+
round.model = { provider: message.provider, id: message.model }
|
|
184
|
+
round.hadOutput ||= Boolean(message.content?.length)
|
|
185
|
+
if (message.stopReason === 'error' || message.stopReason === 'aborted') {
|
|
186
|
+
round.phase = 'settled-error'
|
|
187
|
+
round.observation = { message: message.errorMessage, rawStopReason: message.rawStopReason, status: lastStatus.status, retryAfterMs: lastStatus.retryAfterMs, watchdog: round.watchdog, aborted: message.stopReason === 'aborted' }
|
|
188
|
+
await health.log(`turn failed ${modelKey(round.model)}: ${message.errorMessage ?? message.stopReason}`)
|
|
189
|
+
} else {
|
|
190
|
+
health.recordSuccess(round.model); await health.flush()
|
|
191
|
+
round.phase = 'idle'; round.observation = undefined; ctx.ui.setWorkingMessage(); status(ctx)
|
|
192
|
+
}
|
|
193
|
+
})
|
|
194
|
+
pi.on('agent_settled', async (_event, ctx) => {
|
|
195
|
+
if (!round || round.phase !== 'settled-error') return
|
|
196
|
+
if (!ctx.isIdle()) return
|
|
197
|
+
if (!canRetry(ctx) && round.observation && round.model) {
|
|
198
|
+
const classification = classifyFailure(round.observation)
|
|
199
|
+
if (!classification.roundOnly && classification.scope) {
|
|
200
|
+
const domain = classification.scope === 'model' ? modelKey(round.model) : classification.scope === 'provider' ? round.model.provider : endpointKey(round.model)
|
|
201
|
+
health.recordFailure(classification.scope, domain, classification.kind, round.observation.message, classification.retryAfterMs)
|
|
202
|
+
await health.flush()
|
|
203
|
+
}
|
|
204
|
+
round.phase = 'idle'
|
|
205
|
+
status(ctx)
|
|
206
|
+
return
|
|
207
|
+
}
|
|
208
|
+
await failover(ctx)
|
|
209
|
+
})
|
|
210
|
+
pi.on('model_select', (_event, ctx) => status(ctx))
|
|
211
|
+
|
|
212
|
+
pi.registerCommand('ccswitch', { description: '查看和管理 CCSwitch 自动故障转移健康状态', handler: async (args, ctx) => {
|
|
213
|
+
const [verb, target] = args.trim().split(/\s+/, 2)
|
|
214
|
+
if (!verb || verb === 'status') return showPanel(ctx)
|
|
215
|
+
if (verb === 'help') return showHelp(ctx)
|
|
216
|
+
if (verb === 'refresh') return refresh(ctx)
|
|
217
|
+
if (!target && verb !== 'reset') { notify(ctx, '请指定 provider/model 或 all', 'warning'); return }
|
|
218
|
+
if (verb === 'disable' && target) { health.disable(target, true); await health.flush(); status(ctx); return }
|
|
219
|
+
if (verb === 'reactivate' && target) {
|
|
220
|
+
if (target === 'all') health.reactivateAll()
|
|
221
|
+
else { const [provider, ...rest] = target.split('/'); health.reactivate({ provider, id: rest.join('/') }) }
|
|
222
|
+
await health.flush(); status(ctx); return
|
|
223
|
+
}
|
|
224
|
+
if (verb === 'reset' && target) {
|
|
225
|
+
const ok = !ctx.ui.confirm || await ctx.ui.confirm('重置健康历史', `删除 ${target} 的健康记录?`)
|
|
226
|
+
if (ok) { health.reset(target === 'all' ? 'all' : target); await health.flush(); status(ctx) }
|
|
227
|
+
return
|
|
228
|
+
}
|
|
229
|
+
showHelp(ctx)
|
|
230
|
+
}})
|
|
231
|
+
pi.registerCommand('ccswitch-test', { description: '检查 CCSwitch 候选模型和健康状态(不切换模型)', handler: async (_args, ctx) => {
|
|
232
|
+
await refresh(ctx)
|
|
233
|
+
const count = effectiveCandidates(ctx.scopedModels, ctx.modelRegistry.getAvailable()).length
|
|
234
|
+
notify(ctx, `CCSwitch 自检完成:${count} 个有效候选模型`, count ? 'info' : 'warning')
|
|
235
|
+
}})
|
|
236
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "pi-ccswitch-auto-switch",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Provider-first automatic model failover extension for Pi and CC Switch",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"pi-package",
|
|
8
|
+
"pi-extension",
|
|
9
|
+
"ccswitch",
|
|
10
|
+
"failover",
|
|
11
|
+
"model-routing"
|
|
12
|
+
],
|
|
13
|
+
"repository": {
|
|
14
|
+
"type": "git",
|
|
15
|
+
"url": "git+https://github.com/JunyWuuuu91/pi-ccswitch-auto-switch.git"
|
|
16
|
+
},
|
|
17
|
+
"bugs": {
|
|
18
|
+
"url": "https://github.com/JunyWuuuu91/pi-ccswitch-auto-switch/issues"
|
|
19
|
+
},
|
|
20
|
+
"homepage": "https://github.com/JunyWuuuu91/pi-ccswitch-auto-switch#readme",
|
|
21
|
+
"files": [
|
|
22
|
+
"index.ts",
|
|
23
|
+
"types.ts",
|
|
24
|
+
"classify.ts",
|
|
25
|
+
"candidates.ts",
|
|
26
|
+
"health.ts"
|
|
27
|
+
],
|
|
28
|
+
"pi": {
|
|
29
|
+
"extensions": [
|
|
30
|
+
"./index.ts"
|
|
31
|
+
]
|
|
32
|
+
},
|
|
33
|
+
"type": "module",
|
|
34
|
+
"engines": {
|
|
35
|
+
"node": ">=22.19.0"
|
|
36
|
+
},
|
|
37
|
+
"scripts": {
|
|
38
|
+
"test": "node --test test/*.test.ts",
|
|
39
|
+
"typecheck": "tsc --noEmit --allowImportingTsExtensions --module nodenext --moduleResolution nodenext --target es2022 --skipLibCheck *.ts"
|
|
40
|
+
},
|
|
41
|
+
"devDependencies": {
|
|
42
|
+
"@types/node": "^22.19.19"
|
|
43
|
+
}
|
|
44
|
+
}
|
package/types.ts
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
export interface ModelRef {
|
|
2
|
+
provider: string
|
|
3
|
+
id: string
|
|
4
|
+
name?: string
|
|
5
|
+
baseUrl?: string
|
|
6
|
+
contextWindow?: number
|
|
7
|
+
maxTokens?: number
|
|
8
|
+
reasoning?: boolean
|
|
9
|
+
input?: string[]
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface ScopedModel {
|
|
13
|
+
model: ModelRef
|
|
14
|
+
thinkingLevel?: string
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface ExtensionContext {
|
|
18
|
+
mode: 'tui' | 'rpc' | 'json' | 'print'
|
|
19
|
+
hasUI: boolean
|
|
20
|
+
model?: ModelRef
|
|
21
|
+
scopedModels: readonly ScopedModel[]
|
|
22
|
+
isIdle(): boolean
|
|
23
|
+
hasPendingMessages(): boolean
|
|
24
|
+
abort(): void
|
|
25
|
+
signal?: AbortSignal
|
|
26
|
+
modelRegistry: {
|
|
27
|
+
refresh(): Promise<unknown>
|
|
28
|
+
getAvailable(): ModelRef[]
|
|
29
|
+
}
|
|
30
|
+
ui: {
|
|
31
|
+
theme?: { fg(kind: string, text: string): string }
|
|
32
|
+
notify(message: string, type?: 'info' | 'warning' | 'error'): void
|
|
33
|
+
setStatus(key: string, text: string | undefined): void
|
|
34
|
+
setWorkingMessage(message?: string): void
|
|
35
|
+
select?(title: string, options: string[]): Promise<string | undefined>
|
|
36
|
+
confirm?(title: string, message: string): Promise<boolean>
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ExtensionAPI {
|
|
41
|
+
on(event: string, handler: (event: any, ctx: ExtensionContext) => unknown): void
|
|
42
|
+
registerCommand(name: string, options: {
|
|
43
|
+
description: string
|
|
44
|
+
handler: (args: string, ctx: ExtensionContext) => Promise<void> | void
|
|
45
|
+
}): void
|
|
46
|
+
sendUserMessage(content: string | Array<Record<string, unknown>>): void
|
|
47
|
+
setModel(model: ModelRef): Promise<boolean>
|
|
48
|
+
setThinkingLevel?(level: string): void
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export type FailureClass =
|
|
52
|
+
| 'auth'
|
|
53
|
+
| 'quota'
|
|
54
|
+
| 'rate_limit'
|
|
55
|
+
| 'endpoint'
|
|
56
|
+
| 'timeout'
|
|
57
|
+
| 'model_config'
|
|
58
|
+
| 'content_policy'
|
|
59
|
+
| 'context_overflow'
|
|
60
|
+
| 'unknown'
|
|
61
|
+
|
|
62
|
+
export type HealthScope = 'model' | 'provider' | 'endpoint'
|
|
63
|
+
|
|
64
|
+
export interface FailureObservation {
|
|
65
|
+
message?: string
|
|
66
|
+
status?: number
|
|
67
|
+
retryAfterMs?: number
|
|
68
|
+
watchdog?: boolean
|
|
69
|
+
aborted?: boolean
|
|
70
|
+
rawStopReason?: string
|
|
71
|
+
}
|