pi-ccswitch-auto-switch 0.3.5 → 0.3.6

package/README.md

@@ -151,6 +151,20 @@ The state machine waits for Pi's native retry cycle to settle before switching.
 
 Cooldowns grow exponentially within bounded limits. Learned policy-constrained families are excluded only after a content-policy rejection; they remain eligible during ordinary rate-limit, network, and model-configuration failovers. Context-overflow retries only consider models with a larger context window, and image requests do not move to a model that explicitly supports text only. Unknown errors are isolated to the current model and still fail over normally. A user cancellation is the only aborted turn that does not trigger failover; watchdog cancellations are recorded as timeouts.
 
+### Modality precheck (automatic switch to a multimodal model for images)
+
+When building a provider request, Pi silently handles images according to the current model's `input` capabilities: if the model does not support images (its `input` does not include `image`), Pi replaces the image with a text placeholder `(image omitted: model does not support images)` — **the request does not fail**, so the reactive failover path never fires and the model just answers "I can't see the image", which is useless for OCR tasks.
+
+CCSwitch therefore prechecks **before the request is sent**, instead of waiting for a failure:
+
+- When user input carries images (`input` event with `images`), CCSwitch immediately switches to a healthy multimodal candidate (`input` explicitly includes `image`) before the request is processed, so the images are preserved;
+- When a tool execution returns images (e.g. the `read` tool loading an image file, whose result content includes `image` parts), CCSwitch also switches to a multimodal model before the next LLM call, so the tool-result images are not stripped;
+- Only models that **explicitly** support images (`input` includes `image`) are considered; models with missing metadata are never assumed to support images;
+- The switch reason is recorded as `modality` and counted in both the session and lifetime switch counters;
+- If no multimodal candidate is available, CCSwitch notifies the user and keeps the current model (Pi will still strip the image and add its notice).
+
+The modality precheck applies only in interactive TUI/RPC sessions (the same scope as failover); print/json modes only monitor and never switch.
+
 ## Data and privacy
 
 Health state is stored in Pi's agent directory as `ccswitch-auto-switch-state.json`. The extension stores counters, timestamps, cooldowns, learned model-family policy constraints, and redacted/truncated error summaries. It does not access credentials, authorization headers, CC Switch's database, or Pi's `auth.json`.

package/README.zh-CN.md

@@ -139,6 +139,20 @@ CCS v0.3.5 ✓70/131 · ⏳61 · ⛔0 · 🔄3 · provider/model-id
 
 冷却时间会指数增长但有上限。发生内容审查时，插件只在审查故障转移链中避开已标记系列，普通限流、网络或模型配置故障仍可选择这些模型。上下文溢出时只会选择上下文窗口更大的模型；带图片的请求不会切到明确仅支持文本的模型。无法归类的异常按单模型故障处理并正常切换。用户主动取消是唯一不会触发故障转移的异常终止；看门狗取消会记录为超时。
 
+### 模态预检（OCR / 图片请求自动切换多模态模型）
+
+Pi 在把请求发给 Provider 时，会根据当前模型的 `input` 能力静默处理图片：如果模型不支持图片（`input` 不含 `image`，即非多模态模型），Pi 会把图片替换成一段文本提示 `(image omitted: model does not support images)` 再发送——**请求不会报错**，因此普通的“失败后切换”永远不会触发，模型只会回答“看不到图片”，OCR 结果拿不到。
+
+插件因此在**发送前主动预检**，不等失败：
+
+- **用户输入带图片时**（`input` 事件含 `images`），如果当前模型不支持图片，立即切换到健康的多模态候选（`input` 明确含 `image`）再处理，图片被保留；
+- **工具执行返回图片时**（如 `read` 工具读取图片文件，工具结果含 `image` content），同样在下一轮 LLM 调用前切到多模态模型，确保图片不被剥除；
+- 候选只选**明确声明**支持图片的模型（`input` 含 `image`）；元数据缺失的模型不冒险选择；
+- 切换原因记为 `modality`，计入本 session 切换数与累计切换；
+- 如果没有可用的多模态候选，则通知用户并保持原模型继续（Pi 会照常剥图并附带提示）。
+
+模态预检只在 TUI/RPC 交互模式下生效（与故障转移一致）；print/json 模式只监控不切换。
+
 ## 数据与隐私
 
 健康状态保存在 Pi agent 目录的 `ccswitch-auto-switch-state.json`。其中只有计数、时间、冷却信息、模型系列审查约束和脱敏/截断的错误摘要；插件不会访问凭据、Authorization 请求头、CC Switch 数据库或 Pi 的 `auth.json`。

package/candidates.ts

@@ -75,6 +75,17 @@ export function summarizeCandidateHealth(models: ModelRef[], health: HealthState
   return { total: models.length, healthy, cooling, disabled, breakerRecords }
 }
 
+/**
+ * 模态预检候选：明确声明支持图片输入的健康模型，按优先级排序。
+ * 用于当前模型不支持图片但输入/工具结果带图时的主动切换。
+ * 只选择 input 元数据明确包含 'image' 的模型，元数据缺失的模型不冒险选择。
+ */
+export function multimodalCandidates(candidates: ModelRef[], current: ModelRef | undefined, health: HealthState, now = Date.now()): ModelRef[] {
+  return candidates
+    .filter(model => model.input?.includes('image') && !blocked(model, health, now) && !(current && modelKey(model) === modelKey(current)))
+    .sort((a, b) => score(a, current, health) - score(b, current, health))
+}
+
 export function chooseCandidate(models: ModelRef[], options: CandidateOptions): ModelRef | undefined {
   const current = options.current
   const candidates = models.filter(model => {

package/index.ts

@@ -5,7 +5,7 @@ import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import type { ExtensionAPI, ExtensionContext, FailureObservation, ModelRef } from './types.ts'
 import { classifyFailure, parseRetryAfter } from './classify.ts'
-import { candidateSnapshot, effectiveCandidates, chooseCandidate, modelFamily, summarizeCandidateHealth } from './candidates.ts'
+import { candidateSnapshot, effectiveCandidates, chooseCandidate, multimodalCandidates, modelFamily, summarizeCandidateHealth } from './candidates.ts'
 import { HealthStore, endpointKey, modelKey, type HealthState } from './health.ts'
 
 const FIRST_RESPONSE_TIMEOUT = 90_000
@@ -13,7 +13,7 @@ const STREAM_IDLE_TIMEOUT = 120_000
 const MAX_ATTEMPTS = 5
 const ROUND_LIMIT = 8 * 60_000
 const RPC_PROTOCOL_VERSION = 1
-const EXTENSION_VERSION = '0.3.5'
+const EXTENSION_VERSION = '0.3.6'
 // 同端点（BaseURL 相同）连续失败达到该次数即隔离该端点，避免同一个平台的多个模型逐个试错耗尽本轮切换
 const ENDPOINT_FAIL_THRESHOLD = 3
 
@@ -60,6 +60,19 @@ function resolveModel(ctx: ExtensionContext, provider: string, id: string, previ
   return refs.find(model => model?.provider === provider && model.id === id) ?? { provider, id }
 }
 
+/**
+ * 判断工具执行结果是否包含图片内容（如 read 工具读取图片文件后返回的
+ * { content: [{ type: 'image', ... }] } 结构）。递归查找，兼容各种 result 形状。
+ */
+function resultContainsImage(result: unknown): boolean {
+  if (Array.isArray(result)) return result.some(part => resultContainsImage(part))
+  if (!result || typeof result !== 'object') return false
+  const record = result as Record<string, unknown>
+  if (record.type === 'image') return true
+  if (Array.isArray(record.content)) return record.content.some(part => resultContainsImage(part))
+  return false
+}
+
 export default function (pi: ExtensionAPI) {
   const health = new HealthStore()
   let round: Round | undefined
@@ -185,6 +198,50 @@ export default function (pi: ExtensionAPI) {
     notify(ctx, `CCSwitch：自动切换停止（${reason}），请用 /ccswitch 查看详情`, 'error')
     status(ctx)
   }
+  /**
+   * 模态预检主动切换：当前模型不支持图片但本轮输入/工具结果带图时，
+   * 主动切到一个健康的多模态候选（不等失败）。无候选时通知并保持原模型。
+   */
+  const proactiveModalitySwitch = async (ctx: ExtensionContext, round: Round): Promise<void> => {
+    const current = round.model ?? ctx.model
+    if (!current || !canRetry(ctx) || round.phase !== 'monitoring') return
+    if (current.input?.includes('image')) return
+    const candidates = effectiveCandidates(ctx.scopedModels, ctx.modelRegistry.getAvailable())
+    const next = multimodalCandidates(candidates, current, health.snapshot)[0]
+    if (!next) {
+      await health.log(`modality precheck: no multimodal candidate, staying on ${modelKey(current)}`)
+      notify(ctx, 'CCSwitch：当前模型不支持图片，且没有可用的多模态候选模型，已保持原模型', 'warning')
+      status(ctx)
+      return
+    }
+    const previousModel = current
+    const set = await pi.setModel(next).catch(() => false)
+    if (!set) {
+      await health.log(`modality precheck: Pi refused model selection ${modelKey(next)}`)
+      notify(ctx, `CCSwitch：多模态候选 ${modelKey(next)} 切换失败，已保持原模型`, 'warning')
+      status(ctx)
+      return
+    }
+    round.model = next
+    round.lastSwitchAt = Date.now()
+    sessionSwitches += 1
+    health.recordSwitch(modelKey(previousModel), modelKey(next), 'modality')
+    await health.flush()
+    try {
+      pi.appendEntry?.('ccswitch-switch', {
+        protocolVersion: RPC_PROTOCOL_VERSION,
+        roundId: round.id,
+        from: modelKey(previousModel),
+        to: modelKey(next),
+        reason: 'modality',
+        sessionSwitches,
+      })
+    } catch { /* appendEntry 失败不影响切换 */ }
+    await health.log(`modality precheck switch: ${modelKey(previousModel)} -> ${modelKey(next)}`)
+    notify(ctx, `CCSwitch：检测到图片输入，已切换至多模态模型 ${modelKey(next)}`, 'info')
+    status(ctx)
+  }
+
   const failover = async (ctx: ExtensionContext) => {
     if (!round || !round.observation || !round.model || !canRetry(ctx)) return
     // 窗口从上一次成功切换（或本轮开始）起算：供应商内部重试耗时不应消耗整轮限额
@@ -284,12 +341,15 @@ export default function (pi: ExtensionAPI) {
     await health.log('extension started')
   })
   pi.on('session_shutdown', async (_event, ctx) => { clearWatchdog(); ctx.ui.setWorkingMessage(); if (sessionSwitches > 0) await health.log(`session ended with ${sessionSwitches} successful switches`); await health.flush() })
-  pi.on('input', (event, ctx) => {
+  pi.on('input', async (event, ctx) => {
     if (event.source === 'extension') return { action: 'continue' }
     clearWatchdog()
     lastStatus = {}
     round = { id: (round?.id ?? 0) + 1, phase: 'monitoring', startedAt: Date.now(), text: event.text, images: event.images, tried: new Set(), attempts: 0, hadTool: false, inTool: false, hadOutput: false, watchdog: false, cleanRetry: false, model: ctx.model, endpointFails: undefined, avoidFamilies: undefined }
     status(ctx)
+    // 模态预检：输入带图片但当前模型不支持图片（非多模态）→ 主动切换到多模态模型，
+    // 避免 Pi 静默剥图后模型只回答“看不到图片”（此类情况不会触发 failover）。
+    if (event.images?.length && round.phase === 'monitoring') await proactiveModalitySwitch(ctx, round)
     return { action: 'continue' }
   })
   pi.on('before_provider_request', (_event, ctx) => { lastStatus = {}; if (round) { round.inTool = false; armWatchdog(ctx, FIRST_RESPONSE_TIMEOUT, round.id) } })
@@ -310,7 +370,13 @@ export default function (pi: ExtensionAPI) {
     }
   })
   pi.on('tool_execution_start', () => { if (round) { round.hadTool = true; round.inTool = true; clearWatchdog() } })
-  pi.on('tool_execution_end', () => { if (round) round.inTool = false })
+  pi.on('tool_execution_end', async (_event, ctx) => {
+    if (!round) return
+    round.inTool = false
+    // 工具结果含图片（如 read 图片文件）且当前模型不支持 → 主动切换多模态模型，
+    // 确保下一轮 LLM 调用使用新模型，工具结果中的图片不被 Pi 静默剥除。
+    if (round.phase === 'monitoring' && resultContainsImage(_event.result)) await proactiveModalitySwitch(ctx, round)
+  })
   pi.on('turn_end', async (event, ctx) => {
     const message = event.message
     if (message?.role !== 'assistant' || !round) return

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-ccswitch-auto-switch",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Provider-first automatic model failover extension for Pi and CC Switch",
   "license": "MIT",
   "keywords": [