pi-model-fallback 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -51
- package/README.md +12 -1
- package/docs/usage.md +11 -2
- package/extensions/index.ts +81 -30
- package/lib/config.ts +185 -4
- package/lib/error-status.ts +18 -0
- package/package.json +6 -4
package/CHANGELOG.md
CHANGED
|
@@ -1,7 +1,18 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- Warn on completely shadowed fallback rules in config validation, save confirmations, status output, and config details without rejecting warning-bearing configs.
|
|
8
|
+
- Document the missing `git add`/`git commit` steps in `CONTRIBUTING.md` release instructions so maintainers can trigger auto-release after `npm version patch --no-git-tag-version`.
|
|
9
|
+
- Clarify that maintainers should update `CHANGELOG.md` before staging release files.
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- Bump package version to `0.3.5` for the next patch release.
|
|
14
|
+
|
|
15
|
+
- Declare `@earendil-works/pi-agent-core` as a peerDependency to align with pi-extension-template.
|
|
5
16
|
- Add Buy Me a Coffee sponsor button to README and native GitHub funding link via `.github/FUNDING.yml`.
|
|
6
17
|
|
|
7
18
|
## [0.3.3] - 2026-07-04
|
|
@@ -16,52 +27,51 @@
|
|
|
16
27
|
|
|
17
28
|
- Publish only from release refs, print Node/npm versions, and publish with `--provenance --access public` so npm Trusted Publishing uses the same known-good path as `pi-spotify-widget`.
|
|
18
29
|
- Release instructions now use `npm version patch --no-git-tag-version` so local tags do not race the auto-release workflow.
|
|
19
|
-
|
|
20
|
-
## 0.3.0
|
|
21
|
-
|
|
22
|
-
- Use `agent_start` for persistent preselect; `session_start` was not a Pi extension event.
|
|
23
|
-
- Persist fallback state from assistant error messages on `turn_end`, covering provider SDK errors that do not emit `after_provider_response`.
|
|
24
|
-
|
|
25
|
-
## 0.2.0
|
|
26
|
-
|
|
27
|
-
- Add persistent failover state. A matching provider failure now writes `model-fallback/state.json`, and future sessions preselect the fallback model before the first provider call until the cooldown expires.
|
|
28
|
-
- Parse retry/reset headers when available; otherwise default 429 cooldown to 72h and 5xx cooldown to 10m.
|
|
29
|
-
- `/model-fallback:reset` now clears persistent state.
|
|
30
|
-
|
|
31
|
-
All notable changes to this project will be documented in this file.
|
|
32
|
-
|
|
33
|
-
This project follows semantic versioning.
|
|
34
|
-
|
|
35
|
-
## [0.1.3] - 2026-06-19
|
|
36
|
-
|
|
37
|
-
### Changed
|
|
38
|
-
|
|
39
|
-
- CONTRIBUTING now reminds maintainers to run `npm pack --dry-run` after changing which `docs/` files ship in the package.
|
|
40
|
-
|
|
41
|
-
### Fixed
|
|
42
|
-
|
|
43
|
-
- README License section no longer ends with a literal `\n` placeholder.
|
|
44
|
-
|
|
45
|
-
## [0.1.2] - 2026-06-04
|
|
46
|
-
|
|
47
|
-
### Changed
|
|
48
|
-
|
|
49
|
-
- README and `docs/template-checklist.md` now follow the Pi OSS minimal-docs policy: `docs/` is optional, with explicit post-generation cleanup for template bootstrap docs.
|
|
50
|
-
- Template bootstrap docs (`github-template.md`, `repository-settings.md`, `typescript.md`) are labeled for delete-or-merge after setup.
|
|
51
|
-
|
|
52
|
-
## [0.1.1] - 2026-06-01
|
|
53
|
-
|
|
54
|
-
### Changed
|
|
55
|
-
|
|
56
|
-
- Publish workflow now supports npm publishing on merged package version bumps in addition to tags, releases, and manual dispatch.
|
|
57
|
-
- Publish workflow now installs a current npm CLI so npm Trusted Publishing OIDC is supported.
|
|
58
|
-
- CI and publish workflow commands no longer include literal trailing `\\n` text.
|
|
59
|
-
|
|
60
|
-
## [0.1.0] - YYYY-MM-DD
|
|
61
|
-
|
|
62
|
-
### Added
|
|
63
|
-
|
|
64
|
-
- Initial Pi package template.
|
|
65
|
-
- Example extension, Agent Skill, prompt, and theme.
|
|
66
|
-
- CI and npm Trusted Publishing workflow.
|
|
67
|
-
|
|
30
|
+
|
|
31
|
+
## 0.3.0
|
|
32
|
+
|
|
33
|
+
- Use `agent_start` for persistent preselect; `session_start` was not a Pi extension event.
|
|
34
|
+
- Persist fallback state from assistant error messages on `turn_end`, covering provider SDK errors that do not emit `after_provider_response`.
|
|
35
|
+
|
|
36
|
+
## 0.2.0
|
|
37
|
+
|
|
38
|
+
- Add persistent failover state. A matching provider failure now writes `model-fallback/state.json`, and future sessions preselect the fallback model before the first provider call until the cooldown expires.
|
|
39
|
+
- Parse retry/reset headers when available; otherwise default 429 cooldown to 72h and 5xx cooldown to 10m.
|
|
40
|
+
- `/model-fallback:reset` now clears persistent state.
|
|
41
|
+
|
|
42
|
+
All notable changes to this project will be documented in this file.
|
|
43
|
+
|
|
44
|
+
This project follows semantic versioning.
|
|
45
|
+
|
|
46
|
+
## [0.1.3] - 2026-06-19
|
|
47
|
+
|
|
48
|
+
### Changed
|
|
49
|
+
|
|
50
|
+
- CONTRIBUTING now reminds maintainers to run `npm pack --dry-run` after changing which `docs/` files ship in the package.
|
|
51
|
+
|
|
52
|
+
### Fixed
|
|
53
|
+
|
|
54
|
+
- README License section no longer ends with a literal `\n` placeholder.
|
|
55
|
+
|
|
56
|
+
## [0.1.2] - 2026-06-04
|
|
57
|
+
|
|
58
|
+
### Changed
|
|
59
|
+
|
|
60
|
+
- README and `docs/template-checklist.md` now follow the Pi OSS minimal-docs policy: `docs/` is optional, with explicit post-generation cleanup for template bootstrap docs.
|
|
61
|
+
- Template bootstrap docs (`github-template.md`, `repository-settings.md`, `typescript.md`) are labeled for delete-or-merge after setup.
|
|
62
|
+
|
|
63
|
+
## [0.1.1] - 2026-06-01
|
|
64
|
+
|
|
65
|
+
### Changed
|
|
66
|
+
|
|
67
|
+
- Publish workflow now supports npm publishing on merged package version bumps in addition to tags, releases, and manual dispatch.
|
|
68
|
+
- Publish workflow now installs a current npm CLI so npm Trusted Publishing OIDC is supported.
|
|
69
|
+
- CI and publish workflow commands no longer include literal trailing `\\n` text.
|
|
70
|
+
|
|
71
|
+
## [0.1.0] - YYYY-MM-DD
|
|
72
|
+
|
|
73
|
+
### Added
|
|
74
|
+
|
|
75
|
+
- Initial Pi package template.
|
|
76
|
+
- Example extension, Agent Skill, prompt, and theme.
|
|
77
|
+
- CI and npm Trusted Publishing workflow.
|
package/README.md
CHANGED
|
@@ -90,6 +90,17 @@ Rule fields:
|
|
|
90
90
|
- `cooldownMs`: optional persistent fallback window
|
|
91
91
|
- `fallback`: target model Pi should switch to
|
|
92
92
|
|
|
93
|
+
Rules use first-match order: the first rule whose provider/model and status match wins. Put specific `matchModels` rules before broad `matchProviders` rules when they should take priority. `model_fallback_config validate`, `save`, `read`, and `status` report warning-only diagnostics when a later rule or model entry is completely shadowed by an earlier rule; `/model-fallback:status` also includes a concise warning summary for the current config. Warning-bearing config remains valid and can still be saved.
|
|
94
|
+
|
|
95
|
+
### Cooldowns
|
|
96
|
+
|
|
97
|
+
When a rule does not set `cooldownMs`, the extension uses these defaults:
|
|
98
|
+
|
|
99
|
+
- `429` → 72 hours
|
|
100
|
+
- `5xx` → 10 minutes
|
|
101
|
+
|
|
102
|
+
When the provider response includes `Retry-After` or `x-ratelimit-reset*` headers, those values override `cooldownMs` and the defaults for the persisted fallback window.
|
|
103
|
+
|
|
93
104
|
## State and paths
|
|
94
105
|
|
|
95
106
|
The extension stores:
|
|
@@ -103,7 +114,7 @@ If the package is installed project-locally and the current project references i
|
|
|
103
114
|
|
|
104
115
|
- Successful responses do nothing.
|
|
105
116
|
- Matching failures from `after_provider_response` can trigger fallback immediately.
|
|
106
|
-
- Assistant error messages parsed at `turn_end` can also persist fallback state for SDK/provider failures that do not emit the normal response hook.
|
|
117
|
+
- Assistant error messages parsed at `turn_end` can also persist fallback state for SDK/provider failures that do not emit the normal response hook. Status extraction looks for HTTP-style tokens (for example `status 429`, `HTTP 503`, or `rate limit`) rather than any bare 3-digit number in the message.
|
|
107
118
|
- The failed request is not automatically replayed.
|
|
108
119
|
|
|
109
120
|
## Development
|
package/docs/usage.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Install:
|
|
4
4
|
|
|
5
5
|
```bash
|
|
6
|
-
pi install
|
|
6
|
+
pi install npm:pi-model-fallback
|
|
7
7
|
```
|
|
8
8
|
|
|
9
9
|
Default behavior: when a `zai/*` model receives `429`, `500`, `502`, `503`, or `504`, Pi switches to `deepseek/deepseek-v4-flash` for the next prompt.
|
|
@@ -15,4 +15,13 @@ Commands:
|
|
|
15
15
|
/model-fallback:reset
|
|
16
16
|
```
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
Rule order is first-match: the first rule whose provider/model and status match wins. Place specific `matchModels` rules before broad `matchProviders` rules when the model-specific fallback should take priority. `model_fallback_config validate`, `save`, `read`, and `status` expose warning-only details for completely shadowed later rules or model entries, and `/model-fallback:status` includes a concise current-config warning summary; these warnings do not make the config invalid.
|
|
19
|
+
|
|
20
|
+
Cooldown defaults when a rule omits `cooldownMs`:
|
|
21
|
+
|
|
22
|
+
- `429` → 72 hours
|
|
23
|
+
- `5xx` → 10 minutes
|
|
24
|
+
|
|
25
|
+
`Retry-After` and `x-ratelimit-reset*` response headers override `cooldownMs` and those defaults.
|
|
26
|
+
|
|
27
|
+
The failed request is not automatically replayed.
|
package/extensions/index.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
2
3
|
import { dirname, join, resolve } from "node:path";
|
|
3
4
|
import { fileURLToPath } from "node:url";
|
|
4
5
|
import { getAgentDir, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
5
6
|
import { Type } from "typebox";
|
|
6
7
|
import {
|
|
8
|
+
analyzeRuleWarnings,
|
|
7
9
|
defaultConfig,
|
|
8
10
|
findFallback,
|
|
9
11
|
modelKey,
|
|
@@ -11,8 +13,10 @@ import {
|
|
|
11
13
|
validateConfigShape,
|
|
12
14
|
type ModelFallbackConfig,
|
|
13
15
|
type ModelRef,
|
|
16
|
+
type RuleWarning,
|
|
14
17
|
} from "../lib/config.js";
|
|
15
|
-
import {
|
|
18
|
+
import { parseStatusFromErrorMessage } from "../lib/error-status.js";
|
|
19
|
+
import { emptyState, findActiveStateEntry, pruneExpiredState, readState, upsertStateEntry, validateStateShape, writeState, type FallbackState } from "../lib/state.js";
|
|
16
20
|
import { modelFallbackPaths, readConfig, writeConfig, type ModelFallbackPaths } from "../lib/storage.js";
|
|
17
21
|
|
|
18
22
|
const PACKAGE_ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
|
|
@@ -22,6 +26,7 @@ const DEFAULT_5XX_COOLDOWN_MS = 10 * 60 * 1000;
|
|
|
22
26
|
|
|
23
27
|
export default function modelFallback(pi: ExtensionAPI) {
|
|
24
28
|
let paths: ModelFallbackPaths = modelFallbackPaths(getAgentDir());
|
|
29
|
+
let pathsResolvedForCwd: string | undefined;
|
|
25
30
|
let config: ModelFallbackConfig | undefined;
|
|
26
31
|
let state: FallbackState | undefined;
|
|
27
32
|
let originalModelKey: string | undefined;
|
|
@@ -29,6 +34,8 @@ export default function modelFallback(pi: ExtensionAPI) {
|
|
|
29
34
|
let lastFallbackReason: string | undefined;
|
|
30
35
|
|
|
31
36
|
function syncPaths(ctx: ExtensionContext): void {
|
|
37
|
+
if (pathsResolvedForCwd === ctx.cwd) return;
|
|
38
|
+
pathsResolvedForCwd = ctx.cwd;
|
|
32
39
|
paths = shouldUseProjectLocalState(ctx) ? modelFallbackPaths(join(ctx.cwd, ".pi")) : modelFallbackPaths(getAgentDir());
|
|
33
40
|
}
|
|
34
41
|
|
|
@@ -46,8 +53,18 @@ export default function modelFallback(pi: ExtensionAPI) {
|
|
|
46
53
|
async function loadState(ctx: ExtensionContext): Promise<FallbackState | undefined> {
|
|
47
54
|
try {
|
|
48
55
|
syncPaths(ctx);
|
|
49
|
-
|
|
50
|
-
|
|
56
|
+
const statePath = paths.state;
|
|
57
|
+
if (!existsSync(statePath)) {
|
|
58
|
+
state = emptyState();
|
|
59
|
+
return state;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const before = validateStateShape(JSON.parse(await readFile(statePath, "utf8")));
|
|
63
|
+
const after = pruneExpiredState(before);
|
|
64
|
+
state = after;
|
|
65
|
+
if (JSON.stringify(before) !== JSON.stringify(after)) {
|
|
66
|
+
await writeState(statePath, after);
|
|
67
|
+
}
|
|
51
68
|
return state;
|
|
52
69
|
} catch (error) {
|
|
53
70
|
ctx.ui.notify(`Model fallback state error: ${errorMessage(error)}`, "warning");
|
|
@@ -173,7 +190,7 @@ export default function modelFallback(pi: ExtensionAPI) {
|
|
|
173
190
|
handler: async (_args, ctx) => {
|
|
174
191
|
const loaded = config ?? (await loadConfig(ctx)) ?? defaultConfig();
|
|
175
192
|
const loadedState = state ?? (await loadState(ctx)) ?? { version: 1 as const, entries: [] };
|
|
176
|
-
ctx.ui.notify(formatStatus(loaded, loadedState, paths.config, paths.state, activeFallbackKey, originalModelKey, lastFallbackReason), "info");
|
|
193
|
+
ctx.ui.notify(formatStatus(loaded, loadedState, paths.config, paths.state, analyzeRuleWarnings(loaded), activeFallbackKey, originalModelKey, lastFallbackReason), "info");
|
|
177
194
|
},
|
|
178
195
|
});
|
|
179
196
|
|
|
@@ -227,25 +244,24 @@ export default function modelFallback(pi: ExtensionAPI) {
|
|
|
227
244
|
if (params.action === "read" || params.action === "status") {
|
|
228
245
|
const current = await readConfig(paths.config);
|
|
229
246
|
const currentState = await readState(paths.state);
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
lastFallbackReason,
|
|
237
|
-
});
|
|
247
|
+
const warnings = analyzeRuleWarnings(current);
|
|
248
|
+
const details = configToolDetails(paths.config, paths.state, currentState, warnings, activeFallbackKey, originalModelKey, lastFallbackReason);
|
|
249
|
+
const text = params.action === "status"
|
|
250
|
+
? formatStatus(current, currentState, paths.config, paths.state, warnings, activeFallbackKey, originalModelKey, lastFallbackReason)
|
|
251
|
+
: JSON.stringify(current, null, 2);
|
|
252
|
+
return textResult(text, details);
|
|
238
253
|
}
|
|
239
254
|
if (!params.configJson) throw new Error("configJson is required for validate/save.");
|
|
240
255
|
const nextConfig = validateConfigShape(JSON.parse(params.configJson));
|
|
256
|
+
const warnings = analyzeRuleWarnings(nextConfig);
|
|
241
257
|
validateRegisteredModels(ctx, nextConfig);
|
|
242
|
-
if (params.action === "validate") return textResult("Config is valid.", { config: nextConfig });
|
|
243
|
-
if (!ctx.hasUI) return textResult("Config not saved: confirmation UI is unavailable.", { configPath: paths.config });
|
|
244
|
-
const ok = await ctx.ui.confirm("Save model fallback config?", summarizeConfig(nextConfig));
|
|
245
|
-
if (!ok) return textResult("Config not saved.", { configPath: paths.config });
|
|
258
|
+
if (params.action === "validate") return textResult(configResultText("Config is valid.", warnings), { config: nextConfig, ...warningDetails(warnings) });
|
|
259
|
+
if (!ctx.hasUI) return textResult("Config not saved: confirmation UI is unavailable.", { configPath: paths.config, ...warningDetails(warnings) });
|
|
260
|
+
const ok = await ctx.ui.confirm("Save model fallback config?", summarizeConfig(nextConfig, warnings));
|
|
261
|
+
if (!ok) return textResult("Config not saved.", { configPath: paths.config, ...warningDetails(warnings) });
|
|
246
262
|
await writeConfig(paths.config, nextConfig);
|
|
247
263
|
config = nextConfig;
|
|
248
|
-
return textResult("Config saved.", { configPath: paths.config });
|
|
264
|
+
return textResult(configResultText("Config saved.", warnings), { configPath: paths.config, ...warningDetails(warnings) });
|
|
249
265
|
},
|
|
250
266
|
});
|
|
251
267
|
}
|
|
@@ -284,6 +300,7 @@ function formatStatus(
|
|
|
284
300
|
state: FallbackState,
|
|
285
301
|
configPath: string,
|
|
286
302
|
statePath: string,
|
|
303
|
+
warnings: RuleWarning[],
|
|
287
304
|
activeFallbackKey?: string,
|
|
288
305
|
originalModelKey?: string,
|
|
289
306
|
reason?: string,
|
|
@@ -293,6 +310,7 @@ function formatStatus(
|
|
|
293
310
|
`Config: ${configPath}`,
|
|
294
311
|
`State: ${statePath}`,
|
|
295
312
|
`Rules: ${config.rules.length}`,
|
|
313
|
+
`Config warnings: ${warningSummary(warnings)}`,
|
|
296
314
|
`Persistent entries: ${state.entries.length}`,
|
|
297
315
|
...state.entries.map((entry) => `- ${modelRefKey(entry.source)} -> ${modelRefKey(entry.fallback)} until ${entry.until} (${entry.status})`),
|
|
298
316
|
`Active fallback: ${activeFallbackKey ?? "none"}`,
|
|
@@ -301,13 +319,53 @@ function formatStatus(
|
|
|
301
319
|
].join("\n");
|
|
302
320
|
}
|
|
303
321
|
|
|
304
|
-
function summarizeConfig(config: ModelFallbackConfig): string {
|
|
322
|
+
function summarizeConfig(config: ModelFallbackConfig, warnings: RuleWarning[]): string {
|
|
305
323
|
return [
|
|
306
324
|
`enabled: ${config.enabled}`,
|
|
307
325
|
...config.rules.map((rule) => `${rule.name ?? "rule"}: ${rule.matchProviders?.join(",") ?? "models"} -> ${modelRefKey(rule.fallback)}`),
|
|
326
|
+
`warnings: ${warningSummary(warnings)}`,
|
|
308
327
|
].join("\n");
|
|
309
328
|
}
|
|
310
329
|
|
|
330
|
+
function configToolDetails(
|
|
331
|
+
configPath: string,
|
|
332
|
+
statePath: string,
|
|
333
|
+
state: FallbackState,
|
|
334
|
+
warnings: RuleWarning[],
|
|
335
|
+
activeFallbackKey?: string,
|
|
336
|
+
originalModelKey?: string,
|
|
337
|
+
lastFallbackReason?: string,
|
|
338
|
+
): Record<string, unknown> {
|
|
339
|
+
return {
|
|
340
|
+
configPath,
|
|
341
|
+
statePath,
|
|
342
|
+
state,
|
|
343
|
+
activeFallbackKey,
|
|
344
|
+
originalModelKey,
|
|
345
|
+
lastFallbackReason,
|
|
346
|
+
...warningDetails(warnings),
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function warningDetails(warnings: RuleWarning[]): Record<string, unknown> {
|
|
351
|
+
return {
|
|
352
|
+
warningCount: warnings.length,
|
|
353
|
+
warningSummary: warningSummary(warnings),
|
|
354
|
+
warnings,
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function configResultText(prefix: string, warnings: RuleWarning[]): string {
|
|
359
|
+
if (warnings.length === 0) return prefix;
|
|
360
|
+
return `${prefix} ${warningSummary(warnings)}`;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function warningSummary(warnings: RuleWarning[]): string {
|
|
364
|
+
if (warnings.length === 0) return "none";
|
|
365
|
+
if (warnings.length === 1) return `1 warning: ${warnings[0].message}`;
|
|
366
|
+
return `${warnings.length} warnings: ${warnings.map((warning) => warning.message).join("; ")}`;
|
|
367
|
+
}
|
|
368
|
+
|
|
311
369
|
function textResult(text: string, details?: Record<string, unknown>) {
|
|
312
370
|
return { content: [{ type: "text" as const, text }], details: details ?? {} };
|
|
313
371
|
}
|
|
@@ -343,7 +401,7 @@ function parseRetryAfter(value: string | undefined): number | undefined {
|
|
|
343
401
|
return undefined;
|
|
344
402
|
}
|
|
345
403
|
|
|
346
|
-
function parseResetHeader(value: string | undefined): number | undefined {
|
|
404
|
+
function parseResetHeader(value: string | undefined): number | undefined {
|
|
347
405
|
if (!value) return undefined;
|
|
348
406
|
const trimmed = value.trim();
|
|
349
407
|
const numeric = Number(trimmed);
|
|
@@ -354,17 +412,10 @@ function parseResetHeader(value: string | undefined): number | undefined {
|
|
|
354
412
|
}
|
|
355
413
|
const dateMs = Date.parse(trimmed);
|
|
356
414
|
if (!Number.isNaN(dateMs) && dateMs > Date.now()) return dateMs - Date.now();
|
|
357
|
-
return undefined;
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
function
|
|
361
|
-
const match = message.match(/\b([1-5][0-9]{2})\b/);
|
|
362
|
-
if (!match) return undefined;
|
|
363
|
-
const status = Number(match[1]);
|
|
364
|
-
return Number.isInteger(status) ? status : undefined;
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
function modelToRef(model: { provider: string; id: string }): ModelRef {
|
|
415
|
+
return undefined;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function modelToRef(model: { provider: string; id: string }): ModelRef {
|
|
368
419
|
return { provider: model.provider, model: model.id };
|
|
369
420
|
}
|
|
370
421
|
|
package/lib/config.ts
CHANGED
|
@@ -29,6 +29,28 @@ export interface MatchedFallback {
|
|
|
29
29
|
fallback: ModelRef;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
export type RuleWarningCode = "provider_rule_shadows_model" | "shadowed_rule" | "duplicate_rule";
|
|
33
|
+
|
|
34
|
+
export interface RuleWarning {
|
|
35
|
+
severity: "warning";
|
|
36
|
+
code: RuleWarningCode;
|
|
37
|
+
message: string;
|
|
38
|
+
ruleIndex: number;
|
|
39
|
+
ruleName?: string;
|
|
40
|
+
shadowedByRuleIndex: number;
|
|
41
|
+
shadowedByRuleName?: string;
|
|
42
|
+
statuses: number[];
|
|
43
|
+
matchProviders?: string[];
|
|
44
|
+
matchModels?: ModelRef[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface NormalizedRuleForWarnings {
|
|
48
|
+
providers: Set<string>;
|
|
49
|
+
modelKeys: Set<string>;
|
|
50
|
+
models: ModelRef[];
|
|
51
|
+
statuses: Set<number>;
|
|
52
|
+
}
|
|
53
|
+
|
|
32
54
|
export function defaultConfig(): ModelFallbackConfig {
|
|
33
55
|
return {
|
|
34
56
|
version: CONFIG_VERSION,
|
|
@@ -68,6 +90,43 @@ export function findFallback(config: ModelFallbackConfig, model: Pick<Model<any>
|
|
|
68
90
|
return undefined;
|
|
69
91
|
}
|
|
70
92
|
|
|
93
|
+
export function analyzeRuleWarnings(config: ModelFallbackConfig): RuleWarning[] {
|
|
94
|
+
const normalizedRules = config.rules.map((rule) => normalizeRuleForWarnings(rule));
|
|
95
|
+
const warnings: RuleWarning[] = [];
|
|
96
|
+
|
|
97
|
+
rules:
|
|
98
|
+
for (let ruleIndex = 1; ruleIndex < config.rules.length; ruleIndex += 1) {
|
|
99
|
+
const laterRule = config.rules[ruleIndex];
|
|
100
|
+
const later = normalizedRules[ruleIndex];
|
|
101
|
+
const reportedModelShadowKeys = new Set<string>();
|
|
102
|
+
|
|
103
|
+
for (let earlierIndex = 0; earlierIndex < ruleIndex; earlierIndex += 1) {
|
|
104
|
+
const earlierRule = config.rules[earlierIndex];
|
|
105
|
+
const earlier = normalizedRules[earlierIndex];
|
|
106
|
+
|
|
107
|
+
if (setsEqual(earlier.statuses, later.statuses) && ruleScopesEqual(earlier, later)) {
|
|
108
|
+
warnings.push(duplicateRuleWarning(laterRule, ruleIndex, later, earlierRule, earlierIndex));
|
|
109
|
+
continue rules;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (statusSetCovers(earlier.statuses, later.statuses) && ruleScopeCovers(earlier, later)) {
|
|
113
|
+
warnings.push(scopeShadowWarning(laterRule, ruleIndex, later, earlierRule, earlierIndex, earlier));
|
|
114
|
+
continue rules;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (earlier.providers.size === 0 || later.models.length === 0 || !statusSetCovers(earlier.statuses, later.statuses)) continue;
|
|
118
|
+
|
|
119
|
+
const shadowedModels = later.models.filter((entry) => earlier.providers.has(entry.provider) && !reportedModelShadowKeys.has(modelRefKey(entry)));
|
|
120
|
+
if (shadowedModels.length === 0) continue;
|
|
121
|
+
for (const entry of shadowedModels) reportedModelShadowKeys.add(modelRefKey(entry));
|
|
122
|
+
|
|
123
|
+
warnings.push(providerModelShadowWarning(laterRule, ruleIndex, later, earlierRule, earlierIndex, shadowedModels));
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return warnings;
|
|
128
|
+
}
|
|
129
|
+
|
|
71
130
|
export function modelRefKey(ref: ModelRef): string {
|
|
72
131
|
return `${ref.provider}/${ref.model}`;
|
|
73
132
|
}
|
|
@@ -100,6 +159,128 @@ function statusesFor(rule: FallbackRule): Set<number> {
|
|
|
100
159
|
return new Set(rule.statuses && rule.statuses.length > 0 ? rule.statuses : DEFAULT_FALLBACK_STATUSES);
|
|
101
160
|
}
|
|
102
161
|
|
|
162
|
+
function normalizeRuleForWarnings(rule: FallbackRule): NormalizedRuleForWarnings {
|
|
163
|
+
const modelsByKey = new Map((rule.matchModels ?? []).map((entry) => [modelRefKey(entry), entry]));
|
|
164
|
+
return {
|
|
165
|
+
providers: new Set(rule.matchProviders ?? []),
|
|
166
|
+
modelKeys: new Set(modelsByKey.keys()),
|
|
167
|
+
models: [...modelsByKey.values()],
|
|
168
|
+
statuses: statusesFor(rule),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function ruleScopesEqual(left: NormalizedRuleForWarnings, right: NormalizedRuleForWarnings): boolean {
|
|
173
|
+
return setsEqual(left.providers, right.providers) && setsEqual(left.modelKeys, right.modelKeys);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function ruleScopeCovers(earlier: NormalizedRuleForWarnings, later: NormalizedRuleForWarnings): boolean {
|
|
177
|
+
return setCovers(earlier.providers, later.providers) && later.models.every((entry) => earlier.providers.has(entry.provider) || earlier.modelKeys.has(modelRefKey(entry)));
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function duplicateRuleWarning(laterRule: FallbackRule, ruleIndex: number, later: NormalizedRuleForWarnings, earlierRule: FallbackRule, earlierIndex: number): RuleWarning {
|
|
181
|
+
return {
|
|
182
|
+
severity: "warning",
|
|
183
|
+
code: "duplicate_rule",
|
|
184
|
+
message: `${ruleLabel(laterRule, ruleIndex)} is shadowed by ${ruleLabel(earlierRule, earlierIndex)} because both rules match the same providers/models and statuses; the earlier rule wins first.`,
|
|
185
|
+
ruleIndex,
|
|
186
|
+
ruleName: laterRule.name,
|
|
187
|
+
shadowedByRuleIndex: earlierIndex,
|
|
188
|
+
shadowedByRuleName: earlierRule.name,
|
|
189
|
+
statuses: sortedNumbers(later.statuses),
|
|
190
|
+
matchProviders: sortedStrings(later.providers),
|
|
191
|
+
matchModels: sortedModels(later.models),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function scopeShadowWarning(
|
|
196
|
+
laterRule: FallbackRule,
|
|
197
|
+
ruleIndex: number,
|
|
198
|
+
later: NormalizedRuleForWarnings,
|
|
199
|
+
earlierRule: FallbackRule,
|
|
200
|
+
earlierIndex: number,
|
|
201
|
+
earlier: NormalizedRuleForWarnings,
|
|
202
|
+
): RuleWarning {
|
|
203
|
+
const providerCoveredModels = later.models.filter((entry) => earlier.providers.has(entry.provider));
|
|
204
|
+
const allModelsProviderCovered = later.providers.size === 0 && later.models.length > 0 && providerCoveredModels.length === later.models.length;
|
|
205
|
+
if (allModelsProviderCovered) return providerModelShadowWarning(laterRule, ruleIndex, later, earlierRule, earlierIndex, providerCoveredModels);
|
|
206
|
+
|
|
207
|
+
return {
|
|
208
|
+
severity: "warning",
|
|
209
|
+
code: "shadowed_rule",
|
|
210
|
+
message: `${ruleLabel(laterRule, ruleIndex)} is shadowed by ${ruleLabel(earlierRule, earlierIndex)} because the earlier rule covers its provider/model scope and statuses; the earlier rule wins first.`,
|
|
211
|
+
ruleIndex,
|
|
212
|
+
ruleName: laterRule.name,
|
|
213
|
+
shadowedByRuleIndex: earlierIndex,
|
|
214
|
+
shadowedByRuleName: earlierRule.name,
|
|
215
|
+
statuses: sortedNumbers(later.statuses),
|
|
216
|
+
matchProviders: sortedStrings(later.providers),
|
|
217
|
+
matchModels: sortedModels(later.models),
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function providerModelShadowWarning(
|
|
222
|
+
laterRule: FallbackRule,
|
|
223
|
+
ruleIndex: number,
|
|
224
|
+
later: NormalizedRuleForWarnings,
|
|
225
|
+
earlierRule: FallbackRule,
|
|
226
|
+
earlierIndex: number,
|
|
227
|
+
shadowedModels: ModelRef[],
|
|
228
|
+
): RuleWarning {
|
|
229
|
+
return {
|
|
230
|
+
severity: "warning",
|
|
231
|
+
code: "provider_rule_shadows_model",
|
|
232
|
+
message: `${ruleLabel(laterRule, ruleIndex)} has model match ${formatModelList(shadowedModels)} shadowed by ${ruleLabel(earlierRule, earlierIndex)} for statuses ${formatStatusList(later.statuses)}; the earlier provider-wide rule wins first.`,
|
|
233
|
+
ruleIndex,
|
|
234
|
+
ruleName: laterRule.name,
|
|
235
|
+
shadowedByRuleIndex: earlierIndex,
|
|
236
|
+
shadowedByRuleName: earlierRule.name,
|
|
237
|
+
statuses: sortedNumbers(later.statuses),
|
|
238
|
+
matchProviders: sortedStrings(new Set(shadowedModels.map((entry) => entry.provider))),
|
|
239
|
+
matchModels: sortedModels(shadowedModels),
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function statusSetCovers(earlier: Set<number>, later: Set<number>): boolean {
|
|
244
|
+
return setCovers(earlier, later);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function setsEqual<T>(left: Set<T>, right: Set<T>): boolean {
|
|
248
|
+
return left.size === right.size && setCovers(left, right);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function setCovers<T>(earlier: Set<T>, later: Set<T>): boolean {
|
|
252
|
+
for (const value of later) {
|
|
253
|
+
if (!earlier.has(value)) return false;
|
|
254
|
+
}
|
|
255
|
+
return true;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function sortedNumbers(values: Set<number>): number[] {
|
|
259
|
+
return [...values].sort((left, right) => left - right);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function sortedStrings(values: Set<string>): string[] | undefined {
|
|
263
|
+
const sorted = [...values].sort();
|
|
264
|
+
return sorted.length > 0 ? sorted : undefined;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function sortedModels(models: ModelRef[]): ModelRef[] | undefined {
|
|
268
|
+
const sorted = [...new Map(models.map((entry) => [modelRefKey(entry), entry])).values()].sort((left, right) => modelRefKey(left).localeCompare(modelRefKey(right)));
|
|
269
|
+
return sorted.length > 0 ? sorted : undefined;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function ruleLabel(rule: FallbackRule, index: number): string {
|
|
273
|
+
return rule.name ? `rules[${index}] (${rule.name})` : `rules[${index}]`;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function formatStatusList(statuses: Set<number>): string {
|
|
277
|
+
return sortedNumbers(statuses).join(", ");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function formatModelList(models: ModelRef[]): string {
|
|
281
|
+
return models.map(modelRefKey).sort().join(", ");
|
|
282
|
+
}
|
|
283
|
+
|
|
103
284
|
function readModelRef(value: unknown, path: string): ModelRef {
|
|
104
285
|
if (!isRecord(value)) throw new Error(`${path} must be an object.`);
|
|
105
286
|
return {
|
|
@@ -129,10 +310,10 @@ function readStatuses(value: unknown, path: string): number[] {
|
|
|
129
310
|
});
|
|
130
311
|
}
|
|
131
312
|
|
|
132
|
-
function readPositiveInteger(value: unknown, path: string): number {
|
|
133
|
-
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer.`);
|
|
134
|
-
return value;
|
|
135
|
-
}
|
|
313
|
+
function readPositiveInteger(value: unknown, path: string): number {
|
|
314
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer.`);
|
|
315
|
+
return value;
|
|
316
|
+
}
|
|
136
317
|
|
|
137
318
|
function readNonEmptyString(value: unknown, path: string): string {
|
|
138
319
|
if (typeof value !== "string" || value.trim() === "") throw new Error(`${path} must be a non-empty string.`);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export function parseStatusFromErrorMessage(message: string): number | undefined {
|
|
2
|
+
const patterns: RegExp[] = [
|
|
3
|
+
/\b(?:status|code|http)\s*[:\s]?\s*([1-5][0-9]{2})\b/i,
|
|
4
|
+
/\bHTTP\/\d(?:\.\d)?\s+([1-5][0-9]{2})\b/,
|
|
5
|
+
/\b(?:rate[\s-]?limit(?:ed)?|too many requests)[^0-9]{0,40}([45][0-9]{2})\b/i,
|
|
6
|
+
/\b([45][0-9]{2})\s+(?:error|too many requests|service unavailable|bad gateway|gateway timeout|internal server error)\b/i,
|
|
7
|
+
/\berror\s+([45][0-9]{2})\b/i,
|
|
8
|
+
];
|
|
9
|
+
|
|
10
|
+
for (const pattern of patterns) {
|
|
11
|
+
const match = message.match(pattern);
|
|
12
|
+
if (!match) continue;
|
|
13
|
+
const status = Number(match[1]);
|
|
14
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) return status;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-model-fallback",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.5",
|
|
4
4
|
"description": "Pi extension that switches to a fallback model after provider failures such as 429 rate limits.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -43,15 +43,17 @@
|
|
|
43
43
|
"access": "public"
|
|
44
44
|
},
|
|
45
45
|
"peerDependencies": {
|
|
46
|
+
"@earendil-works/pi-agent-core": "*",
|
|
46
47
|
"@earendil-works/pi-ai": "*",
|
|
47
48
|
"@earendil-works/pi-coding-agent": "*",
|
|
48
49
|
"@earendil-works/pi-tui": "*",
|
|
49
50
|
"typebox": "*"
|
|
50
51
|
},
|
|
51
52
|
"devDependencies": {
|
|
52
|
-
"@earendil-works/pi-
|
|
53
|
-
"@earendil-works/pi-
|
|
54
|
-
"@earendil-works/pi-
|
|
53
|
+
"@earendil-works/pi-agent-core": "^0.80.6",
|
|
54
|
+
"@earendil-works/pi-ai": "^0.80.6",
|
|
55
|
+
"@earendil-works/pi-coding-agent": "^0.80.6",
|
|
56
|
+
"@earendil-works/pi-tui": "^0.80.6",
|
|
55
57
|
"typebox": "latest",
|
|
56
58
|
"@types/node": "^26.0.0",
|
|
57
59
|
"typescript": "^6.0.3"
|