pi-provider-litellm 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -7
- package/package.json +1 -1
- package/src/backend-identity.ts +14 -4
- package/src/discover.ts +41 -16
- package/src/index.ts +191 -47
- package/src/mcp-tools.ts +35 -22
- package/src/model-groups.ts +32 -2
- package/src/public-catalog.ts +15 -8
- package/src/skills.ts +23 -76
package/README.md
CHANGED
|
@@ -154,6 +154,8 @@ Treat the configured LiteLLM proxy as trusted: Skills can add instructions to th
|
|
|
154
154
|
/model
|
|
155
155
|
```
|
|
156
156
|
|
|
157
|
+
To refresh catalogs on demand, run `/litellm-refresh` for every configured LiteLLM provider, or `/litellm-refresh <provider>` for one alias. It works with `PI_OFFLINE` set, which stops Pi's own `/model` refresh from contacting the network. It contacts only the configured proxies: `LITELLM_OFFLINE=1` and `LITELLM_DISCOVERY_TIMEOUT_MS=0` still disable it, and models.dev enrichment stays off under `PI_OFFLINE`. Each provider reports its model count, the failure, or missing credentials. With LiteLLM MCP enabled, the refresh also re-checks that provider's MCP tools.
|
|
158
|
+
|
|
157
159
|
Every request routed through a configured LiteLLM provider includes Pi's canonical session ID in the
|
|
158
160
|
`x-litellm-session-id` header. This groups Chat Completions, Responses, and native Messages requests in LiteLLM without
|
|
159
161
|
adding transport-specific fields to request bodies.
|
|
@@ -173,13 +175,17 @@ Native Messages authenticates with `x-api-key`; every transport carries the `x-l
|
|
|
173
175
|
| `LITELLM_GCLOUD_TOKEN_AUTH` | unset | If set to a non-empty value other than `0`, use Google Application Default Credentials as the LiteLLM bearer token source. This takes precedence over `LITELLM_API_KEY_HELPER` and `LITELLM_API_KEY` when no stored `/login litellm` credential exists. |
|
|
174
176
|
| `GOOGLE_APPLICATION_CREDENTIALS` | Google default ADC path | Optional path to an ADC JSON file used by `LITELLM_GCLOUD_TOKEN_AUTH`. If unset, the extension checks the default gcloud ADC locations. |
|
|
175
177
|
| `LITELLM_OFFLINE` | unset | If `1`, disable all model and MCP discovery, including post-login discovery; use cached models only when their stored canonical proxy root exactly matches the active credential root, including any path prefix. URL-standard host casing and default ports are canonicalized, but paths remain case-sensitive. |
|
|
178
|
+
| `PI_OFFLINE` | unset | Pi documents it as disabling automatic network activity, including model catalog refreshes. Through Pi 0.87.1, any set value (including `0` or an empty string) disables startup model discovery and `/model` network refreshes. Run `/litellm-refresh`, or unset it, to discover; cached models can still be used offline. |
|
|
176
179
|
| `LITELLM_DISCOVERY_TIMEOUT_MS` | `5000` | Background and explicit discovery fetch timeout in ms; `0` disables automatic discovery |
|
|
177
180
|
| `LITELLM_CLI_JWT_EXPIRATION_HOURS` | `24` | CLI SSO token lifetime fallback for older proxies whose poll response omits `expires_in`; mirror a non-default proxy setting locally |
|
|
178
|
-
| `LITELLM_VERBOSE_DISCOVERY` | unset | If `1`, enable progress messages during model and MCP discovery (login, refresh, startup), including MCP prepared/registered/dropped counts. Progress messages are off by default; MCP safety diagnostics (see below) are always reported regardless of this setting |
|
|
181
|
+
| `LITELLM_VERBOSE_DISCOVERY` | unset | If `1`, enable progress messages during model and MCP discovery (login, refresh, startup), including startup skip reasons, defaulted `/model/info` context windows, and MCP prepared/registered/dropped counts. Progress messages are off by default; MCP safety diagnostics (see below) are always reported regardless of this setting |
|
|
182
|
+
| `LITELLM_DEFAULT_CONTEXT_WINDOW` | `128000` | Context window assumed when neither LiteLLM's `/model/info` nor the catalog reports `max_input_tokens`. Set a positive integer for proxies whose custom aliases carry no metadata; an unset or unusable value keeps 128K |
|
|
179
183
|
| `LITELLM_MODELS_DEV` | unset | If `1`, enrich discovered metadata (limits, prices, effort lists) from models.dev, cached for 28 days in `litellm-models-dev.json`. Off by default because LiteLLM's `/model/info` is authoritative; use it when LiteLLM's model map lacks a model's metadata |
|
|
180
184
|
|
|
181
185
|
Only use a trusted `LITELLM_API_KEY_HELPER` or `!command`. Prefer an absolute executable path, keep secrets out of command arguments, and print only the token to stdout without logging it to stderr.
|
|
182
186
|
|
|
187
|
+
With `LITELLM_VERBOSE_DISCOVERY=1`, `/model/info` discovery reports published routes whose context window uses the fallback assumption (including wildcard expansions) on one line with the selected value, a count, and a sample of escaped route names. Each route is reported once per process. Explicit or authoritative catalog limits do not trigger it, even when numerically equal to the default. Set `model_info.max_input_tokens` on every deployment in the route to provide real limits; this diagnostic does not change discovery limits or routing.
|
|
188
|
+
|
|
183
189
|
`LITELLM_DISCOVERY_TIMEOUT_MS=0` disables automatic and explicit refresh model discovery. It does not replace the base URL or API key settings required to send requests when you are not using `/login litellm`.
|
|
184
190
|
|
|
185
191
|
### Google ADC token auth
|
|
@@ -201,11 +207,11 @@ If your LiteLLM proxy exposes MCP REST endpoints, this extension discovers tools
|
|
|
201
207
|
- `GET /mcp-rest/tools/list`
|
|
202
208
|
- `POST /mcp-rest/tools/call`
|
|
203
209
|
|
|
204
|
-
An MCP access denial pauses discovery until a successful `/login litellm` (SSO or API key). The pause survives restarts in `~/.pi/agent/litellm-mcp-pauses/`, scoped to the proxy, login, and headers with salted credential fingerprints. A new API key or header configuration has its own scope. Each explicit login creates a fresh opaque `litellmMcpSession` identifier in Pi's stored credential, even when the key is unchanged; automatic OAuth refresh preserves that identifier and its pause. Older Pi processes cannot pause the new login. Pause files contain no credentials or proxy error text. Model discovery and chat continue normally. Grant MCP access on the proxy before signing in again. An explicit `litellm.mcp.enabled: false` setting remains disabled after login.
|
|
210
|
+
An MCP access denial pauses discovery until a successful `/login litellm` (SSO or API key); an alias provider, which has no login, stays paused until its credentials, base URL, or headers change. The pause survives restarts in `~/.pi/agent/litellm-mcp-pauses/`, scoped to the proxy, login, and headers with salted credential fingerprints. A new API key or header configuration has its own scope. Each explicit login creates a fresh opaque `litellmMcpSession` identifier in Pi's stored credential, even when the key is unchanged; automatic OAuth refresh preserves that identifier and its pause. Older Pi processes cannot pause the new login. Pause files contain no credentials or proxy error text. Model discovery and chat continue normally. Grant MCP access on the proxy before signing in again. An explicit `litellm.mcp.enabled: false` setting remains disabled after login.
|
|
205
211
|
|
|
206
|
-
Each discovered tool is registered as a native Pi tool named `mcp_<server>_<tool>_<hash>`, at most 64 characters, using only `[a-z0-9_]`. The trailing 10-character hash is derived from the tool's identity (`server_id`, `server_name`, `name`) and is always present, so a name depends only on that tool and never on which other tools happen to be in the same catalog — adding or removing a sibling never renames a survivor. When the readable prefix would overflow 64 characters it is truncated from the right, so a long server name can leave little or none of the tool name visible; the hash is what distinguishes such tools. Tools from different servers therefore never overwrite each other, and exact duplicate identities are registered once.
|
|
212
|
+
Each discovered tool is registered as a native Pi tool named `mcp_<server>_<tool>_<hash>`, at most 64 characters, using only `[a-z0-9_]`. The trailing 10-character hash is derived from the tool's identity (`server_id`, `server_name`, `name`) and is always present, so a name depends only on that tool and never on which other tools happen to be in the same catalog — adding or removing a sibling never renames a survivor. When the readable prefix would overflow 64 characters it is truncated from the right, so a long server name can leave little or none of the tool name visible; the hash is what distinguishes such tools. Tools from different servers therefore never overwrite each other, and exact duplicate identities are registered once. Each alias provider discovers its own catalog: its tools are named `mcp_<alias>_<server>_<tool>_<hash>`, with the alias name also folded into the hash, and are called with that alias's credentials, so two providers exposing the same server never replace each other's tools. The default `litellm` provider keeps unprefixed names, and diagnostics for an alias read `LiteLLM MCP ("<alias>"):`.
|
|
207
213
|
|
|
208
|
-
MCP discovery runs after Pi refreshes LiteLLM models
|
|
214
|
+
MCP discovery runs once for every configured provider before the first agent turn (so `-p` and other non-interactive modes get MCP tools), after Pi refreshes LiteLLM models, and after `/login litellm`; extension activation never waits for it. Discovery accepts at most a 5 MiB response and registers at most 512 tools. The response must be a JSON array of tools or an object with a `tools` array; an `unexpected_error` or unknown non-null `error` in the object envelope, an object without `tools`, and every other shape are reported as discovery failures rather than as empty catalogs. Failure reports use extension-owned text and never include LiteLLM's proxy-supplied tag or `message`. A `partial_failure` envelope keeps and registers the returned tools while reporting one bounded generated diagnostic. Each tool is isolated during normalization, so one bad tool never hides valid siblings. A tool is accepted only when its schema has a root of exactly `"type": "object"` (a bare `properties` object with no `type`, or a type union such as `["object","null"]`, is refused), plain-object `properties`, at most 16 levels of JSON nesting (about eight nested schema objects when each level uses `properties`), and a serialized size of at most 64 KiB. Descriptions are limited to 4 KiB, and tool labels and result metadata to 256 bytes, each with a truncation marker.
|
|
209
215
|
|
|
210
216
|
### Which schema a tool ends up with
|
|
211
217
|
|
|
@@ -257,6 +263,8 @@ If your LiteLLM proxy exposes `/claude-code/marketplace.json`, enabled skills ar
|
|
|
257
263
|
- `litellm_skill_create`
|
|
258
264
|
- `litellm_skill_delete`
|
|
259
265
|
|
|
266
|
+
`litellm_skill_create` needs Skill Hub `sourceJson` metadata and a proxy that exposes `/claude-code/plugins`. It does not create skills from code: LiteLLM's `POST /v1/skills` is Anthropic's multipart Skills API.
|
|
267
|
+
|
|
260
268
|
## Mocked LiteLLM smoke workflow
|
|
261
269
|
|
|
262
270
|
The `LiteLLM Smoke` GitHub Actions workflow starts VidaiMock and a real LiteLLM proxy on the runner. LiteLLM exposes route-distinct Chat, Responses, native Messages, and mixed-deployment models whose upstreams are served by VidaiMock. The smoke runner discovers those models through LiteLLM, asserts each model's expected API, exercises `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`, verifies the expected `x-litellm-response-cost` behavior, and proves endpoint coverage from captured LiteLLM request logs rather than response text.
|
|
@@ -296,7 +304,9 @@ Dynamic catalogs are persisted by Pi in `~/.pi/agent/models-store.json`. Credent
|
|
|
296
304
|
|
|
297
305
|
LiteLLM's `/model/info` is the metadata authority, so models.dev enrichment is an opt-in escape hatch. With `LITELLM_MODELS_DEV=1`, for every genuine `/model/info` row, including deployment details fetched through `/health`, the extension requests `https://models.dev/api.json` for enrichment and caches the result for 28 days in `~/.pi/agent/litellm-models-dev.json`. Without it, discovery neither requests models.dev nor reads that cache; Pi's own catalog still applies. Health-only entries without deployment details use the bounded Pi catalog lookup without models.dev enrichment. `PI_OFFLINE` suppresses activation-time discovery and the models.dev request. `LITELLM_OFFLINE=1` also disables LiteLLM discovery; direct discovery callers use only an existing models.dev cache and do not refresh it.
|
|
298
306
|
|
|
299
|
-
Opening `/model` refreshes configured provider catalogs in the background using Pi's native model lifecycle.
|
|
307
|
+
Opening `/model` refreshes configured provider catalogs in the background using Pi's native model lifecycle when network discovery is enabled.
|
|
308
|
+
|
|
309
|
+
Pi documents `PI_OFFLINE` and `--offline` as disabling automatic network activity, including model catalog refreshes. Pi 0.87.1 adopted that wording to resolve [earendil-works/pi#8684](https://github.com/earendil-works/pi/issues/8684), without changing the behavior. Pi still checks only whether the variable is set, so `PI_OFFLINE=0` and an empty value also disable model refreshes. With `LITELLM_VERBOSE_DISCOVERY=1`, disabled startup discovery reports its reason, for example `startup discovery skipped (PI_OFFLINE)`. To discover models while keeping `PI_OFFLINE`, run `/litellm-refresh`. To allow all discovery for one invocation, use `env -u PI_OFFLINE pi` and do not pass `--offline`. If you only want to disable Pi's version check, use `PI_SKIP_VERSION_CHECK=1` instead; it does not apply offline mode's other network restrictions. See [#155](https://github.com/balcsida/pi-provider-litellm/issues/155).
|
|
300
310
|
|
|
301
311
|
### Model host enforcement
|
|
302
312
|
|
|
@@ -319,7 +329,7 @@ Each distinct availability diagnostic is written once per session on stderr. `LI
|
|
|
319
329
|
|
|
320
330
|
### Protocols and prompt caching
|
|
321
331
|
|
|
322
|
-
Discovery selects native Messages for proven Claude groups under the [Model transport](#model-transport) rules, Responses for OpenAI-family backends, and Chat Completions for other OpenAI-compatible backends.
|
|
332
|
+
Discovery selects native Messages for proven Claude groups under the [Model transport](#model-transport) rules, Responses for OpenAI-family backends, and Chat Completions for other OpenAI-compatible backends. Azure and Azure AI deployments stay on Chat Completions unless an explicit Responses mode or a `supported_endpoints` list containing `/v1/responses` authorizes Responses; `api_version` alone does not prove support. Azure is detected from the adapter field, an `azure/` or `azure_ai/` model prefix, or the reported provider; other adapters follow the backend family because LiteLLM bridges `/v1/responses` to Chat Completions when the provider has no native Responses config (`litellm/responses/main.py`, `_bridges_to_chat_completions`). A route whose model prefix and `custom_llm_provider` name different providers is treated as unidentified and stays on Chat Completions, so it also does not get the local 128-tool preflight. When `/model/info` supplies `supported_endpoints`, that list takes precedence, and `mode: "responses"` remains an explicit Responses signal. An evidence-free fallback entry—a bare `/v1/models` id or a `/health` entry with no detail row—takes its protocol from the Pi catalog entry for its id; both evidence-free fallback paths also inherit presentation metadata from the bounded catalog lookup. An unknown id stays on Chat Completions. The route name alone authorizes nothing. A concrete id expanded from wildcard `/model/info` routes inherits the deployment evidence of the wildcard route LiteLLM would select for it, with the requested id substituted into the row's `model` the way LiteLLM serves it; the id is omitted when that selected route is not a published chat-style route. Chat Completions and Responses use `<root>/v1`; native Messages uses the proxy root directly. All three protocols pass through the provider's host guard. When an evidence-free cached `(no metadata)` entry can be enriched from the Pi catalog, it is restored on Responses only for a catalog Responses model and otherwise on Chat Completions, never Anthropic Messages. A model supplied only through `models.json` whose `api` this provider does not implement takes Pi's global API fallback rather than this provider's host guard; the credential-root pin (see [Model host enforcement](#model-host-enforcement)) still keeps the credential on the active proxy.
|
|
323
333
|
|
|
324
334
|
`cacheControlFormat: "anthropic"` applies only to Chat Completions, where Pi adds Anthropic `cache_control` markers. The Responses transport has a different compatibility type and uses native `prompt_cache_key` (plus supported retention fields), so Responses models must not receive `cacheControlFormat`. Freshly discovered OpenAI-family models forced onto Chat Completions are rejected locally when a request contains more than 128 tools; route them through Responses or reduce enabled extensions. Moonshot request-side reasoning suppression follows LiteLLM's routing provider, not `model_info.base_model`; response-side `<think>` normalization follows discovered Kimi-family evidence. Opaque aliases retain display normalization without sending Moonshot-only parameters to Azure- or Bedrock-hosted Kimi models. Strict tool-message repair requires every deployment to identify Kimi; Gemini effort normalization likewise requires consistent backend-family evidence.
|
|
325
335
|
|
|
@@ -331,7 +341,7 @@ LiteLLM may load-balance one public `model_name` across deployments with differe
|
|
|
331
341
|
|
|
332
342
|
- Responses is selected only when every deployment supports it under the protocol rules above; a deployment that requires Chat keeps the group on Chat.
|
|
333
343
|
- Vision and reasoning are advertised only when every routable deployment resolves them as supported. An explicit `supports_reasoning: false` suppresses all thinking controls, even when catalog or router effort metadata exists. Catalog thinking controls are intersected per level: a level is exposed only when every deployment maps it to the same value, while disagreement or absence becomes an explicit denial. Catalog maps are not complete effort lists, so absence has one exception: an omitted `minimal`, `low`, `medium` or `high` counts as Pi's default spelling of that level. An omitted `off`, `xhigh` or `max` still differs from an explicit mapping. Reasoning selectors additionally require an accepted wire carrier on every deployment. Public effort lists are intersected, explicit LiteLLM denials win, and `xhigh`/`max` need unanimous explicit support. Null or absent standard-effort flags have no opinion. A deployment that declares `model_info.reasoning_effort_levels` answers every level from that list, ahead of its per-level flags, as LiteLLM itself reads it.
|
|
334
|
-
- Context and output limits use the minimum resolved value across deployments.
|
|
344
|
+
- Context and output limits use the minimum resolved value across deployments. Pi catalogue lookup prefers the deployment's provider-specific entry before falling back to the generic vendor entry; Azure and Azure AI therefore retain Azure-specific limits rather than being capped by a different OpenAI catalogue entry. When models.dev supplies a sparse vendor record, missing fields still use the provider-specific Pi catalogue.
|
|
335
345
|
- Each displayed base-price field uses the maximum only when every deployment resolves that field; unresolved fields remain zero and the model name is suffixed with ` (incomplete metadata)`.
|
|
336
346
|
- When complete base pricing comes from the catalog under unanimous catalog authority, tiered pricing is the conservative worst-case envelope. Its usable finite non-negative thresholds are the sorted union of every deployment's thresholds, and each price field at each interval is the maximum applicable rate across all deployments, floored by each deployment's base rate. Differing tier breakpoints are therefore preserved rather than treated as incompatible. An explicit `/model/info` base-price field replaces catalog pricing for that field at every threshold; catalog tiers remain for unaffected fields and are omitted only when every base-price field is explicit.
|
|
337
347
|
- Reduced route groups omit tier ladders when complete base-pricing or catalog authority is unavailable. Wildcard expansion retains every known tier from matching groups even when another match has incomplete metadata: suppressing a complete sibling's higher tier could understate the known worst-case rate. The ` (incomplete metadata)` marker remains because the resulting envelope is only as complete as the available ladders.
|
|
@@ -359,6 +369,7 @@ The development probe runs against minimized snapshots with `npm run probe:proxy
|
|
|
359
369
|
| Symptom | Likely cause |
|
|
360
370
|
|---|---|
|
|
361
371
|
| "no credentials" warning at startup | Env vars not set and no OAuth credential — run `/login litellm` |
|
|
372
|
+
| No models, or no new routes, while `PI_OFFLINE` is set | Pi skips every model network refresh, including the one `/model` starts. Run `/litellm-refresh`, or unset `PI_OFFLINE` (`PI_OFFLINE=0` is not enough) |
|
|
362
373
|
| `No models available` at startup, gone after a restart | A Pi startup race, not discovery — see [`No models available` at startup](#no-models-available-at-startup) |
|
|
363
374
|
| "discovered no models" | Proxy returned an empty list — check pi's startup log and verify `/model/info`, `/v1/models`, or `/health` responds |
|
|
364
375
|
| `/model/info` returning 401/403/404 | Expected behavior with virtual keys — extension falls back to `/v1/models` |
|
|
@@ -367,6 +378,7 @@ The development probe runs against minimized snapshots with `npm run probe:proxy
|
|
|
367
378
|
| A configured or restored model fails to complete but never leaks the credential | Its `api` is absent from the current provider catalog, so Pi used global API fallback with the credential root pinned. Open `/model` against the active proxy to repopulate the protocol — see [Model host enforcement](#model-host-enforcement) |
|
|
368
379
|
| `LiteLLM discovery: ... route group(s) have missing or conflicting deployment provider evidence` | One or more deployments lack resolvable backend authority or resolve to different providers or concrete catalog models. Add consistent `litellm_params.model`, `model_info.base_model`, or adapter metadata; catalog-derived limits, pricing, and reasoning metadata are withheld meanwhile. This may be intentional for cross-host routes such as Vertex/Bedrock Claude. |
|
|
369
380
|
| `LiteLLM discovery: ... route group(s) mix chat-style and explicitly incompatible deployment modes` | A public route can select both a chat-style deployment and an incompatible deployment such as an embedding model. Give the embedding deployment its own `model_name`. |
|
|
381
|
+
| `LiteLLM (...): a fallback served "<route>"` | A successful HTTP response reports a LiteLLM router fallback (`x-litellm-attempted-fallbacks`). `/model/info` does not expose fallbacks, so protocol and model handling follow the requested route's own deployments. If the primary and fallback groups support Chat Completions, set `model_info.supported_endpoints: ["/v1/chat/completions"]` on the primary to avoid Responses-to-Chat bridging. This warning does not change routing or fix upstream bridging defects. Warned once per provider and route until the extension reloads; shown in the UI, or stderr without a UI. |
|
|
370
382
|
| A model is marked ` (incomplete metadata)` | `/model/info` or `/health` did not provide enough authoritative metadata. Explicit fields remain usable, but unknown cost fields are shown as zero and route-name cache enrichment stays disabled. |
|
|
371
383
|
| `401 Token expired` | Set `LITELLM_API_KEY_HELPER`. |
|
|
372
384
|
| No models with gcloud auth | Verify `gcloud auth application-default login` has been run or set `GOOGLE_APPLICATION_CREDENTIALS` to an `authorized_user` ADC file |
|
package/package.json
CHANGED
package/src/backend-identity.ts
CHANGED
|
@@ -29,6 +29,10 @@ const PROVIDER_FAMILIES: Readonly<Record<string, BackendFamily>> = {
|
|
|
29
29
|
// Prefixes whose keyword match is trustworthy across the whole "prefix/modelId" string because
|
|
30
30
|
// the prefix itself names a vendor, not just some unrelated string that happens to contain one.
|
|
31
31
|
const KNOWN_VENDOR_PREFIXES = new Set([...Object.keys(PROVIDER_FAMILIES), "openai"]);
|
|
32
|
+
// First segments that are known to be model path, never a provider: Fireworks' account-scoped
|
|
33
|
+
// ids ("accounts/fireworks/models/x"). An allowlist fails closed: any other differing prefix,
|
|
34
|
+
// recognized provider or not, stays a conflict.
|
|
35
|
+
const MODEL_PATH_SEGMENTS = new Set(["accounts"]);
|
|
32
36
|
// `o\d` (OpenAI's o1/o3/o4-mini reasoning models) is only trustworthy at the start of the id or
|
|
33
37
|
// right after a provider path segment — an interior "-o1-" is as likely to be an unrelated
|
|
34
38
|
// product's own version marker (e.g. "custom-o1-clone").
|
|
@@ -62,14 +66,20 @@ export function resolveBackendIdentity(row: BackendIdentityRow): BackendIdentity
|
|
|
62
66
|
wireString(row.model_info?.base_model) ?? wireString(row.litellm_params?.model) ?? wireString(row.model_name);
|
|
63
67
|
if (!raw) return undefined;
|
|
64
68
|
|
|
65
|
-
|
|
69
|
+
let slash = raw.indexOf("/");
|
|
66
70
|
const prefix = slash > 0 ? raw.slice(0, slash).trim().toLowerCase() : undefined;
|
|
67
|
-
|
|
71
|
+
let prefixProvider = prefix && !GENERIC_ADAPTERS.has(prefix) ? prefix : undefined;
|
|
68
72
|
// `custom_llm_provider` is how LiteLLM routes an unprefixed model. It is provider evidence
|
|
69
|
-
// in its own right, so a prefix
|
|
73
|
+
// in its own right, so a differing prefix is a conflict, not a tiebreak. The exception is a
|
|
74
|
+
// known model-path segment: LiteLLM routes by custom_llm_provider, so Fireworks'
|
|
75
|
+
// "accounts/fireworks/models/x" is not a provider named "accounts".
|
|
70
76
|
const custom = wireString(row.litellm_params?.custom_llm_provider)?.toLowerCase();
|
|
71
77
|
const customProvider = custom && !GENERIC_ADAPTERS.has(custom) ? custom : undefined;
|
|
72
|
-
if (prefixProvider && customProvider && prefixProvider !== customProvider)
|
|
78
|
+
if (prefixProvider && customProvider && prefixProvider !== customProvider) {
|
|
79
|
+
if (!MODEL_PATH_SEGMENTS.has(prefixProvider)) return undefined;
|
|
80
|
+
prefixProvider = undefined;
|
|
81
|
+
slash = -1;
|
|
82
|
+
}
|
|
73
83
|
const provider = prefixProvider ?? customProvider;
|
|
74
84
|
const modelId = slash > 0 ? raw.slice(slash + 1) : raw;
|
|
75
85
|
// Scan the full "prefix/modelId" string only when the prefix is itself a whole known vendor
|
package/src/discover.ts
CHANGED
|
@@ -8,9 +8,10 @@ import {
|
|
|
8
8
|
catalogResolution,
|
|
9
9
|
closeSerializerPolicy,
|
|
10
10
|
conservativeCostTiers,
|
|
11
|
-
DEFAULT_CONTEXT_WINDOW,
|
|
12
11
|
DEFAULT_MAX_TOKENS,
|
|
12
|
+
defaultContextWindow,
|
|
13
13
|
hasMixedIncompatibleDeploymentModes,
|
|
14
|
+
isFallbackContextWindow,
|
|
14
15
|
type MessagesBackendCompat,
|
|
15
16
|
meetVendorCompat,
|
|
16
17
|
normalizedMode,
|
|
@@ -39,6 +40,7 @@ const DEFAULT_TIMEOUT_MS = 5000;
|
|
|
39
40
|
const HEALTH_DETAIL_CONCURRENCY = 8;
|
|
40
41
|
const reportedConflictingFamilyRoutes = new Set<string>();
|
|
41
42
|
const reportedWithheldRepairRoutes = new Set<string>();
|
|
43
|
+
const reportedDefaultedContextRoutes = new Set<string>();
|
|
42
44
|
interface HealthDeployment {
|
|
43
45
|
entry: ModelInfoEntry;
|
|
44
46
|
denyLevels: boolean;
|
|
@@ -148,16 +150,11 @@ function supportsResponses(entry: ModelInfoEntry): boolean {
|
|
|
148
150
|
/^azure(?:_ai)?\//.test(configuredModel ?? "") ||
|
|
149
151
|
reportedProvider === "azure" ||
|
|
150
152
|
reportedProvider === "azure_ai";
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
if (apiVersion != null && typeof apiVersion !== "string") return false;
|
|
157
|
-
const version = apiVersion?.trim();
|
|
158
|
-
if (!version) return true;
|
|
159
|
-
const date = version.match(/^(\d{4}-\d{2}-\d{2})(?:-preview)?$/)?.[1];
|
|
160
|
-
return date !== undefined && date >= "2025-03-01";
|
|
153
|
+
// An Azure API version describes the API surface, not whether this deployment
|
|
154
|
+
// serves Responses. Require the explicit mode/endpoint evidence above instead
|
|
155
|
+
// of promoting a working Chat route to an unavailable endpoint.
|
|
156
|
+
// Generic adapters remain eligible for LiteLLM's Responses-to-Chat bridge.
|
|
157
|
+
return !azureAdapter;
|
|
161
158
|
}
|
|
162
159
|
|
|
163
160
|
export function modelProtocol(modelId: string, modeOrEntry?: string | null | ModelInfoEntry): ModelProtocol {
|
|
@@ -231,7 +228,9 @@ export function enrichCachedModel(input: Model<Api>): Model<Api> {
|
|
|
231
228
|
}),
|
|
232
229
|
} as Model<Api>;
|
|
233
230
|
// Reduced deployment groups use a distinct marker; this sentinel remains
|
|
234
|
-
// exclusive to evidence-free fallback models that may be enriched safely.
|
|
231
|
+
// exclusive to evidence-free fallback models that may be enriched safely. A window is
|
|
232
|
+
// fallback evidence when it matches either default, so a cached entry still qualifies
|
|
233
|
+
// after an operator configures LITELLM_DEFAULT_CONTEXT_WINDOW.
|
|
235
234
|
if (
|
|
236
235
|
!model.name.endsWith(" (no metadata)") ||
|
|
237
236
|
model.reasoning ||
|
|
@@ -243,13 +242,15 @@ export function enrichCachedModel(input: Model<Api>): Model<Api> {
|
|
|
243
242
|
model.cost.cacheRead !== 0 ||
|
|
244
243
|
model.cost.cacheWrite !== 0 ||
|
|
245
244
|
model.cost.tiers !== undefined ||
|
|
246
|
-
model.contextWindow
|
|
245
|
+
!isFallbackContextWindow(model.contextWindow) ||
|
|
247
246
|
model.maxTokens !== DEFAULT_MAX_TOKENS
|
|
248
247
|
) {
|
|
249
248
|
return model;
|
|
250
249
|
}
|
|
251
250
|
const catalogModel = findCatalogModel(model.id);
|
|
252
|
-
|
|
251
|
+
// The gate above proved this window is an assumption, so it is re-derived rather than
|
|
252
|
+
// restored: the cache is seeded offline, where discovery never re-reads the setting.
|
|
253
|
+
if (!catalogModel) return { ...model, contextWindow: defaultContextWindow() };
|
|
253
254
|
// Match evidence-free discovery: only an explicit Responses catalog transport
|
|
254
255
|
// changes the protocol; all other catalog APIs continue through Chat.
|
|
255
256
|
const api = catalogModel.api === "openai-responses" ? "openai-responses" : "openai-completions";
|
|
@@ -597,6 +598,7 @@ function mapFromModelInfoGroup(
|
|
|
597
598
|
ambiguousRoutes?: string[];
|
|
598
599
|
conflictingFamilyRoutes?: string[];
|
|
599
600
|
withheldRepairRoutes?: string[];
|
|
601
|
+
defaultedContextRoutes?: Set<string>;
|
|
600
602
|
denyLevels?: boolean;
|
|
601
603
|
allowMessages?: boolean;
|
|
602
604
|
} = {},
|
|
@@ -618,6 +620,7 @@ function mapFromModelInfoGroup(
|
|
|
618
620
|
};
|
|
619
621
|
});
|
|
620
622
|
if (!reduced) return undefined;
|
|
623
|
+
if (reduced.contextWindowDefaulted) options.defaultedContextRoutes?.add(reduced.id);
|
|
621
624
|
if (reduced.catalogAuthorityAmbiguous) options.ambiguousRoutes?.push(reduced.id);
|
|
622
625
|
if (reduced.deploymentFamilies.includes("conflicting")) options.conflictingFamilyRoutes?.push(reduced.id);
|
|
623
626
|
const protocols = entries.map((entry) => modelProtocol(reduced.id, entry));
|
|
@@ -720,7 +723,7 @@ function mapFromModelsList(entry: ModelsListEntry): DiscoveredModel | undefined
|
|
|
720
723
|
}),
|
|
721
724
|
input: catalogModel?.input ?? ["text"],
|
|
722
725
|
cost: catalogModel?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
723
|
-
contextWindow: catalogModel?.contextWindow ??
|
|
726
|
+
contextWindow: catalogModel?.contextWindow ?? defaultContextWindow(),
|
|
724
727
|
maxTokens: catalogModel?.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
725
728
|
api,
|
|
726
729
|
litellmDiscoveryVersion: LITELLM_DISCOVERY_VERSION,
|
|
@@ -1034,6 +1037,7 @@ export async function discoverModels(
|
|
|
1034
1037
|
const incompatibleModeRoutes: string[] = [];
|
|
1035
1038
|
const conflictingFamilyRoutes: string[] = [];
|
|
1036
1039
|
const withheldRepairRoutes: string[] = [];
|
|
1040
|
+
const defaultedContextRoutes = new Set<string>();
|
|
1037
1041
|
const reducedGroups = [...groups.entries()].map(([route, group]) => {
|
|
1038
1042
|
if (hasMixedIncompatibleDeploymentModes(group)) incompatibleModeRoutes.push(route);
|
|
1039
1043
|
return {
|
|
@@ -1042,6 +1046,7 @@ export async function discoverModels(
|
|
|
1042
1046
|
ambiguousRoutes,
|
|
1043
1047
|
conflictingFamilyRoutes,
|
|
1044
1048
|
withheldRepairRoutes,
|
|
1049
|
+
defaultedContextRoutes,
|
|
1045
1050
|
}),
|
|
1046
1051
|
deploymentFamilies: group.map(deploymentFamily),
|
|
1047
1052
|
};
|
|
@@ -1097,7 +1102,27 @@ export async function discoverModels(
|
|
|
1097
1102
|
}
|
|
1098
1103
|
}
|
|
1099
1104
|
reportWithheldToolRepair(withheldRepairRoutes);
|
|
1100
|
-
|
|
1105
|
+
models = deduplicateModels(models);
|
|
1106
|
+
if (process.env.LITELLM_VERBOSE_DISCOVERY === "1") {
|
|
1107
|
+
const defaultedWildcards = wildcardRoutes.filter(({ route }) => defaultedContextRoutes.has(route));
|
|
1108
|
+
// Exact routes override wildcard templates; a tighter measured parent can also win.
|
|
1109
|
+
const defaulted = models.filter((model) =>
|
|
1110
|
+
groups.has(model.id)
|
|
1111
|
+
? defaultedContextRoutes.has(model.id)
|
|
1112
|
+
: defaultedWildcards.some(
|
|
1113
|
+
({ route, model: parent }) =>
|
|
1114
|
+
parent?.contextWindow === model.contextWindow && wildcardMatches(route, model.id),
|
|
1115
|
+
),
|
|
1116
|
+
);
|
|
1117
|
+
reportBoundedRoutes(
|
|
1118
|
+
reportedDefaultedContextRoutes,
|
|
1119
|
+
defaulted.map((model) => JSON.stringify(model.id)),
|
|
1120
|
+
(count) =>
|
|
1121
|
+
`LiteLLM discovery: ${count} route(s) default contextWindow to ${defaultContextWindow()}; ` +
|
|
1122
|
+
"set model_info.max_input_tokens on every deployment in each route",
|
|
1123
|
+
);
|
|
1124
|
+
}
|
|
1125
|
+
return { source: "model_info", models };
|
|
1101
1126
|
}
|
|
1102
1127
|
if (![401, 403, 404].includes(infoResult.status)) {
|
|
1103
1128
|
throw new Error(`/model/info returned ${infoResult.status}`);
|
package/src/index.ts
CHANGED
|
@@ -1643,8 +1643,10 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1643
1643
|
};
|
|
1644
1644
|
}
|
|
1645
1645
|
|
|
1646
|
-
|
|
1647
|
-
|
|
1646
|
+
// Each provider owns its own MCP catalog, keyed by provider name.
|
|
1647
|
+
const registeredMcpIdentities = new Map<string, string>();
|
|
1648
|
+
let mcpSeeded = false;
|
|
1649
|
+
const mcpRegistrations = new Map<string, Promise<void>>();
|
|
1648
1650
|
// Pi's registerTool throws only from assertActive(), whose staleness flag is set with `??=` and
|
|
1649
1651
|
// never cleared, so a refusal is fatal for this extension instance rather than a per-tool or
|
|
1650
1652
|
// retryable condition. Once seen, stop attempting registration here; a reload creates a fresh
|
|
@@ -1655,7 +1657,10 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1655
1657
|
const mcpPauseInMemory = new Set<string>();
|
|
1656
1658
|
let mcpPauseSalt: Buffer | undefined;
|
|
1657
1659
|
let mcpPausePersistent = true;
|
|
1658
|
-
|
|
1660
|
+
// Keyed by provider: only the default provider has /login, and its login must not cancel an
|
|
1661
|
+
// alias's in-flight registration, which login does not retry.
|
|
1662
|
+
const mcpLoginGenerations = new Map<string, number>();
|
|
1663
|
+
const mcpLoginGeneration = (name: string) => mcpLoginGenerations.get(name) ?? 0;
|
|
1659
1664
|
|
|
1660
1665
|
function getMcpPauseSalt(): Buffer {
|
|
1661
1666
|
if (!mcpPauseSalt) {
|
|
@@ -1724,8 +1729,8 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1724
1729
|
}
|
|
1725
1730
|
|
|
1726
1731
|
function resumeMcpDiscovery(): void {
|
|
1727
|
-
mcpLoginGeneration
|
|
1728
|
-
|
|
1732
|
+
mcpLoginGenerations.set(PROVIDER_NAME, mcpLoginGeneration(PROVIDER_NAME) + 1);
|
|
1733
|
+
registeredMcpIdentities.delete(PROVIDER_NAME);
|
|
1729
1734
|
// The new login ID selects a fresh scope; other processes may still use the old scopes.
|
|
1730
1735
|
}
|
|
1731
1736
|
|
|
@@ -1746,17 +1751,17 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1746
1751
|
});
|
|
1747
1752
|
}
|
|
1748
1753
|
|
|
1749
|
-
async function getRuntimeAuth(
|
|
1750
|
-
|
|
1754
|
+
async function getRuntimeAuth(
|
|
1755
|
+
ctx: ExtensionContext,
|
|
1756
|
+
definition = definitions[0]!,
|
|
1757
|
+
): Promise<LiteLLMRuntimeAuth | undefined> {
|
|
1758
|
+
const auth = await ctx.modelRegistry.getProviderAuth(definition.name);
|
|
1751
1759
|
if (!auth) return undefined;
|
|
1752
|
-
const provider = ctx.modelRegistry.getProvider(
|
|
1760
|
+
const provider = ctx.modelRegistry.getProvider(definition.name);
|
|
1753
1761
|
const apiKey = auth.auth.apiKey;
|
|
1754
1762
|
const baseUrl = cleanConfig(auth.auth.baseUrl) ?? cleanConfig(auth.env?.[ENV_BASE_URL]) ?? provider?.baseUrl;
|
|
1755
1763
|
if (!baseUrl || !apiKey) return undefined;
|
|
1756
|
-
const runtimeRoot = requireCredentialRoot(
|
|
1757
|
-
normalizeBaseUrl(baseUrl, definitions[0]?.allowInsecureHttp),
|
|
1758
|
-
PROVIDER_NAME,
|
|
1759
|
-
);
|
|
1764
|
+
const runtimeRoot = requireCredentialRoot(normalizeBaseUrl(baseUrl, definition.allowInsecureHttp), definition.name);
|
|
1760
1765
|
const headers = Object.fromEntries(
|
|
1761
1766
|
Object.entries(auth.auth.headers ?? provider?.headers ?? {}).filter(
|
|
1762
1767
|
(entry): entry is [string, string] => typeof entry[1] === "string",
|
|
@@ -1766,16 +1771,23 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1766
1771
|
baseUrl: runtimeRoot,
|
|
1767
1772
|
apiKey,
|
|
1768
1773
|
headers: Object.keys(headers).length > 0 ? headers : undefined,
|
|
1769
|
-
allowInsecureHttp:
|
|
1774
|
+
allowInsecureHttp: definition.allowInsecureHttp,
|
|
1770
1775
|
};
|
|
1771
1776
|
}
|
|
1772
1777
|
|
|
1778
|
+
function missingCredentials(definition: ProviderDefinition): string {
|
|
1779
|
+
const fix = definition.name === PROVIDER_NAME ? "Run /login litellm or set env vars" : "Set its baseUrl and apiKey";
|
|
1780
|
+
return `no credentials for ${definition.name}. ${fix}.`;
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
async function requireRuntimeAuth(ctx: ExtensionContext, definition = definitions[0]!): Promise<LiteLLMRuntimeAuth> {
|
|
1784
|
+
const auth = await getRuntimeAuth(ctx, definition);
|
|
1785
|
+
if (auth) return auth;
|
|
1786
|
+
throw new Error(missingCredentials(definition));
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1773
1789
|
async function resolveDefaultRuntimeAuth(ctx?: ExtensionContext): Promise<LiteLLMRuntimeAuth> {
|
|
1774
|
-
if (ctx?.modelRegistry)
|
|
1775
|
-
const auth = await getRuntimeAuth(ctx);
|
|
1776
|
-
if (auth) return auth;
|
|
1777
|
-
throw new Error("no credentials for litellm. Run /login litellm or set env vars.");
|
|
1778
|
-
}
|
|
1790
|
+
if (ctx?.modelRegistry) return requireRuntimeAuth(ctx);
|
|
1779
1791
|
if (!defaultRuntimeAuth) throw new Error("no credentials for litellm. Run /login litellm or set env vars.");
|
|
1780
1792
|
return defaultRuntimeAuth;
|
|
1781
1793
|
}
|
|
@@ -1798,34 +1810,49 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1798
1810
|
});
|
|
1799
1811
|
}
|
|
1800
1812
|
|
|
1801
|
-
async function registerMcpTools(
|
|
1813
|
+
async function registerMcpTools(
|
|
1814
|
+
definition: ProviderDefinition,
|
|
1815
|
+
auth: McpRuntimeAuth,
|
|
1816
|
+
credential: Credential,
|
|
1817
|
+
signal?: AbortSignal,
|
|
1818
|
+
): Promise<void> {
|
|
1802
1819
|
if (!mcpEnabled || discoveryDisabledReason() || mcpRegistrationFatal || isMcpPaused(auth, credential)) return;
|
|
1803
|
-
const loginGeneration = mcpLoginGeneration;
|
|
1820
|
+
const loginGeneration = mcpLoginGeneration(definition.name);
|
|
1804
1821
|
const identity = mcpCatalogIdentity(auth, credential);
|
|
1805
|
-
|
|
1806
|
-
|
|
1822
|
+
const isDefault = definition.name === PROVIDER_NAME;
|
|
1823
|
+
// The default provider keeps its unprefixed tool names and diagnostics; an alias is scoped by name.
|
|
1824
|
+
const namespace = isDefault ? undefined : definition.name;
|
|
1825
|
+
const notify = isDefault
|
|
1826
|
+
? notifyMcp
|
|
1827
|
+
: Object.assign((message: string) => notifyMcp(message), { incidentScope: definition.name });
|
|
1828
|
+
const label = isDefault ? "LiteLLM MCP" : `LiteLLM MCP (${JSON.stringify(definition.name)})`;
|
|
1829
|
+
let pending = mcpRegistrations.get(definition.name);
|
|
1830
|
+
while (pending) {
|
|
1831
|
+
await waitForMcpRegistration(pending, signal);
|
|
1807
1832
|
signal?.throwIfAborted();
|
|
1808
1833
|
if (
|
|
1809
1834
|
mcpRegistrationFatal ||
|
|
1810
1835
|
isMcpPaused(auth, credential) ||
|
|
1811
|
-
loginGeneration !== mcpLoginGeneration ||
|
|
1812
|
-
|
|
1836
|
+
loginGeneration !== mcpLoginGeneration(definition.name) ||
|
|
1837
|
+
registeredMcpIdentities.get(definition.name) === identity
|
|
1813
1838
|
)
|
|
1814
1839
|
return;
|
|
1840
|
+
pending = mcpRegistrations.get(definition.name);
|
|
1815
1841
|
}
|
|
1816
|
-
if (
|
|
1842
|
+
if (registeredMcpIdentities.get(definition.name) === identity) return;
|
|
1817
1843
|
|
|
1818
1844
|
const registration = (async () => {
|
|
1819
1845
|
try {
|
|
1820
1846
|
signal?.throwIfAborted();
|
|
1821
1847
|
const { definitions, report } = await createMcpToolDefinitions(
|
|
1822
|
-
(ctx) => (ctx?.modelRegistry ?
|
|
1823
|
-
isVerboseDiscovery() ? (message) => notifyMcp(
|
|
1848
|
+
(ctx) => (ctx?.modelRegistry ? requireRuntimeAuth(ctx, definition) : Promise.resolve(auth)),
|
|
1849
|
+
isVerboseDiscovery() ? (message) => notifyMcp(`${label}: ${message}`, "info") : undefined,
|
|
1824
1850
|
signal,
|
|
1825
|
-
|
|
1851
|
+
notify,
|
|
1852
|
+
namespace,
|
|
1826
1853
|
);
|
|
1827
1854
|
signal?.throwIfAborted();
|
|
1828
|
-
if (loginGeneration !== mcpLoginGeneration) return;
|
|
1855
|
+
if (loginGeneration !== mcpLoginGeneration(definition.name)) return;
|
|
1829
1856
|
const registeredNames: string[] = [];
|
|
1830
1857
|
try {
|
|
1831
1858
|
for (const definition of definitions) {
|
|
@@ -1840,14 +1867,14 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1840
1867
|
// Fatal for this instance: report once, with a bounded Pi-authored cause and no proxy text,
|
|
1841
1868
|
// then stop retrying so a stale instance cannot churn discovery on every later refresh.
|
|
1842
1869
|
mcpRegistrationFatal = true;
|
|
1843
|
-
reportMcpRegistrationFatal(registeredNames.length, definitions.length, error,
|
|
1870
|
+
reportMcpRegistrationFatal(registeredNames.length, definitions.length, error, notify);
|
|
1844
1871
|
return;
|
|
1845
1872
|
}
|
|
1846
|
-
reportMcpRegistrationSuccess();
|
|
1847
|
-
reportMcpPartialDiscovery(report.partialFailure, registeredNames,
|
|
1873
|
+
reportMcpRegistrationSuccess(notify);
|
|
1874
|
+
reportMcpPartialDiscovery(report.partialFailure, registeredNames, notify);
|
|
1848
1875
|
if (isVerboseDiscovery()) {
|
|
1849
1876
|
notifyMcp(
|
|
1850
|
-
|
|
1877
|
+
`${label}: registered ${registeredNames.length} of ${definitions.length} prepared MCP tools ` +
|
|
1851
1878
|
`(${report.discovered} raw, ${report.enveloped} enveloped).`,
|
|
1852
1879
|
"info",
|
|
1853
1880
|
);
|
|
@@ -1855,26 +1882,29 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1855
1882
|
// A catalog that produced nothing or came from a partial-failure response is not settled:
|
|
1856
1883
|
// leaving the identity unset lets a later refresh retry discovery, which is network-only and
|
|
1857
1884
|
// non-blocking. Re-registering surviving tools is safe because Pi replaces tools by name.
|
|
1858
|
-
reportMcpCatalogOutcome(report.discovered, definitions.length,
|
|
1859
|
-
if (definitions.length > 0 && !report.partialFailure)
|
|
1885
|
+
reportMcpCatalogOutcome(report.discovered, definitions.length, notify);
|
|
1886
|
+
if (definitions.length > 0 && !report.partialFailure) registeredMcpIdentities.set(definition.name, identity);
|
|
1860
1887
|
} catch (error) {
|
|
1861
1888
|
if (signal?.aborted) throw signal.reason;
|
|
1862
|
-
if (loginGeneration !== mcpLoginGeneration) return;
|
|
1889
|
+
if (loginGeneration !== mcpLoginGeneration(definition.name)) return;
|
|
1863
1890
|
if (error instanceof McpAccessDeniedError) {
|
|
1864
1891
|
pauseMcpDiscovery(auth, credential);
|
|
1865
|
-
|
|
1892
|
+
// Aliases have no /login; their pause scope is their URL, key, and headers, so a change lifts it.
|
|
1893
|
+
notifyMcp(
|
|
1894
|
+
`${label}: access denied; discovery paused until ${isDefault ? "/login litellm succeeds" : "its credentials change"}.`,
|
|
1895
|
+
);
|
|
1866
1896
|
return;
|
|
1867
1897
|
}
|
|
1868
1898
|
notifyMcp(
|
|
1869
|
-
`LiteLLM (${
|
|
1899
|
+
`LiteLLM (${definition.name}): MCP tool discovery failed (${error instanceof Error ? error.message : String(error)}).`,
|
|
1870
1900
|
);
|
|
1871
1901
|
}
|
|
1872
1902
|
})();
|
|
1873
|
-
|
|
1903
|
+
mcpRegistrations.set(definition.name, registration);
|
|
1874
1904
|
try {
|
|
1875
1905
|
await registration;
|
|
1876
1906
|
} finally {
|
|
1877
|
-
if (
|
|
1907
|
+
if (mcpRegistrations.get(definition.name) === registration) mcpRegistrations.delete(definition.name);
|
|
1878
1908
|
}
|
|
1879
1909
|
}
|
|
1880
1910
|
|
|
@@ -1896,7 +1926,13 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1896
1926
|
// network phase, so the provider would stay empty until the user opens /model. Discover here
|
|
1897
1927
|
// instead, the way 1.x did, and hand the catalog over as the provider's baseline models.
|
|
1898
1928
|
async function seedModels(definition: ProviderDefinition): Promise<Model<LiteLLMApi>[]> {
|
|
1899
|
-
|
|
1929
|
+
const disabledReason = discoveryDisabledReason() ?? (isHostOffline() ? "PI_OFFLINE" : null);
|
|
1930
|
+
if (disabledReason) {
|
|
1931
|
+
if (isVerboseDiscovery()) {
|
|
1932
|
+
process.stderr.write(`LiteLLM (${definition.name}): startup discovery skipped (${disabledReason}).\n`);
|
|
1933
|
+
}
|
|
1934
|
+
return [];
|
|
1935
|
+
}
|
|
1900
1936
|
try {
|
|
1901
1937
|
const stored = readStoredCredential(definition.name, join(getAgentDir(), "auth.json"));
|
|
1902
1938
|
// executeHelpers:false — activation must never run the user's key helper as a side effect,
|
|
@@ -1916,6 +1952,9 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1916
1952
|
}
|
|
1917
1953
|
|
|
1918
1954
|
const seeded = await Promise.all(definitions.map(seedModels));
|
|
1955
|
+
// Pi runs a provider's network phase only once a credential resolves, so /litellm-refresh reads an
|
|
1956
|
+
// unchanged count as "no credentials" rather than as a successful refresh.
|
|
1957
|
+
const networkRefreshAttempts = new Map<string, number>();
|
|
1919
1958
|
|
|
1920
1959
|
for (const [index, definition] of definitions.entries()) {
|
|
1921
1960
|
const auth = createProviderAuth(
|
|
@@ -1975,24 +2014,26 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
1975
2014
|
Object.assign(provider, { headers: resolveHeaders(definition) });
|
|
1976
2015
|
const refreshModels = provider.refreshModels!;
|
|
1977
2016
|
provider.refreshModels = async (context) => {
|
|
1978
|
-
const loginGeneration = mcpLoginGeneration;
|
|
2017
|
+
const loginGeneration = mcpLoginGeneration(definition.name);
|
|
2018
|
+
if (context.allowNetwork) {
|
|
2019
|
+
networkRefreshAttempts.set(definition.name, (networkRefreshAttempts.get(definition.name) ?? 0) + 1);
|
|
2020
|
+
}
|
|
1979
2021
|
try {
|
|
1980
2022
|
await refreshModels(context);
|
|
1981
2023
|
} finally {
|
|
1982
2024
|
if (
|
|
1983
|
-
definition.name === PROVIDER_NAME &&
|
|
1984
2025
|
context.allowNetwork &&
|
|
1985
2026
|
!discoveryDisabledReason() &&
|
|
1986
|
-
loginGeneration === mcpLoginGeneration &&
|
|
2027
|
+
loginGeneration === mcpLoginGeneration(definition.name) &&
|
|
1987
2028
|
context.credential
|
|
1988
2029
|
) {
|
|
1989
2030
|
// Best-effort: refreshing the cached default auth / MCP catalog must not let a bad or
|
|
1990
2031
|
// placeholder credential override refreshModels' own try/throw outcome via `finally`.
|
|
1991
2032
|
try {
|
|
1992
2033
|
const auth = await authForCredential(definition, context.credential);
|
|
1993
|
-
if (loginGeneration === mcpLoginGeneration) {
|
|
1994
|
-
defaultRuntimeAuth = auth;
|
|
1995
|
-
void registerMcpTools(auth, context.credential, context.signal).catch(() => undefined);
|
|
2034
|
+
if (loginGeneration === mcpLoginGeneration(definition.name)) {
|
|
2035
|
+
if (definition.name === PROVIDER_NAME) defaultRuntimeAuth = auth;
|
|
2036
|
+
void registerMcpTools(definition, auth, context.credential, context.signal).catch(() => undefined);
|
|
1996
2037
|
}
|
|
1997
2038
|
} catch {
|
|
1998
2039
|
// ignored — authForCredential already reported/will report this via the paths that use it directly.
|
|
@@ -2003,6 +2044,54 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
2003
2044
|
pi.registerProvider(provider);
|
|
2004
2045
|
}
|
|
2005
2046
|
|
|
2047
|
+
// Pi documents PI_OFFLINE as disabling model catalog refreshes, including the one /model starts, and a
|
|
2048
|
+
// provider cannot tell that refresh from an automatic one. This is the explicit refresh: it reaches only
|
|
2049
|
+
// the configured proxies, so LITELLM_OFFLINE=1 and a zero timeout still disable it and models.dev stays off.
|
|
2050
|
+
pi.registerCommand("litellm-refresh", {
|
|
2051
|
+
description: "Refresh LiteLLM model catalogs from the proxy, even with PI_OFFLINE set",
|
|
2052
|
+
getArgumentCompletions: (prefix) => {
|
|
2053
|
+
const matches = [...providerNames].filter((name) => name.startsWith(prefix));
|
|
2054
|
+
return matches.length > 0 ? matches.map((name) => ({ value: name, label: name })) : null;
|
|
2055
|
+
},
|
|
2056
|
+
handler: async (args, ctx) => {
|
|
2057
|
+
const notify = (message: string, level: "info" | "warning"): void => {
|
|
2058
|
+
if (ctx.hasUI) ctx.ui.notify(message, level);
|
|
2059
|
+
else process.stderr.write(`${message}\n`);
|
|
2060
|
+
};
|
|
2061
|
+
const disabledReason = discoveryDisabledReason();
|
|
2062
|
+
if (disabledReason) {
|
|
2063
|
+
notify(`LiteLLM: model refresh skipped (${disabledReason}).`, "warning");
|
|
2064
|
+
return;
|
|
2065
|
+
}
|
|
2066
|
+
const requested = args.trim();
|
|
2067
|
+
const selected = requested ? definitions.filter(({ name }) => name === requested) : definitions;
|
|
2068
|
+
if (selected.length === 0) {
|
|
2069
|
+
const configured = [...providerNames].map((name) => JSON.stringify(name)).join(", ");
|
|
2070
|
+
notify(`LiteLLM: unknown provider ${JSON.stringify(requested)}; configured: ${configured}.`, "warning");
|
|
2071
|
+
return;
|
|
2072
|
+
}
|
|
2073
|
+
const attempts = new Map(selected.map(({ name }) => [name, networkRefreshAttempts.get(name) ?? 0]));
|
|
2074
|
+
const result = await ctx.modelRegistry.refresh({
|
|
2075
|
+
providers: selected.map(({ name }) => name),
|
|
2076
|
+
allowNetwork: true,
|
|
2077
|
+
force: true,
|
|
2078
|
+
});
|
|
2079
|
+
const models = ctx.modelRegistry.getAll();
|
|
2080
|
+
for (const definition of selected) {
|
|
2081
|
+
const label = `LiteLLM (${JSON.stringify(definition.name)})`;
|
|
2082
|
+
const error = result.errors.get(definition.name);
|
|
2083
|
+
if (error) {
|
|
2084
|
+
notify(`${label}: model refresh failed (${error.message}).`, "warning");
|
|
2085
|
+
} else if ((networkRefreshAttempts.get(definition.name) ?? 0) === attempts.get(definition.name)) {
|
|
2086
|
+
notify(`${label}: model refresh skipped; ${missingCredentials(definition)}`, "warning");
|
|
2087
|
+
} else {
|
|
2088
|
+
const count = models.filter((model) => model.provider === definition.name).length;
|
|
2089
|
+
notify(`${label}: refreshed ${count} model${count === 1 ? "" : "s"}.`, "info");
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
},
|
|
2093
|
+
});
|
|
2094
|
+
|
|
2006
2095
|
setupLiteLLMCostTracking(pi, [...providerNames]);
|
|
2007
2096
|
|
|
2008
2097
|
if (skillsEnabled) {
|
|
@@ -2026,6 +2115,27 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
2026
2115
|
);
|
|
2027
2116
|
});
|
|
2028
2117
|
|
|
2118
|
+
// Router fallbacks are invisible to discovery, so protocol and model policy were chosen for the
|
|
2119
|
+
// requested route even when another group answered. Name the route only; header text is proxy-supplied.
|
|
2120
|
+
const warnedFallbackRoutes = new Set<string>();
|
|
2121
|
+
pi.on("after_provider_response", (event, ctx) => {
|
|
2122
|
+
const model = ctx.model;
|
|
2123
|
+
if (!model?.provider || !providerNames.has(model.provider)) return;
|
|
2124
|
+
if (event.status < 200 || event.status >= 300) return;
|
|
2125
|
+
const attemptedFallbacks = Number(event.headers["x-litellm-attempted-fallbacks"]);
|
|
2126
|
+
if (!Number.isSafeInteger(attemptedFallbacks) || attemptedFallbacks <= 0) return;
|
|
2127
|
+
const key = `${model.provider}\0${model.id}`;
|
|
2128
|
+
if (warnedFallbackRoutes.has(key)) return;
|
|
2129
|
+
warnedFallbackRoutes.add(key);
|
|
2130
|
+
const route = JSON.stringify(model.id);
|
|
2131
|
+
const message =
|
|
2132
|
+
`LiteLLM (${JSON.stringify(model.provider)}): a fallback served ${route}, but protocol and model handling were chosen ` +
|
|
2133
|
+
`for ${route}'s own deployments. If its fallbacks cross model families, pin the protocol with ` +
|
|
2134
|
+
"`model_info.supported_endpoints`.";
|
|
2135
|
+
if (ctx.hasUI) ctx.ui.notify(message, "warning");
|
|
2136
|
+
else process.stderr.write(`${message}\n`);
|
|
2137
|
+
});
|
|
2138
|
+
|
|
2029
2139
|
// Skills enrichment is best-effort: an expired credential or an unreachable proxy must not
|
|
2030
2140
|
// report an extension error on every turn. The failing model request states the real problem.
|
|
2031
2141
|
pi.on("before_agent_start", async (event, ctx) => {
|
|
@@ -2048,6 +2158,40 @@ export default async function (pi: ExtensionAPI): Promise<void> {
|
|
|
2048
2158
|
return { systemPrompt: `${event.systemPrompt}\n\n${section}` };
|
|
2049
2159
|
});
|
|
2050
2160
|
|
|
2161
|
+
// MCP tools otherwise register only from refreshModels' `finally`, which needs
|
|
2162
|
+
// context.allowNetwork. Pi core sets that solely for the interactive TUI and the RPC
|
|
2163
|
+
// background refresh, so `-p` and `--list-models` register no MCP tool at all, even when
|
|
2164
|
+
// credentials resolve and the proxy is reachable — the defect #136 fixed for models.
|
|
2165
|
+
//
|
|
2166
|
+
// Seeding here rather than at activation is deliberate: registerMcpTools serialises callers
|
|
2167
|
+
// per provider, so an activation-time call would become an in-flight registration
|
|
2168
|
+
// that Pi's own refresh then waits on. Running once before the first turn keeps that ordering
|
|
2169
|
+
// intact, still precedes any tool use, and leaves startup latency untouched.
|
|
2170
|
+
async function seedMcpTools(definition: ProviderDefinition): Promise<void> {
|
|
2171
|
+
try {
|
|
2172
|
+
const stored = readStoredCredential(definition.name, join(getAgentDir(), "auth.json"));
|
|
2173
|
+
// executeHelpers:false — seeding must never run the user's key helper as a side effect.
|
|
2174
|
+
const auth = await authForCredential(definition, stored, false);
|
|
2175
|
+
// An env-configured provider has no stored credential and must still seed. The credential
|
|
2176
|
+
// only derives the pause identity, so the resolved key reproduces the stored api_key scope.
|
|
2177
|
+
const credential: Credential = stored ?? { type: "api_key", key: auth.apiKey };
|
|
2178
|
+
await registerMcpTools(definition, auth, credential, AbortSignal.timeout(getSeedTimeoutMs()));
|
|
2179
|
+
} catch (error) {
|
|
2180
|
+
if (isVerboseDiscovery()) {
|
|
2181
|
+
process.stderr.write(
|
|
2182
|
+
`LiteLLM (${definition.name}): MCP seeding skipped (${error instanceof Error ? error.message : String(error)}).\n`,
|
|
2183
|
+
);
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
pi.on("before_agent_start", async () => {
|
|
2189
|
+
if (!mcpEnabled || mcpSeeded || discoveryDisabledReason() || isHostOffline()) return;
|
|
2190
|
+
mcpSeeded = true;
|
|
2191
|
+
// Catalogs are independent per provider, so every configured provider seeds in parallel.
|
|
2192
|
+
await Promise.all(definitions.map(seedMcpTools));
|
|
2193
|
+
});
|
|
2194
|
+
|
|
2051
2195
|
pi.on("message_end", (event, ctx) => {
|
|
2052
2196
|
if (event.message.role !== "assistant") return;
|
|
2053
2197
|
// Resolve the discovered model that produced this message so the display
|
package/src/mcp-tools.ts
CHANGED
|
@@ -27,7 +27,9 @@ const TRUNCATION_MARKER = "\n[truncated by pi-provider-litellm]";
|
|
|
27
27
|
const DESCRIPTION_TRUNCATION_MARKER = "… [truncated]";
|
|
28
28
|
const SHORT_TRUNCATION_MARKER = "…";
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
// `incidentScope` separates the suppression state of catalogs from different providers, so one
|
|
31
|
+
// provider's clean pass cannot clear, and its differing report cannot re-trigger, another's incident.
|
|
32
|
+
type DiagnosticSink = ((message: string) => void) & { readonly incidentScope?: string };
|
|
31
33
|
|
|
32
34
|
interface RawLiteLLMMcpTool {
|
|
33
35
|
name?: unknown;
|
|
@@ -130,13 +132,16 @@ function emitSafetyDiagnostic(
|
|
|
130
132
|
process.stderr.write(line);
|
|
131
133
|
},
|
|
132
134
|
): void {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
135
|
+
const scope = onDiagnostic.incidentScope;
|
|
136
|
+
const key = scope === undefined ? incident : `${scope}\0${incident}`;
|
|
137
|
+
if (lastEmittedIncident.get(key) === identity) return;
|
|
138
|
+
lastEmittedIncident.set(key, identity);
|
|
139
|
+
onDiagnostic(`LiteLLM MCP${scope === undefined ? "" : ` (${JSON.stringify(scope)})`}: ${message}\n`);
|
|
136
140
|
}
|
|
137
141
|
|
|
138
|
-
function clearIncident(incident: string): void {
|
|
139
|
-
|
|
142
|
+
function clearIncident(incident: string, onDiagnostic?: DiagnosticSink): void {
|
|
143
|
+
const scope = onDiagnostic?.incidentScope;
|
|
144
|
+
lastEmittedIncident.delete(scope === undefined ? incident : `${scope}\0${incident}`);
|
|
140
145
|
}
|
|
141
146
|
|
|
142
147
|
function membershipIdentity(tools: readonly string[]): string {
|
|
@@ -184,8 +189,8 @@ export function reportMcpRegistrationFatal(
|
|
|
184
189
|
|
|
185
190
|
// A successful pass clears any fatal incident left by an earlier extension instance, so the same
|
|
186
191
|
// failure is reported if it later recurs.
|
|
187
|
-
export function reportMcpRegistrationSuccess(): void {
|
|
188
|
-
clearIncident("registration-fatal");
|
|
192
|
+
export function reportMcpRegistrationSuccess(onDiagnostic?: DiagnosticSink): void {
|
|
193
|
+
clearIncident("registration-fatal", onDiagnostic);
|
|
189
194
|
}
|
|
190
195
|
|
|
191
196
|
// `registered` is the generated names of the tools that survived, so a failure that changes
|
|
@@ -196,7 +201,7 @@ export function reportMcpPartialDiscovery(
|
|
|
196
201
|
onDiagnostic?: DiagnosticSink,
|
|
197
202
|
): void {
|
|
198
203
|
if (!partialFailure) {
|
|
199
|
-
clearIncident("discovery-partial-failure");
|
|
204
|
+
clearIncident("discovery-partial-failure", onDiagnostic);
|
|
200
205
|
return;
|
|
201
206
|
}
|
|
202
207
|
emitSafetyDiagnostic(
|
|
@@ -212,7 +217,7 @@ export function reportMcpPartialDiscovery(
|
|
|
212
217
|
// an unchanged message.
|
|
213
218
|
export function reportMcpCatalogOutcome(raw: number, registered: number, onDiagnostic?: DiagnosticSink): void {
|
|
214
219
|
if (registered > 0) {
|
|
215
|
-
clearIncident("empty-catalog");
|
|
220
|
+
clearIncident("empty-catalog", onDiagnostic);
|
|
216
221
|
return;
|
|
217
222
|
}
|
|
218
223
|
emitSafetyDiagnostic(
|
|
@@ -281,7 +286,7 @@ async function readBoundedText(
|
|
|
281
286
|
onDiagnostic?: DiagnosticSink,
|
|
282
287
|
): Promise<string> {
|
|
283
288
|
if (!response.body) {
|
|
284
|
-
clearIncident(`${surface}-body-cap
|
|
289
|
+
clearIncident(`${surface}-body-cap`, onDiagnostic);
|
|
285
290
|
return "";
|
|
286
291
|
}
|
|
287
292
|
const reader = response.body.getReader();
|
|
@@ -312,7 +317,7 @@ async function readBoundedText(
|
|
|
312
317
|
}
|
|
313
318
|
// A response that stayed within the cap clears the incident, so a later breach on this surface
|
|
314
319
|
// is reported again instead of being suppressed as an unchanged message.
|
|
315
|
-
clearIncident(`${surface}-body-cap
|
|
320
|
+
clearIncident(`${surface}-body-cap`, onDiagnostic);
|
|
316
321
|
return Buffer.concat(
|
|
317
322
|
chunks.map((chunk) => Buffer.from(chunk)),
|
|
318
323
|
size,
|
|
@@ -743,9 +748,13 @@ function toolIdentity(tool: LiteLLMMcpTool): string {
|
|
|
743
748
|
// A conditional hash would make a survivor's name depend on which *other* tools happened to be in
|
|
744
749
|
// the same catalog, so adding or removing a sibling would rename it — and because Pi cannot
|
|
745
750
|
// unregister, the old name would linger and the model would see one tool twice.
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
751
|
+
// An alias provider's catalog is namespaced by the provider name, in both the prefix and the hash,
|
|
752
|
+
// so two providers exposing the same server never replace each other's tools.
|
|
753
|
+
function buildPiToolName(tool: LiteLLMMcpTool, namespace?: string): string {
|
|
754
|
+
const identity = namespace === undefined ? toolIdentity(tool) : JSON.stringify([namespace, toolIdentity(tool)]);
|
|
755
|
+
const hash = createHash("sha256").update(identity).digest("hex").slice(0, TOOL_NAME_HASH_LENGTH);
|
|
756
|
+
const prefix = namespace === undefined ? "mcp" : `mcp_${sanitizeName(namespace)}`;
|
|
757
|
+
const base = `${prefix}_${sanitizeName(tool.server_name)}_${sanitizeName(tool.name)}`;
|
|
749
758
|
return `${base.slice(0, MAX_TOOL_NAME_LENGTH - hash.length - 1)}_${hash}`;
|
|
750
759
|
}
|
|
751
760
|
|
|
@@ -1135,7 +1144,10 @@ function buildParameters(tool: LiteLLMMcpTool): BuiltParameters | McpDropReason
|
|
|
1135
1144
|
return { parameters: Type.Unsafe(inputSchema), syntheticArgsEnvelope: false };
|
|
1136
1145
|
}
|
|
1137
1146
|
|
|
1138
|
-
export function prepareTools(
|
|
1147
|
+
export function prepareTools(
|
|
1148
|
+
discovery: McpDiscovery,
|
|
1149
|
+
namespace?: string,
|
|
1150
|
+
): {
|
|
1139
1151
|
prepared: PreparedTool[];
|
|
1140
1152
|
report: McpPreparationReport;
|
|
1141
1153
|
} {
|
|
@@ -1160,12 +1172,12 @@ export function prepareTools(discovery: McpDiscovery): {
|
|
|
1160
1172
|
for (const tool of discovery.tools) {
|
|
1161
1173
|
const built = buildParameters(tool);
|
|
1162
1174
|
if (typeof built === "string") {
|
|
1163
|
-
recordDrop(built, buildPiToolName(tool));
|
|
1175
|
+
recordDrop(built, buildPiToolName(tool, namespace));
|
|
1164
1176
|
continue;
|
|
1165
1177
|
}
|
|
1166
1178
|
const identity = toolIdentity(tool);
|
|
1167
1179
|
if (acceptedIdentities.has(identity)) {
|
|
1168
|
-
recordDrop("duplicate-identity", buildPiToolName(tool));
|
|
1180
|
+
recordDrop("duplicate-identity", buildPiToolName(tool, namespace));
|
|
1169
1181
|
continue;
|
|
1170
1182
|
}
|
|
1171
1183
|
acceptedIdentities.add(identity);
|
|
@@ -1178,12 +1190,12 @@ export function prepareTools(discovery: McpDiscovery): {
|
|
|
1178
1190
|
}
|
|
1179
1191
|
|
|
1180
1192
|
const candidates = accepted.slice(0, MAX_REGISTERED_TOOLS);
|
|
1181
|
-
const overflowTools = accepted.slice(MAX_REGISTERED_TOOLS).map(({ tool }) => buildPiToolName(tool));
|
|
1193
|
+
const overflowTools = accepted.slice(MAX_REGISTERED_TOOLS).map(({ tool }) => buildPiToolName(tool, namespace));
|
|
1182
1194
|
const finalNames = new Set<string>();
|
|
1183
1195
|
const prepared: PreparedTool[] = [];
|
|
1184
1196
|
for (const candidate of candidates) {
|
|
1185
1197
|
// Membership-independent: the name depends only on this tool's identity.
|
|
1186
|
-
const name = buildPiToolName(candidate.tool);
|
|
1198
|
+
const name = buildPiToolName(candidate.tool, namespace);
|
|
1187
1199
|
if (finalNames.has(name)) {
|
|
1188
1200
|
recordDrop("name-collision", name);
|
|
1189
1201
|
continue;
|
|
@@ -1251,7 +1263,7 @@ function emitPreparationDiagnostics(report: McpPreparationReport, onDiagnostic?:
|
|
|
1251
1263
|
// Clear classes that did not recur, so the same incident is reported again if it comes back
|
|
1252
1264
|
// after a clean refresh instead of being suppressed as an unchanged identity.
|
|
1253
1265
|
for (const reason of [...Object.keys(INCIDENT_REASON_TEXT), "tool-cap"]) {
|
|
1254
|
-
if (!seen.has(reason)) clearIncident(reason);
|
|
1266
|
+
if (!seen.has(reason)) clearIncident(reason, onDiagnostic);
|
|
1255
1267
|
}
|
|
1256
1268
|
}
|
|
1257
1269
|
|
|
@@ -1260,6 +1272,7 @@ export async function createMcpToolDefinitions(
|
|
|
1260
1272
|
onProgress?: (message: string) => void,
|
|
1261
1273
|
signal?: AbortSignal,
|
|
1262
1274
|
onDiagnostic?: DiagnosticSink,
|
|
1275
|
+
namespace?: string,
|
|
1263
1276
|
): Promise<{ definitions: ToolDefinition[]; report: McpPreparationReport }> {
|
|
1264
1277
|
const discoveryAuth = await getAuth();
|
|
1265
1278
|
const discovery = await discoverMcpTools(
|
|
@@ -1271,7 +1284,7 @@ export async function createMcpToolDefinitions(
|
|
|
1271
1284
|
discoveryAuth.allowInsecureHttp,
|
|
1272
1285
|
onDiagnostic,
|
|
1273
1286
|
);
|
|
1274
|
-
const { prepared, report } = prepareTools(discovery);
|
|
1287
|
+
const { prepared, report } = prepareTools(discovery, namespace);
|
|
1275
1288
|
emitPreparationDiagnostics(report, onDiagnostic);
|
|
1276
1289
|
const lostTotal = report.dropped.reduce((total, entry) => total + entry.tools.length, 0) + report.overflow;
|
|
1277
1290
|
const lostDetail = [
|
package/src/model-groups.ts
CHANGED
|
@@ -2,7 +2,32 @@ import type { Api, Model } from "@earendil-works/pi-ai";
|
|
|
2
2
|
import { intersectThinkingLevelMaps, THINKING_LEVEL_DEFINITIONS } from "./thinking-levels.js";
|
|
3
3
|
import type { DiscoveredModel, ModelInfoEntry } from "./types.js";
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
const BUILTIN_CONTEXT_WINDOW = 128_000;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Context window used when neither /model/info nor the catalog reports one.
|
|
9
|
+
* LITELLM_DEFAULT_CONTEXT_WINDOW raises it for proxies whose model map cannot
|
|
10
|
+
* populate `max_input_tokens`; an unset or unusable value keeps the 128K default.
|
|
11
|
+
*/
|
|
12
|
+
export function defaultContextWindow(): number {
|
|
13
|
+
// Number, not parseInt: parseInt takes a numeric prefix, so `1.5` would become a
|
|
14
|
+
// 1-token window and `922000junk` would pass as a limit instead of being rejected.
|
|
15
|
+
const parsed = Number(process.env.LITELLM_DEFAULT_CONTEXT_WINDOW ?? "");
|
|
16
|
+
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : BUILTIN_CONTEXT_WINDOW;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Whether a stored window is one this extension assumed rather than read. A model cached
|
|
21
|
+
* under the built-in default must still read as evidence-free once an operator configures
|
|
22
|
+
* LITELLM_DEFAULT_CONTEXT_WINDOW, or the setting would never reach cached entries.
|
|
23
|
+
* ponytail: a window stored under a setting that was later lowered is indistinguishable
|
|
24
|
+
* from measured evidence and stays until the next online discovery; persist a fallback
|
|
25
|
+
* marker on the model if that ever matters.
|
|
26
|
+
*/
|
|
27
|
+
export function isFallbackContextWindow(contextWindow: number): boolean {
|
|
28
|
+
return contextWindow === BUILTIN_CONTEXT_WINDOW || contextWindow === defaultContextWindow();
|
|
29
|
+
}
|
|
30
|
+
|
|
6
31
|
export const DEFAULT_MAX_TOKENS = 16_384;
|
|
7
32
|
|
|
8
33
|
export type SemanticFamily = "claude" | "deepseek" | "gemini" | "kimi" | "openai";
|
|
@@ -52,6 +77,7 @@ export interface ReducedModelGroup {
|
|
|
52
77
|
thinkingLevelMap?: DiscoveredModel["thinkingLevelMap"];
|
|
53
78
|
vision: boolean;
|
|
54
79
|
contextWindow: number;
|
|
80
|
+
contextWindowDefaulted?: boolean;
|
|
55
81
|
maxTokens: number;
|
|
56
82
|
cost: DiscoveredModel["cost"];
|
|
57
83
|
hasCompleteCost: boolean;
|
|
@@ -801,7 +827,8 @@ export function reduceModelGroup(
|
|
|
801
827
|
(entry) => wireBoolean(entry.model_info?.supports_reasoning) === false,
|
|
802
828
|
);
|
|
803
829
|
const vision = visionEvidence.every((value) => value ?? false);
|
|
804
|
-
const
|
|
830
|
+
const fallbackContextWindow = defaultContextWindow();
|
|
831
|
+
const contextWindow = min(contextWindowEvidence.map((value) => value ?? fallbackContextWindow));
|
|
805
832
|
const maxTokens = min(maxTokensEvidence.map((value) => value ?? DEFAULT_MAX_TOKENS));
|
|
806
833
|
|
|
807
834
|
const costValues = COST_FIELDS.map((field) =>
|
|
@@ -930,6 +957,9 @@ export function reduceModelGroup(
|
|
|
930
957
|
...(thinkingLevelMap ? { thinkingLevelMap } : {}),
|
|
931
958
|
vision,
|
|
932
959
|
contextWindow,
|
|
960
|
+
...(contextWindowEvidence.includes(undefined) && contextWindow === fallbackContextWindow
|
|
961
|
+
? { contextWindowDefaulted: true }
|
|
962
|
+
: {}),
|
|
933
963
|
maxTokens,
|
|
934
964
|
cost,
|
|
935
965
|
hasCompleteCost,
|
package/src/public-catalog.ts
CHANGED
|
@@ -73,6 +73,7 @@ const PROVIDER_ALIASES: Readonly<Record<string, readonly string[]>> = {
|
|
|
73
73
|
const PI_PROVIDER_ALIASES: Readonly<Record<string, readonly string[]>> = {
|
|
74
74
|
"amazon-bedrock": ["amazon-bedrock"],
|
|
75
75
|
azure: ["azure-openai-responses"],
|
|
76
|
+
azure_ai: ["azure-openai-responses"],
|
|
76
77
|
chatgpt: ["openai-codex"],
|
|
77
78
|
"fireworks-ai": ["fireworks"],
|
|
78
79
|
openai: ["openai"],
|
|
@@ -245,6 +246,7 @@ export async function loadPublicCatalog(options: LoadPublicCatalogOptions = {}):
|
|
|
245
246
|
const providers = providerCandidates(provider);
|
|
246
247
|
const ids = lookupIds(id);
|
|
247
248
|
const vendor = providers.find((candidate) => candidate !== "azure") ?? providers[0];
|
|
249
|
+
const adapter = provider?.trim().toLowerCase();
|
|
248
250
|
for (const candidate of providers) {
|
|
249
251
|
const models = catalog?.[candidate]?.models;
|
|
250
252
|
for (const modelId of ids) {
|
|
@@ -252,20 +254,25 @@ export async function loadPublicCatalog(options: LoadPublicCatalogOptions = {}):
|
|
|
252
254
|
if (!model) continue;
|
|
253
255
|
const record = mapModelsDev(candidate, modelId, model);
|
|
254
256
|
if (candidate === providers[0] || !vendor) return record;
|
|
255
|
-
|
|
257
|
+
const fallback =
|
|
258
|
+
adapter && (PI_PROVIDER_ALIASES[adapter] ?? [adapter]).some((piProvider) => findPiModel(piProvider, ids))
|
|
259
|
+
? adapter
|
|
260
|
+
: vendor;
|
|
261
|
+
return { ...record, piProvider: (PI_PROVIDER_ALIASES[fallback] ?? [fallback])[0] };
|
|
256
262
|
}
|
|
257
263
|
}
|
|
258
|
-
|
|
259
|
-
|
|
264
|
+
// Provider-specific limits and pricing must not be replaced by the generic
|
|
265
|
+
// vendor's entry just because both catalogs recognize the model id.
|
|
266
|
+
if (adapter && adapter !== vendor) {
|
|
267
|
+
for (const piProvider of PI_PROVIDER_ALIASES[adapter] ?? [adapter]) {
|
|
260
268
|
const model = findPiModel(piProvider, ids);
|
|
261
|
-
if (model) return toPiRecord("pi-
|
|
269
|
+
if (model) return toPiRecord("pi-adapter", piProvider, model);
|
|
262
270
|
}
|
|
263
271
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
for (const piProvider of PI_PROVIDER_ALIASES[adapter] ?? [adapter]) {
|
|
272
|
+
if (vendor) {
|
|
273
|
+
for (const piProvider of PI_PROVIDER_ALIASES[vendor] ?? [vendor]) {
|
|
267
274
|
const model = findPiModel(piProvider, ids);
|
|
268
|
-
if (model) return toPiRecord("pi-
|
|
275
|
+
if (model) return toPiRecord("pi-vendor", piProvider, model);
|
|
269
276
|
}
|
|
270
277
|
}
|
|
271
278
|
return undefined;
|
package/src/skills.ts
CHANGED
|
@@ -42,97 +42,55 @@ export async function listSkills(
|
|
|
42
42
|
return skillsCache.skills;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
-
const fetchSkills = async (path: string): Promise<LiteLLMSkill[]> => {
|
|
45
|
+
const fetchSkills = async (path: string): Promise<LiteLLMSkill[] | undefined> => {
|
|
46
46
|
try {
|
|
47
47
|
const response = await fetch(`${normalizedBaseUrl}${path}`, {
|
|
48
48
|
headers: { ...headers, Authorization: `Bearer ${apiKey}`, Accept: "application/json" },
|
|
49
49
|
signal: AbortSignal.timeout(10_000),
|
|
50
50
|
});
|
|
51
|
-
return response.ok ? getSkillsFromBody(await response.json()) :
|
|
51
|
+
return response.ok ? getSkillsFromBody(await response.json()) : undefined;
|
|
52
52
|
} catch {
|
|
53
|
-
return
|
|
53
|
+
return undefined;
|
|
54
54
|
}
|
|
55
55
|
};
|
|
56
56
|
|
|
57
|
-
const
|
|
58
|
-
fetchSkills("/claude-code/marketplace.json"),
|
|
59
|
-
fetchSkills("/v1/skills"),
|
|
60
|
-
]);
|
|
61
|
-
const skillHubNames = new Set(skillHub.map((skill) => skill.name));
|
|
62
|
-
const skills = [...skillHub, ...legacy.filter((skill) => !skillHubNames.has(skill.name))];
|
|
57
|
+
const skills = (await fetchSkills("/claude-code/marketplace.json")) ?? (await fetchSkills("/v1/skills")) ?? [];
|
|
63
58
|
skillsCache = { baseUrl: normalizedBaseUrl, apiKey, fetchedAt: Date.now(), skills };
|
|
64
59
|
return skills;
|
|
65
60
|
}
|
|
66
61
|
|
|
62
|
+
// Only the Skill Hub creates skills from JSON. LiteLLM's POST /v1/skills is Anthropic's multipart
|
|
63
|
+
// Skills API, so a JSON code skill sent there never becomes a listable skill.
|
|
67
64
|
export async function createSkill(
|
|
68
65
|
baseUrl: string,
|
|
69
66
|
apiKey: string,
|
|
70
|
-
input: {
|
|
71
|
-
name: string;
|
|
72
|
-
description?: string;
|
|
73
|
-
source?: Record<string, unknown>;
|
|
74
|
-
code?: string;
|
|
75
|
-
inputSchema?: Record<string, unknown>;
|
|
76
|
-
},
|
|
67
|
+
input: { name: string; description?: string; source: Record<string, unknown> },
|
|
77
68
|
headers?: Record<string, string>,
|
|
78
69
|
allowInsecureHttp = false,
|
|
79
70
|
): Promise<unknown> {
|
|
80
|
-
if (!input.source
|
|
71
|
+
if (!input.source) throw new Error("source is required: skills are created through the LiteLLM Skill Hub");
|
|
81
72
|
const normalizedBaseUrl = normalizeBaseUrl(baseUrl, allowInsecureHttp);
|
|
82
|
-
|
|
83
|
-
const skillHubPayload = input.source
|
|
84
|
-
? {
|
|
85
|
-
name: input.name,
|
|
86
|
-
description: input.description,
|
|
87
|
-
source: input.source,
|
|
88
|
-
}
|
|
89
|
-
: {
|
|
90
|
-
name: input.name,
|
|
91
|
-
description: input.description,
|
|
92
|
-
code: input.code,
|
|
93
|
-
input_schema: input.inputSchema ?? { type: "object", properties: {} },
|
|
94
|
-
};
|
|
95
|
-
let response = await fetch(`${normalizedBaseUrl}${input.source ? "/claude-code/plugins" : "/v1/skills"}`, {
|
|
73
|
+
const response = await fetch(`${normalizedBaseUrl}/claude-code/plugins`, {
|
|
96
74
|
method: "POST",
|
|
97
75
|
headers: {
|
|
98
76
|
...headers,
|
|
99
77
|
Authorization: `Bearer ${apiKey}`,
|
|
100
78
|
"Content-Type": "application/json",
|
|
101
79
|
},
|
|
102
|
-
body: JSON.stringify(
|
|
80
|
+
body: JSON.stringify({ name: input.name, description: input.description, source: input.source }),
|
|
103
81
|
signal: AbortSignal.timeout(10_000),
|
|
104
82
|
});
|
|
105
|
-
if (input.source && response.status === 404) {
|
|
106
|
-
usedSkillHub = false;
|
|
107
|
-
response = await fetch(`${normalizedBaseUrl}/v1/skills`, {
|
|
108
|
-
method: "POST",
|
|
109
|
-
headers: {
|
|
110
|
-
...headers,
|
|
111
|
-
Authorization: `Bearer ${apiKey}`,
|
|
112
|
-
"Content-Type": "application/json",
|
|
113
|
-
},
|
|
114
|
-
body: JSON.stringify({
|
|
115
|
-
name: input.name,
|
|
116
|
-
description: input.description,
|
|
117
|
-
code: input.code,
|
|
118
|
-
input_schema: input.inputSchema ?? { type: "object", properties: {} },
|
|
119
|
-
}),
|
|
120
|
-
signal: AbortSignal.timeout(10_000),
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
83
|
if (!response.ok) throw new Error(`LiteLLM skill create failed: HTTP ${response.status}`);
|
|
124
84
|
skillsCache = undefined;
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
if (!enableResponse.ok) throw new Error(`LiteLLM skill enable failed: HTTP ${enableResponse.status}`);
|
|
135
|
-
}
|
|
85
|
+
const enableResponse = await fetch(
|
|
86
|
+
`${normalizedBaseUrl}/claude-code/plugins/${encodeURIComponent(input.name)}/enable`,
|
|
87
|
+
{
|
|
88
|
+
method: "POST",
|
|
89
|
+
headers: { ...headers, Authorization: `Bearer ${apiKey}` },
|
|
90
|
+
signal: AbortSignal.timeout(10_000),
|
|
91
|
+
},
|
|
92
|
+
);
|
|
93
|
+
if (!enableResponse.ok) throw new Error(`LiteLLM skill enable failed: HTTP ${enableResponse.status}`);
|
|
136
94
|
return response.json().catch(() => ({}));
|
|
137
95
|
}
|
|
138
96
|
|
|
@@ -196,9 +154,7 @@ function formatSkills(skills: LiteLLMSkill[]): string {
|
|
|
196
154
|
const CreateSkillParams = Type.Object({
|
|
197
155
|
name: Type.String({ description: "Skill name" }),
|
|
198
156
|
description: Type.Optional(Type.String({ description: "Skill description" })),
|
|
199
|
-
sourceJson: Type.
|
|
200
|
-
code: Type.Optional(Type.String({ description: "Legacy Skills Gateway implementation or prompt code" })),
|
|
201
|
-
inputSchemaJson: Type.Optional(Type.String({ description: "Optional JSON Schema string for skill inputs" })),
|
|
157
|
+
sourceJson: Type.String({ description: "Skill Hub source metadata JSON object" }),
|
|
202
158
|
});
|
|
203
159
|
|
|
204
160
|
const DeleteSkillParams = Type.Object({
|
|
@@ -232,24 +188,15 @@ export function createSkillToolDefinitions(
|
|
|
232
188
|
defineTool({
|
|
233
189
|
name: "litellm_skill_create",
|
|
234
190
|
label: "Create LiteLLM Skill",
|
|
235
|
-
description: "Create a skill on the LiteLLM proxy
|
|
191
|
+
description: "Create and enable a skill on the LiteLLM proxy Skill Hub from source metadata.",
|
|
236
192
|
parameters: CreateSkillParams,
|
|
237
193
|
async execute(_toolCallId, params: Static<typeof CreateSkillParams>, _signal, _onUpdate, ctx) {
|
|
238
194
|
const auth = await getAuth(ctx);
|
|
239
|
-
const source =
|
|
240
|
-
const inputSchema = params.inputSchemaJson
|
|
241
|
-
? parseJsonObject(params.inputSchemaJson, "inputSchemaJson")
|
|
242
|
-
: undefined;
|
|
195
|
+
const source = parseJsonObject(params.sourceJson, "sourceJson");
|
|
243
196
|
const result = await createSkill(
|
|
244
197
|
auth.baseUrl,
|
|
245
198
|
auth.apiKey,
|
|
246
|
-
{
|
|
247
|
-
name: params.name,
|
|
248
|
-
description: params.description,
|
|
249
|
-
source,
|
|
250
|
-
code: params.code,
|
|
251
|
-
inputSchema,
|
|
252
|
-
},
|
|
199
|
+
{ name: params.name, description: params.description, source },
|
|
253
200
|
auth.headers,
|
|
254
201
|
auth.allowInsecureHttp,
|
|
255
202
|
);
|