pi-provider-litellm 4.7.3 → 4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -7
- package/package.json +2 -2
- package/src/backend-identity.ts +21 -5
- package/src/discover.ts +95 -21
- package/src/index.ts +12 -1
- package/src/model-groups.ts +19 -1
- package/src/types.ts +1 -0
package/README.md
CHANGED
|
@@ -4,6 +4,40 @@ LiteLLM proxy native Provider extension for [Pi](https://pi.dev). Pi 0.99.2+ is
|
|
|
4
4
|
|
|
5
5
|
Discovers models from self-hosted LiteLLM proxies and registers them under Pi providers. The default provider is `litellm`; optional aliases can register additional LiteLLM providers with separate credentials. Supports `/login litellm`, LiteLLM MCP tools, LiteLLM Skills Gateway prompt injection, budget status, image generation and classifier routes, and Google ADC token auth. A [Go port](#pig) brings it to PiG, without image generation and classifier routes. Tries `/model/info` first (admin endpoint with rich metadata), falls back to `/v1/models` (OpenAI-compatible) on 401/403/404 or an empty list, then tries `/health` plus per-endpoint `/model/info` for older LiteLLM proxies.
|
|
6
6
|
|
|
7
|
+
<p align="center"><strong>Trusted* by engineers** at***</strong></p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://github.com/Netflix"><img src="https://github.com/Netflix.png?size=96" width="48" height="48" alt="Netflix" title="Netflix"></a>
|
|
11
|
+
<a href="https://github.com/elastic"><img src="https://github.com/elastic.png?size=96" width="48" height="48" alt="Elastic" title="Elastic"></a>
|
|
12
|
+
<a href="https://github.com/dbt-labs"><img src="https://github.com/dbt-labs.png?size=96" width="48" height="48" alt="dbt Labs" title="dbt Labs"></a>
|
|
13
|
+
<a href="https://github.com/verily-src"><img src="https://github.com/verily-src.png?size=96" width="48" height="48" alt="Verily" title="Verily"></a>
|
|
14
|
+
<a href="https://github.com/GoodNotes"><img src="https://github.com/GoodNotes.png?size=96" width="48" height="48" alt="Goodnotes" title="Goodnotes"></a>
|
|
15
|
+
<a href="https://github.com/Rocket-Money"><img src="https://github.com/Rocket-Money.png?size=96" width="48" height="48" alt="Rocket Money" title="Rocket Money"></a>
|
|
16
|
+
<a href="https://github.com/tata1mg"><img src="https://github.com/tata1mg.png?size=96" width="48" height="48" alt="Tata 1mg" title="Tata 1mg"></a>
|
|
17
|
+
<a href="https://github.com/invitae"><img src="https://github.com/invitae.png?size=96" width="48" height="48" alt="Invitae" title="Invitae"></a>
|
|
18
|
+
<a href="https://github.com/AOEpeople"><img src="https://github.com/AOEpeople.png?size=96" width="48" height="48" alt="AOE" title="AOE"></a>
|
|
19
|
+
<a href="https://github.com/anjaraeu"><img src="https://github.com/anjaraeu.png?size=96" width="48" height="48" alt="Anjara" title="Anjara"></a>
|
|
20
|
+
<a href="https://github.com/automatearmy"><img src="https://github.com/automatearmy.png?size=96" width="48" height="48" alt="Automate Army" title="Automate Army"></a>
|
|
21
|
+
<a href="https://github.com/Baur-Software"><img src="https://github.com/Baur-Software.png?size=96" width="48" height="48" alt="Baur Software" title="Baur Software"></a>
|
|
22
|
+
<a href="https://github.com/braincompany"><img src="https://github.com/braincompany.png?size=96" width="48" height="48" alt="Brain Co." title="Brain Co."></a>
|
|
23
|
+
<a href="https://github.com/cin-ufpe"><img src="https://github.com/cin-ufpe.png?size=96" width="48" height="48" alt="CIn-UFPE" title="CIn-UFPE"></a>
|
|
24
|
+
<a href="https://github.com/envite-consulting"><img src="https://github.com/envite-consulting.png?size=96" width="48" height="48" alt="envite consulting" title="envite consulting"></a>
|
|
25
|
+
<a href="https://github.com/Franken-Tech"><img src="https://github.com/Franken-Tech.png?size=96" width="48" height="48" alt="Franken.Tech" title="Franken.Tech"></a>
|
|
26
|
+
<a href="https://github.com/hyperfiddle"><img src="https://github.com/hyperfiddle.png?size=96" width="48" height="48" alt="Hyperfiddle" title="Hyperfiddle"></a>
|
|
27
|
+
<a href="https://github.com/Possehl-Digital-Services"><img src="https://github.com/Possehl-Digital-Services.png?size=96" width="48" height="48" alt="Possehl Digital Services" title="Possehl Digital Services"></a>
|
|
28
|
+
<a href="https://github.com/Premian-Labs"><img src="https://github.com/Premian-Labs.png?size=96" width="48" height="48" alt="Premia" title="Premia"></a>
|
|
29
|
+
<a href="https://github.com/Pronovix"><img src="https://github.com/Pronovix.png?size=96" width="48" height="48" alt="Pronovix" title="Pronovix"></a>
|
|
30
|
+
<a href="https://github.com/WikiTeq"><img src="https://github.com/WikiTeq.png?size=96" width="48" height="48" alt="WikiTeq" title="WikiTeq"></a>
|
|
31
|
+
</p>
|
|
32
|
+
|
|
33
|
+
<p align="center"><sub>
|
|
34
|
+
* "Trusted" means somebody starred the repo, opened an issue or sent a PR. Yes, bug reports count as trust.<br>
|
|
35
|
+
** Could be managers, I didn't check.<br>
|
|
36
|
+
*** Going by the company field of their GitHub profile, which everybody updates right after changing jobs.<br>
|
|
37
|
+
None of these companies has endorsed this extension, paid for it, or most likely heard of it. There is no enterprise plan.<br>
|
|
38
|
+
Want your logo here? Star the repo and wait for me to notice. Want it gone? <a href="https://github.com/balcsida/pi-provider-litellm/issues/new">Open an issue</a>, which, sadly, counts as trust.
|
|
39
|
+
</sub></p>
|
|
40
|
+
|
|
7
41
|
## Install
|
|
8
42
|
|
|
9
43
|
```bash
|
|
@@ -286,10 +320,12 @@ LiteLLM Skills, MCP integration, and the budget footer are enabled by default. D
|
|
|
286
320
|
|
|
287
321
|
Setting `skills.enabled` to `false` disables the Skills Gateway management tools, skill fetching, and system-prompt injection. Setting `mcp.enabled` to `false` stops the extension from registering the proxy's MCP server. Setting `budget.enabled` to `false` registers no budget footer, polls, or `/litellm-budget` command; `budget.display` is described under [Budget status](#budget-status). Restart Pi after changing these settings.
|
|
288
322
|
|
|
289
|
-
MCP can also be enabled or disabled per provider, though a provider setting cannot turn it on while `mcp.enabled` is `false
|
|
323
|
+
MCP can also be enabled or disabled per provider, though a provider setting cannot turn it on while MCP is off as a whole (`mcp.enabled` is `false`, or `LITELLM_ENABLE_MCP` switches it off):
|
|
290
324
|
- In `~/.pi/agent/settings.json`: set `providers.<name>.mcp.enabled` (or `providers.<name>.enableMcp`) to `false` (or `true`).
|
|
291
325
|
- In environment variables: set `LITELLM_PROVIDER_<NAME>_ENABLE_MCP` or `LITELLM_PROVIDER_<NAME>_MCP_ENABLED` to `"0"` or `"false"` to disable MCP for a specific provider.
|
|
292
326
|
|
|
327
|
+
The three features also read global environment variables that override the `settings.json` block: `LITELLM_ENABLE_SKILLS`, `LITELLM_ENABLE_MCP`, and `LITELLM_ENABLE_BUDGET`. A set value replaces the block's `enabled`, read like every other boolean here: `1` or `true` (in any case) turns the feature on, and anything else, including `0` and `false`, turns it off. An unset, empty or blank value defers to the block. A provider with MCP disabled stays without it under `LITELLM_ENABLE_MCP=1`. They let one settings file serve hosts whose features differ, and a container or CI job can run with model sync only: `LITELLM_ENABLE_MCP=0 LITELLM_ENABLE_SKILLS=0 LITELLM_ENABLE_BUDGET=0 pi`.
|
|
328
|
+
|
|
293
329
|
Restart Pi after changing these settings.
|
|
294
330
|
|
|
295
331
|
`mcp.exposure` and `mcp.toolExposure` are passed to Pi unchanged and mean what they mean in a Pi `mcp.json` server entry: `codemode` (Pi's default when `exposure` is unset; callable from Pi's `codemode` scripts), `deferred` (declared to the model once Pi's `tool_search` loads it), `direct` (declared on every request), or `hidden`. `toolExposure` keys are tool names as LiteLLM's `/mcp` endpoint offers them, which LiteLLM prefixes with the server name, for example `github-search_code`; `*` matches any characters. Pi validates both settings, and an unusable value is reported once.
|
|
@@ -355,9 +391,9 @@ LiteLLM 1.104.0 and later answer such a request with 422, and earlier releases w
|
|
|
355
391
|
|
|
356
392
|
## Model transport
|
|
357
393
|
|
|
358
|
-
A `/model/info` route group uses native Anthropic `/v1/messages` only when every deployment explicitly reports Chat mode, identifies a Claude backend through an Anthropic-, Bedrock-, or
|
|
394
|
+
A `/model/info` route group uses native Anthropic `/v1/messages` only when every deployment explicitly reports Chat mode, identifies a Claude backend through an Anthropic-, Bedrock-, Vertex-, or Azure AI-capable adapter (`anthropic`, `bedrock`, `bedrock_converse`, `vertex_ai`, `vertex_ai-anthropic_models`, `azure_ai`), and every declared backend identity resolves the same Anthropic compatibility policy. Claude routed through Azure AI Foundry (`litellm_provider: azure_ai`, an `azure_ai/` model prefix, or `custom_llm_provider: azure_ai`, which covers a deployment LiteLLM reports as `anthropic` through its `base_model`) has two more conditions, both taken from LiteLLM: the configured `litellm_params.model` must contain `claude`, because LiteLLM selects its native Azure AI Messages config by that name alone and bridges any other deployment name to Chat Completions, and the proxy must report LiteLLM v1.80.12 or later. Earlier releases either have no native Messages config for `azure_ai` (before v1.80.10) or send the key in a header Azure's Anthropic endpoint refuses (v1.80.10 and v1.80.11). The version is read with the same single `GET /v1/responses/resp_version_probe` request described under [Protocols and prompt caching](#protocols-and-prompt-caching), sent only when such a group would otherwise be published on Messages; a pre-release or a proxy that omits `x-litellm-version` keeps it on Chat Completions. When the version cannot be read (no header, an unparsable value, or no answer), discovery warns once per route: `LiteLLM discovery: the proxy reported no readable version (x-litellm-version header); <n> azure_ai route(s) whose transport depends on it stay on Chat Completions: <routes>`. A `claude-<name>-<major>-<minor>` backend id resolves through its `claude-<name>-<major>` entry when the exact id is absent; an unknown generation stays on Chat Completions. When `supported_endpoints` is a list, it must include `/v1/messages`; an explicit endpoint list without it keeps the group on Chat Completions. A `null` value, which LiteLLM v1.100 and later report for models absent from their model map, counts as absent. An unresolved routing or base-model identity denies native Messages rather than being discarded from the unanimity check. Mixed-generation Claude groups can use Messages when their serializer policies match, with reasoning levels restricted to their common supported set. Groups with incompatible policies, mixed families, or unknown backend evidence remain on Chat Completions; unanimous explicit Responses mode takes precedence. Routes discovered through `/health` never select native Messages: Messages candidates are downgraded to Chat Completions with reasoning controls closed, while correlated Responses detail remains on Responses. Evidence-free `/v1/models` and wholly health-only `/health` groups take transport and presentation metadata from the bounded Pi catalog lookup: `openai-responses` when the catalog says so, otherwise Chat Completions. They expose no speculative reasoning selector and never select native Messages. Catalog identity such as Amazon Bedrock remains separate from the selected wire protocol.
|
|
359
395
|
|
|
360
|
-
Native Messages authenticates with `x-api-key`; every transport carries the `x-litellm-session-id` header described above. Strict JSON-schema tools are enabled separately from Messages transport and thinking compatibility: every deployment must affirm support for the actual route/API. Generic `supports_response_schema` and `supports_native_structured_output` metadata does not grant strict tools. Only deployments that LiteLLM reports with the `anthropic` adapter carry that evidence today. Claude served through Bedrock or
|
|
396
|
+
Native Messages authenticates with `x-api-key`; every transport carries the `x-litellm-session-id` header described above. Strict JSON-schema tools are enabled separately from Messages transport and thinking compatibility: every deployment must affirm support for the actual route/API. Generic `supports_response_schema` and `supports_native_structured_output` metadata does not grant strict tools. Only deployments that LiteLLM reports with the `anthropic` adapter carry that evidence today. Claude served through Bedrock, Vertex or Azure AI therefore uses ordinary tool serialization, and a tool that requires strict sampling fails before the request is sent. The same routing rule applies to managed effort, which adds `thinking.block_binding` and mid-conversation effort markers under their `anthropic-beta` flags: hosted Claude endpoints such as Azure reject that field, so a Messages route with any deployment not routed only through Anthropic declares `supportsMidConvoEffort: false`. Thinking is otherwise unchanged: an adaptive-thinking Claude model still sends `thinking.type: "adaptive"` with the selected level as `output_config.effort`.
|
|
361
397
|
|
|
362
398
|
## Image generation
|
|
363
399
|
|
|
@@ -421,6 +457,9 @@ such as OpenAI's GPT-6 Luna, is published as both a chat model and a classifier.
|
|
|
421
457
|
|---|---|---|
|
|
422
458
|
| `LITELLM_API_KEY_HELPER` | unset | Command that prints a fresh LiteLLM bearer token. Takes precedence over `LITELLM_API_KEY`. The extension runs it while resolving request auth, and Pi's per-request auth path is uncached, so rotating/short-lived tokens stay fresh. |
|
|
423
459
|
| `LITELLM_DISPLAY_NAME` | `LiteLLM` | Custom display name for the default `litellm` provider in the Pi UI. |
|
|
460
|
+
| `LITELLM_ENABLE_BUDGET` | unset (feature on) | When set, overrides `litellm.budget.enabled` from `settings.json`: `1` or `true` keeps the budget footer, polls and `/litellm-budget`, anything else, including `0` and `false`, removes them; unset, empty or blank defers to the block. See [Optional LiteLLM features](#optional-litellm-features). |
|
|
461
|
+
| `LITELLM_ENABLE_MCP` | unset (feature on) | When set, overrides `litellm.mcp.enabled` from `settings.json`: `1` or `true` registers the proxy's MCP server, anything else, including `0` and `false`, registers none; unset, empty or blank defers to the block. See [Optional LiteLLM features](#optional-litellm-features). |
|
|
462
|
+
| `LITELLM_ENABLE_SKILLS` | unset (feature on) | When set, overrides `litellm.skills.enabled` from `settings.json`: `1` or `true` keeps the Skills tools and prompt injection, anything else, including `0` and `false`, removes them; unset, empty or blank defers to the block. See [Optional LiteLLM features](#optional-litellm-features). |
|
|
424
463
|
| `LITELLM_HEADERS` | unset | JSON object of extra headers sent to LiteLLM provider, discovery, MCP, and Skills Gateway requests. Provider aliases can use it with `"headers": "$LITELLM_HEADERS"`. |
|
|
425
464
|
| `LITELLM_PROVIDERS` | unset | Canonical comma- or whitespace-separated list of provider IDs to register. |
|
|
426
465
|
| `LITELLM_PROVIDERS_JSON` | unset | JSON string of provider configuration objects matching the `litellm.providers` settings schema. |
|
|
@@ -558,7 +597,7 @@ Each distinct availability diagnostic is written once per session on stderr. `LI
|
|
|
558
597
|
|
|
559
598
|
### Protocols and prompt caching
|
|
560
599
|
|
|
561
|
-
Discovery selects native Messages for proven Claude groups under the [Model transport](#model-transport) rules, Responses for OpenAI-family backends, and Chat Completions for other OpenAI-compatible backends. Azure deployments stay on Chat Completions unless an explicit Responses mode or a `supported_endpoints` list containing `/v1/responses` authorizes Responses; `api_version` alone does not prove support. Azure AI (`azure_ai/` prefix or `custom_llm_provider: azure_ai`) deployments additionally depend on the proxy version. On LiteLLM before v1.103.0, or when the version is unknown, they ignore a `supported_endpoints` list that contains `/v1/responses` and need an explicit `mode: "responses"`: those releases have no native Responses config for that provider, so they bridge `/v1/responses` through Chat Completions, and the bridge fails streaming with `list index out of range` on Azure's empty-choices chunks (BerriAI/litellm#34455) while the endpoint list, copied from the model cost map, still advertises `/v1/responses`. From v1.103.0 the list is honored again. Discovery reads the version from the `x-litellm-version` header of one extra request, `GET /v1/responses/resp_version_probe`, sent only when such a deployment is published; LiteLLM answers it with an error and calls no model. A pre-release counts as older than the release it names, and a proxy that omits the header keeps these deployments on Chat Completions. Azure is detected from the adapter field, an `azure/` or `azure_ai/` model prefix, or the reported provider; other adapters follow the backend family because LiteLLM bridges `/v1/responses` to Chat Completions when the provider has no native Responses config (`litellm/responses/main.py`, `_bridges_to_chat_completions`). The family comes from the model id, not from LiteLLM's `openai/`, `openai_like/` or `custom_openai/` adapter prefix, so `openai/qwen3` stays on Chat Completions while `openai/gpt-5.5` uses Responses; a deployment opts into Responses with `mode: "responses"` or a `supported_endpoints` list containing `/v1/responses`. A route whose `model_info.base_model` prefix and `custom_llm_provider` name different providers is treated as unidentified and stays on Chat Completions, so it also does not get the local 128-tool preflight. When `/model/info` supplies `supported_endpoints`, that list takes precedence, and `mode: "responses"` remains an explicit Responses signal. An evidence-free fallback entry—a bare `/v1/models` id or a `/health` entry with no detail row—takes its protocol from the Pi catalog entry for its id; both evidence-free fallback paths also inherit presentation metadata from the bounded catalog lookup. An unknown id stays on Chat Completions. The route name alone authorizes nothing. A concrete id expanded from wildcard `/model/info` routes inherits the deployment evidence of the wildcard route LiteLLM would select for it, with the requested id substituted into the row's `model` the way LiteLLM serves it; the id is omitted when that selected route is not a published chat-style route. Chat Completions and Responses use `<root>/v1`; native Messages uses the proxy root directly. All three protocols pass through the provider's host guard. A model that reaches the provider with Pi's `azure-openai-responses` API, for example a stored entry taken from Pi's Azure catalog, is served as `openai-responses` at `<root>/v1` with the proxy credential, never with Azure URLs or `api-key` auth; any other API the proxy does not serve is still rejected before a request is sent. When an evidence-free cached `(no metadata)` entry can be enriched from the Pi catalog, it is restored on Responses only for a catalog Responses model and otherwise on Chat Completions, never Anthropic Messages. A model supplied only through `models.json` whose `api` this provider does not implement takes Pi's global API fallback rather than this provider's host guard; the credential-root pin (see [Model host enforcement](#model-host-enforcement)) still keeps the credential on the active proxy.
|
|
600
|
+
Discovery selects native Messages for proven Claude groups under the [Model transport](#model-transport) rules, Responses for OpenAI-family backends, and Chat Completions for other OpenAI-compatible backends. Azure deployments stay on Chat Completions unless an explicit Responses mode or a `supported_endpoints` list containing `/v1/responses` authorizes Responses; `api_version` alone does not prove support. Azure AI (`azure_ai/` prefix or `custom_llm_provider: azure_ai`) deployments additionally depend on the proxy version. On LiteLLM before v1.103.0, or when the version is unknown, they ignore a `supported_endpoints` list that contains `/v1/responses` and need an explicit `mode: "responses"`: those releases have no native Responses config for that provider, so they bridge `/v1/responses` through Chat Completions, and the bridge fails streaming with `list index out of range` on Azure's empty-choices chunks (BerriAI/litellm#34455) while the endpoint list, copied from the model cost map, still advertises `/v1/responses`. From v1.103.0 the list is honored again. Discovery reads the version from the `x-litellm-version` header of one extra request, `GET /v1/responses/resp_version_probe`, sent only when such a deployment (or an `azure_ai` Claude group awaiting native Messages, see [Model transport](#model-transport)) is published; LiteLLM answers it with an error and calls no model. A pre-release counts as older than the release it names, and a proxy that omits the header keeps these deployments on Chat Completions and draws the version warning described under [Model transport](#model-transport). Azure is detected from the adapter field, an `azure/` or `azure_ai/` model prefix, or the reported provider; other adapters follow the backend family because LiteLLM bridges `/v1/responses` to Chat Completions when the provider has no native Responses config (`litellm/responses/main.py`, `_bridges_to_chat_completions`). The family comes from the model id, not from LiteLLM's `openai/`, `openai_like/` or `custom_openai/` adapter prefix, so `openai/qwen3` stays on Chat Completions while `openai/gpt-5.5` uses Responses; a deployment opts into Responses with `mode: "responses"` or a `supported_endpoints` list containing `/v1/responses`. A route whose `model_info.base_model` prefix and `custom_llm_provider` name different providers is treated as unidentified and stays on Chat Completions, so it also does not get the local 128-tool preflight. When `/model/info` supplies `supported_endpoints`, that list takes precedence, and `mode: "responses"` remains an explicit Responses signal. An evidence-free fallback entry—a bare `/v1/models` id or a `/health` entry with no detail row—takes its protocol from the Pi catalog entry for its id; both evidence-free fallback paths also inherit presentation metadata from the bounded catalog lookup. An unknown id stays on Chat Completions. The route name alone authorizes nothing. A concrete id expanded from wildcard `/model/info` routes inherits the deployment evidence of the wildcard route LiteLLM would select for it, with the requested id substituted into the row's `model` the way LiteLLM serves it; the id is omitted when that selected route is not a published chat-style route. Chat Completions and Responses use `<root>/v1`; native Messages uses the proxy root directly. All three protocols pass through the provider's host guard. A model that reaches the provider with Pi's `azure-openai-responses` API, for example a stored entry taken from Pi's Azure catalog, is served as `openai-responses` at `<root>/v1` with the proxy credential, never with Azure URLs or `api-key` auth; any other API the proxy does not serve is still rejected before a request is sent. When an evidence-free cached `(no metadata)` entry can be enriched from the Pi catalog, it is restored on Responses only for a catalog Responses model and otherwise on Chat Completions, never Anthropic Messages. A model supplied only through `models.json` whose `api` this provider does not implement takes Pi's global API fallback rather than this provider's host guard; the credential-root pin (see [Model host enforcement](#model-host-enforcement)) still keeps the credential on the active proxy.
|
|
562
601
|
|
|
563
602
|
`cacheControlFormat: "anthropic"` applies only to Chat Completions, where Pi adds Anthropic `cache_control` markers. The Responses transport has a different compatibility type and uses native `prompt_cache_key` (plus supported retention fields), so Responses models must not receive `cacheControlFormat`. Freshly discovered OpenAI-family models forced onto Chat Completions are rejected locally when a request contains more than 128 tools; route them through Responses or reduce enabled extensions. Moonshot request-side reasoning suppression follows LiteLLM's routing provider, not `model_info.base_model`; response-side `<think>` normalization follows discovered Kimi-family evidence. Opaque aliases retain display normalization without sending Moonshot-only parameters to Azure- or Bedrock-hosted Kimi models. Strict tool-message repair requires every deployment to identify Kimi; Gemini effort normalization likewise requires consistent backend-family evidence. Claude-backed routes on native Messages or Chat Completions declare Pi's `promptCache` lifetimes (`short: 300`, `long: 3600` seconds, the values Pi's built-in Anthropic models use), so Pi's `cacheWarming` can keep their caches warm. Responses routes whose every deployment is GPT-5.6 or later and routes through LiteLLM's `openai`, `azure` or `azure_ai` provider declare `short: 1800`, the 30-minute default minimum lifetime OpenAI and Azure document for those models, and no `long` value. Claude routes on Responses, Chat routes to OpenAI models, earlier GPT models, GPT models served through a relay such as OpenRouter or GitHub Copilot, routes identified only by their name, and other backend families declare none, so Pi does not warm them. A `modelOverrides` entry for the route in the global `models.json` can change the values, and `cacheWarming: "off"` disables warming.
|
|
564
603
|
|
|
@@ -566,7 +605,7 @@ Discovery policy is versioned in Pi's persisted model catalog. After upgrading,
|
|
|
566
605
|
|
|
567
606
|
### Deployment groups and metadata authority
|
|
568
607
|
|
|
569
|
-
LiteLLM may load-balance one public `model_name` across deployments with different backends or model versions. The extension reduces `/model/info` rows conservatively before publishing one Pi model. A route group that mixes a chat-style deployment with an explicitly incompatible mode such as embedding is withheld entirely and reported with a bounded diagnostic.
|
|
608
|
+
LiteLLM may load-balance one public `model_name` across deployments with different backends or model versions. The extension reduces `/model/info` rows conservatively before publishing one Pi model. A deployment paused in LiteLLM's Model Management (`model_info.blocked: true`, LiteLLM v1.87.0 and later) is left out of the reduction because the router never serves it, and a route whose every deployment is paused is not published. A route group that mixes a chat-style deployment with an explicitly incompatible mode such as embedding is withheld entirely and reported with a bounded diagnostic.
|
|
570
609
|
|
|
571
610
|
- Responses is selected only when every deployment supports it under the protocol rules above; a deployment that requires Chat keeps the group on Chat.
|
|
572
611
|
- Vision and reasoning are advertised only when every routable deployment resolves them as supported. An explicit `supports_reasoning: false` suppresses all thinking controls, even when catalog or router effort metadata exists. Catalog thinking controls are intersected per level: a level is exposed only when every deployment maps it to the same value, while disagreement or absence becomes an explicit denial. Catalog maps are not complete effort lists, so absence has one exception: an omitted `minimal`, `low`, `medium` or `high` counts as Pi's default spelling of that level. An omitted `off`, `xhigh` or `max` still differs from an explicit mapping. Reasoning selectors additionally require an accepted wire carrier on every deployment. Public effort lists are intersected, explicit LiteLLM denials win, and `xhigh`/`max` need unanimous explicit support. Null or absent standard-effort flags have no opinion. A deployment that declares `model_info.reasoning_effort_levels` answers every level from that list, ahead of its per-level flags, as LiteLLM itself reads it.
|
|
@@ -583,7 +622,7 @@ LiteLLM may load-balance one public `model_name` across deployments with differe
|
|
|
583
622
|
```
|
|
584
623
|
|
|
585
624
|
Pi catalogue lookup prefers the deployment's provider-specific entry before falling back to the generic vendor entry; Azure and Azure AI therefore retain Azure-specific limits rather than being capped by a different OpenAI catalogue entry. When models.dev supplies a sparse vendor record, missing fields still use the provider-specific Pi catalogue.
|
|
586
|
-
- Each displayed base-price field uses the maximum only when every deployment resolves that field; unresolved fields remain zero and the model name is suffixed with ` (incomplete metadata)`. LiteLLM omits `cache_creation_input_token_cost` for backends that bill no cache write, so a deployment's missing cache-write price resolves to zero when its own catalog entry prices cache writes at zero (even when the group withholds catalog authority), or, when no catalog entry prices them, when its backend is DeepSeek.
|
|
625
|
+
- Each displayed base-price field uses the maximum only when every deployment resolves that field; unresolved fields remain zero and the model name is suffixed with ` (incomplete metadata)`. LiteLLM omits `cache_creation_input_token_cost` for backends that bill no cache write, so a deployment's missing cache-write price resolves to zero when its own catalog entry prices cache writes at zero (even when the group withholds catalog authority), or, when no catalog entry prices them, when its backend is DeepSeek. Otherwise a cache price the deployment omits takes its explicit `input_cost_per_token`, the rate LiteLLM bills it at (v1.103.0 and later; earlier releases bill it at zero, so the figure is an upper bound there): a cache-read price when the deployment's own catalog entry prices no cache read, and a cache-write price when that entry prices no cache write and the backend's family is unidentified. A self-hosted route priced with input and output only is therefore complete. A missing cache-write price for an identified family, Claude and GPT-5.6+ included, stays unresolved, because those backends can bill writes above the input rate.
|
|
587
626
|
- When complete base pricing comes from the catalog under unanimous catalog authority, tiered pricing is the conservative worst-case envelope. Its usable finite non-negative thresholds are the sorted union of every deployment's thresholds, and each price field at each interval is the maximum applicable rate across all deployments, floored by each deployment's base rate. Differing tier breakpoints are therefore preserved rather than treated as incompatible. An explicit `/model/info` base-price field replaces catalog pricing for that field at every threshold; catalog tiers remain for unaffected fields and are omitted only when every base-price field is explicit.
|
|
588
627
|
- Reduced route groups omit tier ladders when complete base-pricing or catalog authority is unavailable. Wildcard expansion retains every known tier from matching groups even when another match has incomplete metadata: suppressing a complete sibling's higher tier could understate the known worst-case rate. The ` (incomplete metadata)` marker remains because the resulting envelope is only as complete as the available ladders.
|
|
589
628
|
- Wildcard routes expand through `/v1/models`; dropped exact or wildcard groups suppress matching expanded ids.
|
|
@@ -593,7 +632,7 @@ LiteLLM may load-balance one public `model_name` across deployments with differe
|
|
|
593
632
|
|
|
594
633
|
The ` (no metadata)` suffix is reserved for unresolved, evidence-free entries mapped by the full `/v1/models` fallback or wholly health-only `/health` groups. During wildcard expansion after a successful `/model/info` response, IDs that match no surviving wildcard route are discarded. A matched wildcard expansion inherits its matching groups' authority and is either unmarked or marked ` (incomplete metadata)`, never ` (no metadata)`. A recognized `owned_by` provider that conflicts with a provider-qualified model ID instead receives ` (incomplete metadata)`, permanently withholding cache enrichment rather than falling through to the ID prefix. Only an unchanged evidence-free sentinel shape is eligible for bounded catalog enrichment on a later cache read, when it also refreshes provider compatibility metadata from the model ID, adopts Responses only for a catalog `openai-responses` model, and otherwise uses Chat.
|
|
595
634
|
|
|
596
|
-
The ` (incomplete metadata)` suffix marks reduced `/model/info` groups, incomplete matched wildcard expansions, or `/health` groups with deployment details and permanently prevents route-name cache enrichment. It means at least one metadata field is unknown, including cache pricing that the proxy omitted
|
|
635
|
+
The ` (incomplete metadata)` suffix marks reduced `/model/info` groups, incomplete matched wildcard expansions, or `/health` groups with deployment details and permanently prevents route-name cache enrichment. It means at least one metadata field is unknown, including cache pricing that the proxy omitted and neither the zero cache-write evidence nor the input-rate rule above resolves; known input/output prices may still be shown alongside the suffix.
|
|
597
636
|
|
|
598
637
|
### Reasoning controls
|
|
599
638
|
|
|
@@ -623,6 +662,7 @@ The development probe runs against minimized snapshots with `npm run probe:proxy
|
|
|
623
662
|
| `LiteLLM discovery: ... route group(s) mix chat-style and explicitly incompatible deployment modes` | A public route can select both a chat-style deployment and an incompatible deployment such as an embedding model. Give the embedding deployment its own `model_name`. |
|
|
624
663
|
| `LiteLLM (...): the request for "<route>" was answered by a fallback` | A successful HTTP response reports a LiteLLM router fallback (`x-litellm-attempted-fallbacks`). `/model/info` does not expose fallbacks, so protocol and model handling follow the requested route's own deployments. If the primary and fallback groups support Chat Completions, set `model_info.supported_endpoints: ["/v1/chat/completions"]` on the primary to avoid Responses-to-Chat bridging. This warning does not change routing or fix upstream bridging defects. It names only the requested route; LiteLLM reports the group and model that answered in the `x-litellm-model-group` and `x-litellm-model-name` response headers. Warned once per provider and route until the extension reloads; shown in the UI, or stderr without a UI. |
|
|
625
664
|
| A model is marked ` (incomplete metadata)` | `/model/info` or `/health` did not provide enough authoritative metadata. Explicit fields remain usable, but unknown cost fields are shown as zero and route-name cache enrichment stays disabled. |
|
|
665
|
+
| `the proxy reported no readable version (x-litellm-version header)` | The named `azure_ai` routes would use Responses or native Messages on a recent LiteLLM, but the proxy (or something in front of it) did not return `x-litellm-version` on the probe request, so they stay on Chat Completions. Let the header through, or for Responses set `mode: "responses"` on the deployment. See [Model transport](#model-transport). |
|
|
626
666
|
| Cache miss notices after idle, no cache warming | The model declares no `promptCache`: only Claude-backed Messages and Chat routes and GPT-5.6+ Responses routes on OpenAI or Azure get one. Run `/litellm-refresh` after updating so stored models are rediscovered. |
|
|
627
667
|
| `401 Token expired` | Set `LITELLM_API_KEY_HELPER`. |
|
|
628
668
|
| No models with gcloud auth | Verify `gcloud auth application-default login` has been run or set `GOOGLE_APPLICATION_CREDENTIALS` to an `authorized_user` ADC file |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-provider-litellm",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.9.0",
|
|
4
4
|
"description": "LiteLLM proxy provider extension for Pi",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
"@typescript/native-preview": "7.0.0-dev.20260707.2",
|
|
75
75
|
"es-module-lexer": "^3.0.2",
|
|
76
76
|
"tsx": "^4.21.0",
|
|
77
|
-
"typebox": "1.4.
|
|
77
|
+
"typebox": "1.4.7",
|
|
78
78
|
"vitest": "^5.0.0"
|
|
79
79
|
}
|
|
80
80
|
}
|
package/src/backend-identity.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export const LITELLM_DISCOVERY_VERSION =
|
|
1
|
+
export const LITELLM_DISCOVERY_VERSION = 16 as const;
|
|
2
2
|
|
|
3
3
|
export type BackendFamily = "claude" | "deepseek" | "gemini" | "kimi" | "openai";
|
|
4
4
|
|
|
@@ -17,6 +17,10 @@ export interface BackendIdentity {
|
|
|
17
17
|
|
|
18
18
|
const GENERIC_ADAPTERS = new Set(["azure", "azure_ai", "custom_openai", "openai_like"]);
|
|
19
19
|
const GENERIC_CATALOG_PROVIDERS = new Set([...GENERIC_ADAPTERS, "openai", "text-completion-openai"]);
|
|
20
|
+
// LiteLLM's `openai` provider is an OpenAI-compatible transport, so the name carries no backend
|
|
21
|
+
// vendor. A model or base_model id there may namespace itself by vendor
|
|
22
|
+
// (`commandcode/deepseek/deepseek-v4.1-flash`) without contradicting the routing provider.
|
|
23
|
+
const OPENAI_TRANSPORT_PROVIDERS = new Set(["openai", "text-completion-openai"]);
|
|
20
24
|
// Providers whose name alone settles the family. `openai` is deliberately absent: LiteLLM
|
|
21
25
|
// uses that provider for any OpenAI-compatible server, so it says nothing about the model.
|
|
22
26
|
const PROVIDER_FAMILIES: Readonly<Record<string, BackendFamily>> = {
|
|
@@ -80,10 +84,22 @@ export function resolveBackendIdentity(row: BackendIdentityRow): BackendIdentity
|
|
|
80
84
|
const custom = wireString(row.litellm_params?.custom_llm_provider)?.toLowerCase();
|
|
81
85
|
const customProvider = custom && !GENERIC_ADAPTERS.has(custom) ? custom : undefined;
|
|
82
86
|
if (prefixProvider && customProvider && prefixProvider !== customProvider) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
prefixProvider
|
|
87
|
+
// An OpenAI-compatible transport says nothing about the backend, so a differing first segment
|
|
88
|
+
// on a base_model is a vendor namespace there too; only a real routing provider contradicts it.
|
|
89
|
+
const namespacesOverTransport = OPENAI_TRANSPORT_PROVIDERS.has(customProvider);
|
|
90
|
+
if ((baseModel || !configuredModel) && !MODEL_PATH_SEGMENTS.has(prefixProvider) && !namespacesOverTransport)
|
|
91
|
+
return undefined;
|
|
92
|
+
if (namespacesOverTransport && OPENAI_TRANSPORT_PROVIDERS.has(prefixProvider)) {
|
|
93
|
+
// Both segments name the same transport ("openai/qwen3" on a "text-completion-openai"
|
|
94
|
+
// route), so the prefix carries no vendor and must not reach the family scan: dropping it
|
|
95
|
+
// keeps "qwen3" familyless instead of reading it as OpenAI. A transport prefix on a real
|
|
96
|
+
// provider is vendor evidence and stays ("openai/gpt-4o" on OpenRouter).
|
|
97
|
+
prefixProvider = undefined;
|
|
98
|
+
} else {
|
|
99
|
+
raw = `${customProvider}/${raw}`;
|
|
100
|
+
slash = customProvider.length;
|
|
101
|
+
prefixProvider = customProvider;
|
|
102
|
+
}
|
|
87
103
|
}
|
|
88
104
|
const provider = prefixProvider ?? customProvider;
|
|
89
105
|
const modelId = slash > 0 ? raw.slice(slash + 1) : raw;
|
package/src/discover.ts
CHANGED
|
@@ -439,8 +439,13 @@ function anthropicBackendLookupIds(id: string): string[] {
|
|
|
439
439
|
// These are the LiteLLM adapters whose native request path can terminate at a
|
|
440
440
|
// Claude backend. Other adapters may expose Claude-like public aliases, but an
|
|
441
441
|
// alias alone is not evidence that LiteLLM accepts the Anthropic Messages schema.
|
|
442
|
+
// `azure_ai` (Azure AI Foundry) is a generic adapter that serves many vendors, so
|
|
443
|
+
// it qualifies only under `isAzureAiClaudeDeployment` and the proxy-version floor.
|
|
444
|
+
// Both also bind a deployment LiteLLM reports as `anthropic` (the cost-map provider
|
|
445
|
+
// of its `base_model`) while routing it through azure_ai.
|
|
442
446
|
const CLAUDE_CAPABLE_ADAPTERS = new Set([
|
|
443
447
|
"anthropic",
|
|
448
|
+
"azure_ai",
|
|
444
449
|
"bedrock",
|
|
445
450
|
"bedrock_converse",
|
|
446
451
|
"vertex_ai",
|
|
@@ -448,11 +453,41 @@ const CLAUDE_CAPABLE_ADAPTERS = new Set([
|
|
|
448
453
|
]);
|
|
449
454
|
const CLAUDE_MODEL_PATTERN = /(?:^|[./_-])(?:claude|opus|sonnet|haiku|fable)(?:$|[./_:-])/i;
|
|
450
455
|
|
|
456
|
+
// LiteLLM picks its native azure_ai Messages config by `"claude" in model.lower()`
|
|
457
|
+
// on the configured model alone and bridges a deployment named otherwise to Chat
|
|
458
|
+
// Completions, so a deployment routed through azure_ai stays on Chat here too.
|
|
459
|
+
function isAzureAiClaudeDeployment(entry: ModelInfoEntry): boolean {
|
|
460
|
+
return wireString(entry.litellm_params?.model)?.toLowerCase().includes("claude") === true;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// LiteLLM registers that config in v1.80.10, but v1.80.10 and v1.80.11 send the
|
|
464
|
+
// key as `api-key`, which Azure's Anthropic endpoint refuses; v1.80.12 restores
|
|
465
|
+
// `x-api-key`. Earlier releases bridge `/v1/messages` to Chat Completions.
|
|
466
|
+
const AZURE_AI_MESSAGES_FLOOR: ProxyVersionFloor = [1, 80, 12];
|
|
467
|
+
const AZURE_AI_MESSAGES_FLOOR_VERSION: ProxyVersion = {
|
|
468
|
+
major: AZURE_AI_MESSAGES_FLOOR[0],
|
|
469
|
+
minor: AZURE_AI_MESSAGES_FLOOR[1],
|
|
470
|
+
patch: AZURE_AI_MESSAGES_FLOOR[2],
|
|
471
|
+
prerelease: false,
|
|
472
|
+
};
|
|
473
|
+
|
|
474
|
+
function routesThroughAzureAi(entry: ModelInfoEntry): boolean {
|
|
475
|
+
return (
|
|
476
|
+
wireString(entry.model_info?.litellm_provider)?.trim().toLowerCase() === "azure_ai" || isAzureAiDeployment(entry)
|
|
477
|
+
);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
// Whether the proxy version decides this deployment's Messages transport.
|
|
481
|
+
function messagesAwaitProxyVersion(entry: ModelInfoEntry): boolean {
|
|
482
|
+
return routesThroughAzureAi(entry) && nativeMessagesCatalog(entry).messagesCompat !== undefined;
|
|
483
|
+
}
|
|
484
|
+
|
|
451
485
|
function nativeMessagesCatalog(
|
|
452
486
|
entry: ModelInfoEntry,
|
|
453
487
|
): Pick<CatalogResolution, "messagesCompat" | "messagesStrictTools" | "messagesThinkingLevelMap"> {
|
|
454
488
|
const adapter = wireString(entry.model_info?.litellm_provider)?.trim().toLowerCase();
|
|
455
489
|
if (!adapter || !CLAUDE_CAPABLE_ADAPTERS.has(adapter) || deploymentFamily(entry) !== "claude") return {};
|
|
490
|
+
if (routesThroughAzureAi(entry) && !isAzureAiClaudeDeployment(entry)) return {};
|
|
456
491
|
const candidates = [entry.litellm_params?.model, entry.model_info?.base_model]
|
|
457
492
|
.map((id) => wireString(id)?.trim())
|
|
458
493
|
.filter((id): id is string => Boolean(id));
|
|
@@ -689,6 +724,21 @@ function reportAmbiguousCatalogAuthority(report: Report, routes: readonly string
|
|
|
689
724
|
);
|
|
690
725
|
}
|
|
691
726
|
|
|
727
|
+
const reportedUnreadVersionRoutes = new Set<string>();
|
|
728
|
+
|
|
729
|
+
// The version decides these routes' transport, and nothing else tells an operator
|
|
730
|
+
// why they stay on the workaround when the proxy does not say which release it is.
|
|
731
|
+
function reportUnreadProxyVersion(report: Report, routes: readonly string[]): void {
|
|
732
|
+
reportBoundedRoutes(
|
|
733
|
+
report,
|
|
734
|
+
reportedUnreadVersionRoutes,
|
|
735
|
+
routes,
|
|
736
|
+
(count) =>
|
|
737
|
+
`LiteLLM discovery: the proxy reported no readable version (x-litellm-version header); ${count} azure_ai ` +
|
|
738
|
+
"route(s) whose transport depends on it stay on Chat Completions",
|
|
739
|
+
);
|
|
740
|
+
}
|
|
741
|
+
|
|
692
742
|
const reportedIncompatibleModeRoutes = new Set<string>();
|
|
693
743
|
|
|
694
744
|
function reportIncompatibleDeploymentModes(report: Report, routes: readonly string[]): void {
|
|
@@ -732,7 +782,10 @@ function mapFromModelInfoGroup(
|
|
|
732
782
|
const model = new Set(generations).size === 1 && family !== "conflicting" ? generations[0] : undefined;
|
|
733
783
|
return {
|
|
734
784
|
...catalog,
|
|
735
|
-
...(options.allowMessages === false
|
|
785
|
+
...(options.allowMessages === false ||
|
|
786
|
+
(messagesAwaitProxyVersion(entry) && !proxyVersionAtLeast(options.proxyVersion, AZURE_AI_MESSAGES_FLOOR))
|
|
787
|
+
? { messagesCompat: undefined, messagesStrictTools: undefined }
|
|
788
|
+
: {}),
|
|
736
789
|
...(family ? { semanticFamily: family } : {}),
|
|
737
790
|
...(model ? { semanticModel: model } : {}),
|
|
738
791
|
};
|
|
@@ -865,8 +918,17 @@ function mapFromModelInfoGroup(
|
|
|
865
918
|
function publishesVersionGatedTransport(
|
|
866
919
|
group: readonly ModelInfoEntry[],
|
|
867
920
|
publicCatalog: PublicCatalog | undefined,
|
|
921
|
+
allowMessages = true,
|
|
868
922
|
): boolean {
|
|
869
|
-
|
|
923
|
+
if (group.some(transportAwaitsProxyVersion)) return mapFromModelInfoGroup(group, publicCatalog) !== undefined;
|
|
924
|
+
// Messages awaits the version only where a proxy at the floor would get it: an
|
|
925
|
+
// endpoint list or a sibling deployment can keep the group on Chat regardless.
|
|
926
|
+
return (
|
|
927
|
+
allowMessages &&
|
|
928
|
+
group.some(messagesAwaitProxyVersion) &&
|
|
929
|
+
mapFromModelInfoGroup(group, publicCatalog, { proxyVersion: AZURE_AI_MESSAGES_FLOOR_VERSION })?.api ===
|
|
930
|
+
"anthropic-messages"
|
|
931
|
+
);
|
|
870
932
|
}
|
|
871
933
|
|
|
872
934
|
function catalogProtocol(modelId: string, catalogModel: Model<Api> | undefined): ModelProtocol {
|
|
@@ -902,6 +964,13 @@ function mapFromModelsList(entry: ModelsListEntry): DiscoveredModel | undefined
|
|
|
902
964
|
};
|
|
903
965
|
}
|
|
904
966
|
|
|
967
|
+
// A deployment paused in LiteLLM's Model Management (v1.87.0+) keeps its `/model/info` row with
|
|
968
|
+
// `model_info.blocked: true`, but the router never serves it and `/v1/models` hides a route whose
|
|
969
|
+
// every deployment is paused. It is no evidence for its route and publishes nothing.
|
|
970
|
+
function isPaused(entry: ModelInfoEntry | undefined): boolean {
|
|
971
|
+
return entry?.model_info?.blocked === true;
|
|
972
|
+
}
|
|
973
|
+
|
|
905
974
|
async function discoverFromHealth(
|
|
906
975
|
base: string,
|
|
907
976
|
apiKey: string,
|
|
@@ -938,7 +1007,7 @@ async function discoverFromHealth(
|
|
|
938
1007
|
if (completed % 10 === 0 || completed === endpoints.length) {
|
|
939
1008
|
progress?.(`Fetched ${completed}/${endpoints.length} models...`);
|
|
940
1009
|
}
|
|
941
|
-
deployments[index] = healthDeployment(detail, route, deploymentId);
|
|
1010
|
+
if (!isPaused(detail)) deployments[index] = healthDeployment(detail, route, deploymentId);
|
|
942
1011
|
}
|
|
943
1012
|
};
|
|
944
1013
|
await Promise.all(Array.from({ length: Math.min(HEALTH_DETAIL_CONCURRENCY, endpoints.length) }, () => worker()));
|
|
@@ -953,14 +1022,18 @@ async function discoverFromHealth(
|
|
|
953
1022
|
const publicCatalog = deployments.some((deployment) => deployment && !deployment.synthetic)
|
|
954
1023
|
? await loadDiscoveryPublicCatalog(options)
|
|
955
1024
|
: undefined;
|
|
956
|
-
const
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
1025
|
+
const versionGatedRoutes = [...groups]
|
|
1026
|
+
.filter(([, group]) =>
|
|
1027
|
+
// Health discovery never selects Messages, so only Responses can await the version.
|
|
1028
|
+
publishesVersionGatedTransport(
|
|
1029
|
+
group.map(({ entry }) => entry),
|
|
1030
|
+
publicCatalog,
|
|
1031
|
+
false,
|
|
1032
|
+
),
|
|
1033
|
+
)
|
|
1034
|
+
.map(([route]) => route);
|
|
1035
|
+
const proxyVersion = versionGatedRoutes.length > 0 ? await probeProxyVersion(base, apiKey, options) : undefined;
|
|
1036
|
+
if (!proxyVersion) reportUnreadProxyVersion(report, versionGatedRoutes);
|
|
964
1037
|
const incompatibleModeRoutes: string[] = [];
|
|
965
1038
|
const ambiguousRoutes: string[] = [];
|
|
966
1039
|
const conflictingFamilyRoutes: string[] = [];
|
|
@@ -1232,7 +1305,7 @@ export async function discoverModels(
|
|
|
1232
1305
|
const infoResult = await fetchJson<ModelInfoResponse>(`${base}/model/info`, apiKey, options);
|
|
1233
1306
|
// An empty `/model/info` list is treated like an unavailable endpoint: some proxies
|
|
1234
1307
|
// answer 200 with no rows for keys that can still list models through `/v1/models`.
|
|
1235
|
-
const infoEntries = infoResult.ok ? (infoResult.data.data ?? []) : [];
|
|
1308
|
+
const infoEntries = (infoResult.ok ? (infoResult.data.data ?? []) : []).filter((entry) => !isPaused(entry));
|
|
1236
1309
|
if (infoEntries.length > 0) {
|
|
1237
1310
|
const groups = new Map<string, ModelInfoEntry[]>();
|
|
1238
1311
|
for (const entry of infoEntries) {
|
|
@@ -1251,11 +1324,10 @@ export async function discoverModels(
|
|
|
1251
1324
|
proxyVersionRead = true;
|
|
1252
1325
|
return probeProxyVersion(base, apiKey, options);
|
|
1253
1326
|
};
|
|
1254
|
-
|
|
1255
|
-
([route, group]) => !route.includes("*") && publishesVersionGatedTransport(group, publicCatalog)
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
: undefined;
|
|
1327
|
+
const versionGatedRoutes = [...groups]
|
|
1328
|
+
.filter(([route, group]) => !route.includes("*") && publishesVersionGatedTransport(group, publicCatalog))
|
|
1329
|
+
.map(([route]) => route);
|
|
1330
|
+
let proxyVersion = versionGatedRoutes.length > 0 ? await readProxyVersion() : undefined;
|
|
1259
1331
|
const ambiguousRoutes: string[] = [];
|
|
1260
1332
|
const incompatibleModeRoutes: string[] = [];
|
|
1261
1333
|
const conflictingFamilyRoutes: string[] = [];
|
|
@@ -1317,13 +1389,14 @@ export async function discoverModels(
|
|
|
1317
1389
|
);
|
|
1318
1390
|
});
|
|
1319
1391
|
let sources = wildcards;
|
|
1320
|
-
const
|
|
1392
|
+
const versionGatedExpansions = candidates.flatMap((entry) => {
|
|
1321
1393
|
const id = wireString(entry.id);
|
|
1322
|
-
if (!id || id.includes("*")) return
|
|
1394
|
+
if (!id || id.includes("*")) return [];
|
|
1323
1395
|
const parents = selectedWildcardRows(id, wildcardRows, publishedWildcardIds);
|
|
1324
|
-
return parents !== undefined && publishesVersionGatedTransport(parents, publicCatalog);
|
|
1396
|
+
return parents !== undefined && publishesVersionGatedTransport(parents, publicCatalog) ? [id] : [];
|
|
1325
1397
|
});
|
|
1326
|
-
|
|
1398
|
+
versionGatedRoutes.push(...versionGatedExpansions);
|
|
1399
|
+
if (!proxyVersionRead && versionGatedExpansions.length > 0) {
|
|
1327
1400
|
proxyVersion = await readProxyVersion();
|
|
1328
1401
|
// The parents were reduced before the version was known; an expansion
|
|
1329
1402
|
// must inherit from parents reduced under the version it is published with.
|
|
@@ -1353,6 +1426,7 @@ export async function discoverModels(
|
|
|
1353
1426
|
}
|
|
1354
1427
|
}
|
|
1355
1428
|
reportWithheldToolRepair(report, withheldRepairRoutes);
|
|
1429
|
+
if (!proxyVersion) reportUnreadProxyVersion(report, versionGatedRoutes);
|
|
1356
1430
|
models = deduplicateModels(models);
|
|
1357
1431
|
if (process.env.LITELLM_VERBOSE_DISCOVERY === "1") {
|
|
1358
1432
|
const defaultedWildcards = wildcardRoutes.filter(({ route }) => defaultedContextRoutes.has(route));
|
package/src/index.ts
CHANGED
|
@@ -781,10 +781,21 @@ function normalizeProviderSettings(raw: unknown): RawProviderSettings | undefine
|
|
|
781
781
|
};
|
|
782
782
|
}
|
|
783
783
|
|
|
784
|
-
|
|
784
|
+
// A feature's LITELLM_ENABLE_* variable overrides its settings block, read like every other boolean
|
|
785
|
+
// here: 1 or true switch the feature on and any other value off, so a misspelt kill switch still
|
|
786
|
+
// kills. An unset or blank value defers to the block.
|
|
787
|
+
const ENV_FEATURES: Record<"skills" | "mcp" | "budget", string> = {
|
|
788
|
+
skills: "LITELLM_ENABLE_SKILLS",
|
|
789
|
+
mcp: "LITELLM_ENABLE_MCP",
|
|
790
|
+
budget: "LITELLM_ENABLE_BUDGET",
|
|
791
|
+
};
|
|
792
|
+
|
|
793
|
+
export function isFeatureEnabled(
|
|
785
794
|
settings: Record<string, unknown> | undefined,
|
|
786
795
|
feature: "skills" | "mcp" | "budget",
|
|
787
796
|
): boolean {
|
|
797
|
+
const envValue = process.env[ENV_FEATURES[feature]];
|
|
798
|
+
if (envValue?.trim()) return parseBooleanSetting(envValue);
|
|
788
799
|
const raw = settings?.[feature];
|
|
789
800
|
return !isPlainObject(raw) || raw.enabled !== false;
|
|
790
801
|
}
|
package/src/model-groups.ts
CHANGED
|
@@ -671,6 +671,22 @@ function unbilledCacheWrite(catalog: CatalogResolution | undefined): boolean {
|
|
|
671
671
|
return declared === undefined ? catalog?.semanticFamily === "deepseek" : declared === 0;
|
|
672
672
|
}
|
|
673
673
|
|
|
674
|
+
// LiteLLM bills a cache rate the deployment omits at its input rate (v1.103.0 and later;
|
|
675
|
+
// zero before, so this is never an understatement), which makes the router's own input price
|
|
676
|
+
// the missing cache price. A catalog price for the field, usable or not, says the backend
|
|
677
|
+
// bills something else, and an identified family may bill writes above input (Claude,
|
|
678
|
+
// GPT-5.6+), so both keep the price unknown.
|
|
679
|
+
function inputRateCachePrice(
|
|
680
|
+
entry: ModelInfoEntry,
|
|
681
|
+
catalog: CatalogResolution | undefined,
|
|
682
|
+
field: CostField,
|
|
683
|
+
): number | undefined {
|
|
684
|
+
if (field !== "cacheRead" && field !== "cacheWrite") return undefined;
|
|
685
|
+
if (catalog?.cost?.[field] !== undefined) return undefined;
|
|
686
|
+
if (field === "cacheWrite" && catalog?.semanticFamily !== undefined) return undefined;
|
|
687
|
+
return explicitCost(entry, "input");
|
|
688
|
+
}
|
|
689
|
+
|
|
674
690
|
function min(values: readonly number[]): number {
|
|
675
691
|
return Math.min(...values);
|
|
676
692
|
}
|
|
@@ -863,7 +879,9 @@ export function reduceModelGroup(
|
|
|
863
879
|
deployments.map(
|
|
864
880
|
(entry, index) =>
|
|
865
881
|
resolvedCost(entry, catalogAuthority[index], field) ??
|
|
866
|
-
(field === "cacheWrite" && unbilledCacheWrite(catalogs[index])
|
|
882
|
+
(field === "cacheWrite" && unbilledCacheWrite(catalogs[index])
|
|
883
|
+
? 0
|
|
884
|
+
: inputRateCachePrice(entry, catalogs[index], field)),
|
|
867
885
|
),
|
|
868
886
|
);
|
|
869
887
|
const completeCostFields = costValues.map((values) => values.every((value) => value !== undefined));
|