pi-provider-litellm 3.0.1 → 3.2.0

package/README.md

@@ -154,6 +154,8 @@ Treat the configured LiteLLM proxy as trusted: Skills can add instructions to th
 /model
 ```
 
+To refresh catalogs on demand, run `/litellm-refresh` for every configured LiteLLM provider, or `/litellm-refresh <provider>` for one alias. It works with `PI_OFFLINE` set, which stops Pi's own `/model` refresh from contacting the network. It contacts only the configured proxies: `LITELLM_OFFLINE=1` and `LITELLM_DISCOVERY_TIMEOUT_MS=0` still disable it, and models.dev enrichment stays off under `PI_OFFLINE`. Each provider reports its model count, the failure, or missing credentials. With LiteLLM MCP enabled, the refresh also re-checks that provider's MCP tools.
+
 Every request routed through a configured LiteLLM provider includes Pi's canonical session ID in the
 `x-litellm-session-id` header. This groups Chat Completions, Responses, and native Messages requests in LiteLLM without
 adding transport-specific fields to request bodies.
@@ -173,12 +175,17 @@ Native Messages authenticates with `x-api-key`; every transport carries the `x-l
 | `LITELLM_GCLOUD_TOKEN_AUTH` | unset | If set to a non-empty value other than `0`, use Google Application Default Credentials as the LiteLLM bearer token source. This takes precedence over `LITELLM_API_KEY_HELPER` and `LITELLM_API_KEY` when no stored `/login litellm` credential exists. |
 | `GOOGLE_APPLICATION_CREDENTIALS` | Google default ADC path | Optional path to an ADC JSON file used by `LITELLM_GCLOUD_TOKEN_AUTH`. If unset, the extension checks the default gcloud ADC locations. |
 | `LITELLM_OFFLINE` | unset | If `1`, disable all model and MCP discovery, including post-login discovery; use cached models only when their stored canonical proxy root exactly matches the active credential root, including any path prefix. URL-standard host casing and default ports are canonicalized, but paths remain case-sensitive. |
+| `PI_OFFLINE` | unset | Pi documents it as disabling automatic network activity, including model catalog refreshes. Through Pi 0.87.1, any set value (including `0` or an empty string) disables startup model discovery and `/model` network refreshes. Run `/litellm-refresh`, or unset it, to discover; cached models can still be used offline. |
 | `LITELLM_DISCOVERY_TIMEOUT_MS` | `5000` | Background and explicit discovery fetch timeout in ms; `0` disables automatic discovery |
 | `LITELLM_CLI_JWT_EXPIRATION_HOURS` | `24` | CLI SSO token lifetime fallback for older proxies whose poll response omits `expires_in`; mirror a non-default proxy setting locally |
-| `LITELLM_VERBOSE_DISCOVERY` | unset | If `1`, enable progress messages during model and MCP discovery (login, refresh, startup), including MCP prepared/registered/dropped counts. Progress messages are off by default; MCP safety diagnostics (see below) are always reported regardless of this setting |
+| `LITELLM_VERBOSE_DISCOVERY` | unset | If `1`, enable progress messages during model and MCP discovery (login, refresh, startup), including startup skip reasons, defaulted `/model/info` context windows, and MCP prepared/registered/dropped counts. Progress messages are off by default; MCP safety diagnostics (see below) are always reported regardless of this setting |
+| `LITELLM_DEFAULT_CONTEXT_WINDOW` | `128000` | Context window assumed when neither LiteLLM's `/model/info` nor the catalog reports `max_input_tokens`. Set a positive integer for proxies whose custom aliases carry no metadata; an unset or unusable value keeps 128K |
+| `LITELLM_MODELS_DEV` | unset | If `1`, enrich discovered metadata (limits, prices, effort lists) from models.dev, cached for 28 days in `litellm-models-dev.json`. Off by default because LiteLLM's `/model/info` is authoritative; use it when LiteLLM's model map lacks a model's metadata |
 
 Only use a trusted `LITELLM_API_KEY_HELPER` or `!command`. Prefer an absolute executable path, keep secrets out of command arguments, and print only the token to stdout without logging it to stderr.
 
+With `LITELLM_VERBOSE_DISCOVERY=1`, `/model/info` discovery reports published routes whose context window uses the fallback assumption (including wildcard expansions) on one line with the selected value, a count, and a sample of escaped route names. Each route is reported once per process. Explicit or authoritative catalog limits do not trigger it, even when numerically equal to the default. Set `model_info.max_input_tokens` on every deployment in the route to provide real limits; this diagnostic does not change discovery limits or routing.
+
 `LITELLM_DISCOVERY_TIMEOUT_MS=0` disables automatic and explicit refresh model discovery. It does not replace the base URL or API key settings required to send requests when you are not using `/login litellm`.
 
 ### Google ADC token auth
@@ -200,11 +207,11 @@ If your LiteLLM proxy exposes MCP REST endpoints, this extension discovers tools
 - `GET /mcp-rest/tools/list`
 - `POST /mcp-rest/tools/call`
 
-An MCP access denial pauses discovery until a successful `/login litellm` (SSO or API key). The pause survives restarts in `~/.pi/agent/litellm-mcp-pauses/`, scoped to the proxy, login, and headers with salted credential fingerprints. A new API key or header configuration has its own scope. Each explicit login creates a fresh opaque `litellmMcpSession` identifier in Pi's stored credential, even when the key is unchanged; automatic OAuth refresh preserves that identifier and its pause. Older Pi processes cannot pause the new login. Pause files contain no credentials or proxy error text. Model discovery and chat continue normally. Grant MCP access on the proxy before signing in again. An explicit `litellm.mcp.enabled: false` setting remains disabled after login.
+An MCP access denial pauses discovery until a successful `/login litellm` (SSO or API key); an alias provider, which has no login, stays paused until its credentials, base URL, or headers change. The pause survives restarts in `~/.pi/agent/litellm-mcp-pauses/`, scoped to the proxy, login, and headers with salted credential fingerprints. A new API key or header configuration has its own scope. Each explicit login creates a fresh opaque `litellmMcpSession` identifier in Pi's stored credential, even when the key is unchanged; automatic OAuth refresh preserves that identifier and its pause. Older Pi processes cannot pause the new login. Pause files contain no credentials or proxy error text. Model discovery and chat continue normally. Grant MCP access on the proxy before signing in again. An explicit `litellm.mcp.enabled: false` setting remains disabled after login.
 
-Each discovered tool is registered as a native Pi tool named `mcp_<server>_<tool>_<hash>`, at most 64 characters, using only `[a-z0-9_]`. The trailing 10-character hash is derived from the tool's identity (`server_id`, `server_name`, `name`) and is always present, so a name depends only on that tool and never on which other tools happen to be in the same catalog — adding or removing a sibling never renames a survivor. When the readable prefix would overflow 64 characters it is truncated from the right, so a long server name can leave little or none of the tool name visible; the hash is what distinguishes such tools. Tools from different servers therefore never overwrite each other, and exact duplicate identities are registered once.
+Each discovered tool is registered as a native Pi tool named `mcp_<server>_<tool>_<hash>`, at most 64 characters, using only `[a-z0-9_]`. The trailing 10-character hash is derived from the tool's identity (`server_id`, `server_name`, `name`) and is always present, so a name depends only on that tool and never on which other tools happen to be in the same catalog — adding or removing a sibling never renames a survivor. When the readable prefix would overflow 64 characters it is truncated from the right, so a long server name can leave little or none of the tool name visible; the hash is what distinguishes such tools. Tools from different servers therefore never overwrite each other, and exact duplicate identities are registered once. Each alias provider discovers its own catalog: its tools are named `mcp_<alias>_<server>_<tool>_<hash>`, with the alias name also folded into the hash, and are called with that alias's credentials, so two providers exposing the same server never replace each other's tools. The default `litellm` provider keeps unprefixed names, and diagnostics for an alias read `LiteLLM MCP ("<alias>"):`.
 
-MCP discovery runs after Pi refreshes LiteLLM models or after `/login litellm`; extension activation never waits for it. Discovery accepts at most a 5 MiB response and registers at most 512 tools. The response must be a JSON array of tools or an object with a `tools` array; an `unexpected_error` or unknown non-null `error` in the object envelope, an object without `tools`, and every other shape are reported as discovery failures rather than as empty catalogs. Failure reports use extension-owned text and never include LiteLLM's proxy-supplied tag or `message`. A `partial_failure` envelope keeps and registers the returned tools while reporting one bounded generated diagnostic. Each tool is isolated during normalization, so one bad tool never hides valid siblings. A tool is accepted only when its schema has a root of exactly `"type": "object"` (a bare `properties` object with no `type`, or a type union such as `["object","null"]`, is refused), plain-object `properties`, at most 16 levels of JSON nesting (about eight nested schema objects when each level uses `properties`), and a serialized size of at most 64 KiB. Descriptions are limited to 4 KiB, and tool labels and result metadata to 256 bytes, each with a truncation marker.
+MCP discovery runs once for every configured provider before the first agent turn (so `-p` and other non-interactive modes get MCP tools), after Pi refreshes LiteLLM models, and after `/login litellm`; extension activation never waits for it. Discovery accepts at most a 5 MiB response and registers at most 512 tools. The response must be a JSON array of tools or an object with a `tools` array; an `unexpected_error` or unknown non-null `error` in the object envelope, an object without `tools`, and every other shape are reported as discovery failures rather than as empty catalogs. Failure reports use extension-owned text and never include LiteLLM's proxy-supplied tag or `message`. A `partial_failure` envelope keeps and registers the returned tools while reporting one bounded generated diagnostic. Each tool is isolated during normalization, so one bad tool never hides valid siblings. A tool is accepted only when its schema has a root of exactly `"type": "object"` (a bare `properties` object with no `type`, or a type union such as `["object","null"]`, is refused), plain-object `properties`, at most 16 levels of JSON nesting (about eight nested schema objects when each level uses `properties`), and a serialized size of at most 64 KiB. Descriptions are limited to 4 KiB, and tool labels and result metadata to 256 bytes, each with a truncation marker.
 
 ### Which schema a tool ends up with
 
@@ -256,6 +263,8 @@ If your LiteLLM proxy exposes `/claude-code/marketplace.json`, enabled skills ar
 - `litellm_skill_create`
 - `litellm_skill_delete`
 
+`litellm_skill_create` needs Skill Hub `sourceJson` metadata and a proxy that exposes `/claude-code/plugins`. It does not create skills from code: LiteLLM's `POST /v1/skills` is Anthropic's multipart Skills API.
+
 ## Mocked LiteLLM smoke workflow
 
 The `LiteLLM Smoke` GitHub Actions workflow starts VidaiMock and a real LiteLLM proxy on the runner. LiteLLM exposes route-distinct Chat, Responses, native Messages, and mixed-deployment models whose upstreams are served by VidaiMock. The smoke runner discovers those models through LiteLLM, asserts each model's expected API, exercises `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`, verifies the expected `x-litellm-response-cost` behavior, and proves endpoint coverage from captured LiteLLM request logs rather than response text.
@@ -293,9 +302,11 @@ Before tagging a release, keep `package.json` and `package-lock.json` versions i
 
 Dynamic catalogs are persisted by Pi in `~/.pi/agent/models-store.json`. Credentials remain in `~/.pi/agent/auth.json`. Legacy `litellm-models.json` model caches are ignored and never deleted. `litellm-models-dev.json` is the models.dev cache and is refreshed in place.
 
-For every genuine `/model/info` row, including deployment details fetched through `/health`, the extension requests `https://models.dev/api.json` for enrichment and caches the result for 28 days in `~/.pi/agent/litellm-models-dev.json`. Health-only entries without deployment details use the bounded Pi catalog lookup without models.dev enrichment. `PI_OFFLINE` suppresses activation-time discovery and the models.dev request. `LITELLM_OFFLINE=1` also disables LiteLLM discovery; direct discovery callers use only an existing models.dev cache and do not refresh it.
+LiteLLM's `/model/info` is the metadata authority, so models.dev enrichment is an opt-in escape hatch. With `LITELLM_MODELS_DEV=1`, for every genuine `/model/info` row, including deployment details fetched through `/health`, the extension requests `https://models.dev/api.json` for enrichment and caches the result for 28 days in `~/.pi/agent/litellm-models-dev.json`. Without it, discovery neither requests models.dev nor reads that cache; Pi's own catalog still applies. Health-only entries without deployment details use the bounded Pi catalog lookup without models.dev enrichment. `PI_OFFLINE` suppresses activation-time discovery and the models.dev request. `LITELLM_OFFLINE=1` also disables LiteLLM discovery; direct discovery callers use only an existing models.dev cache and do not refresh it.
 
-Opening `/model` refreshes configured provider catalogs in the background using Pi's native model lifecycle.
+Opening `/model` refreshes configured provider catalogs in the background using Pi's native model lifecycle when network discovery is enabled.
+
+Pi documents `PI_OFFLINE` and `--offline` as disabling automatic network activity, including model catalog refreshes. Pi 0.87.1 adopted that wording to resolve [earendil-works/pi#8684](https://github.com/earendil-works/pi/issues/8684), without changing the behavior. Pi still checks only whether the variable is set, so `PI_OFFLINE=0` and an empty value also disable model refreshes. With `LITELLM_VERBOSE_DISCOVERY=1`, disabled startup discovery reports its reason, for example `startup discovery skipped (PI_OFFLINE)`. To discover models while keeping `PI_OFFLINE`, run `/litellm-refresh`. To allow all discovery for one invocation, use `env -u PI_OFFLINE pi` and do not pass `--offline`. If you only want to disable Pi's version check, use `PI_SKIP_VERSION_CHECK=1` instead; it does not apply offline mode's other network restrictions. See [#155](https://github.com/balcsida/pi-provider-litellm/issues/155).
 
 ### Model host enforcement
 
@@ -318,7 +329,7 @@ Each distinct availability diagnostic is written once per session on stderr. `LI
 
 ### Protocols and prompt caching
 
-Discovery selects native Messages for proven Claude groups under the [Model transport](#model-transport) rules, Responses for OpenAI-family backends, and Chat Completions for other OpenAI-compatible backends. An Azure deployment with an explicit `api_version` older than `2025-03-01-preview` stays on Chat Completions; an unset or newer version uses Responses. Azure is detected from the adapter field, an `azure/` or `azure_ai/` model prefix, or the reported provider; other adapters follow the backend family because LiteLLM bridges `/v1/responses` to Chat Completions when the provider has no native Responses config (`litellm/responses/main.py`, `_bridges_to_chat_completions`). A route whose model prefix and `custom_llm_provider` name different providers is treated as unidentified and stays on Chat Completions, so it also does not get the local 128-tool preflight. When `/model/info` supplies `supported_endpoints`, that list takes precedence, and `mode: "responses"` remains an explicit Responses signal. An evidence-free fallback entry—a bare `/v1/models` id or a `/health` entry with no detail row—takes its protocol from the Pi catalog entry for its id; both evidence-free fallback paths also inherit presentation metadata from the bounded catalog lookup. An unknown id stays on Chat Completions. The route name alone authorizes nothing. A concrete id expanded from wildcard `/model/info` routes inherits the deployment evidence of the wildcard route LiteLLM would select for it, with the requested id substituted into the row's `model` the way LiteLLM serves it; the id is omitted when that selected route is not a published chat-style route. Chat Completions and Responses use `<root>/v1`; native Messages uses the proxy root directly. All three protocols pass through the provider's host guard. When an evidence-free cached `(no metadata)` entry can be enriched from the Pi catalog, it is restored on Responses only for a catalog Responses model and otherwise on Chat Completions, never Anthropic Messages. A model supplied only through `models.json` whose `api` this provider does not implement takes Pi's global API fallback rather than this provider's host guard; the credential-root pin (see [Model host enforcement](#model-host-enforcement)) still keeps the credential on the active proxy.
+Discovery selects native Messages for proven Claude groups under the [Model transport](#model-transport) rules, Responses for OpenAI-family backends, and Chat Completions for other OpenAI-compatible backends. Azure and Azure AI deployments stay on Chat Completions unless an explicit Responses mode or a `supported_endpoints` list containing `/v1/responses` authorizes Responses; `api_version` alone does not prove support. Azure is detected from the adapter field, an `azure/` or `azure_ai/` model prefix, or the reported provider; other adapters follow the backend family because LiteLLM bridges `/v1/responses` to Chat Completions when the provider has no native Responses config (`litellm/responses/main.py`, `_bridges_to_chat_completions`). A route whose model prefix and `custom_llm_provider` name different providers is treated as unidentified and stays on Chat Completions, so it also does not get the local 128-tool preflight. When `/model/info` supplies `supported_endpoints`, that list takes precedence, and `mode: "responses"` remains an explicit Responses signal. An evidence-free fallback entry—a bare `/v1/models` id or a `/health` entry with no detail row—takes its protocol from the Pi catalog entry for its id; both evidence-free fallback paths also inherit presentation metadata from the bounded catalog lookup. An unknown id stays on Chat Completions. The route name alone authorizes nothing. A concrete id expanded from wildcard `/model/info` routes inherits the deployment evidence of the wildcard route LiteLLM would select for it, with the requested id substituted into the row's `model` the way LiteLLM serves it; the id is omitted when that selected route is not a published chat-style route. Chat Completions and Responses use `<root>/v1`; native Messages uses the proxy root directly. All three protocols pass through the provider's host guard. When an evidence-free cached `(no metadata)` entry can be enriched from the Pi catalog, it is restored on Responses only for a catalog Responses model and otherwise on Chat Completions, never Anthropic Messages. A model supplied only through `models.json` whose `api` this provider does not implement takes Pi's global API fallback rather than this provider's host guard; the credential-root pin (see [Model host enforcement](#model-host-enforcement)) still keeps the credential on the active proxy.
 
 `cacheControlFormat: "anthropic"` applies only to Chat Completions, where Pi adds Anthropic `cache_control` markers. The Responses transport has a different compatibility type and uses native `prompt_cache_key` (plus supported retention fields), so Responses models must not receive `cacheControlFormat`. Freshly discovered OpenAI-family models forced onto Chat Completions are rejected locally when a request contains more than 128 tools; route them through Responses or reduce enabled extensions. Moonshot request-side reasoning suppression follows LiteLLM's routing provider, not `model_info.base_model`; response-side `<think>` normalization follows discovered Kimi-family evidence. Opaque aliases retain display normalization without sending Moonshot-only parameters to Azure- or Bedrock-hosted Kimi models. Strict tool-message repair requires every deployment to identify Kimi; Gemini effort normalization likewise requires consistent backend-family evidence.
 
@@ -329,8 +340,8 @@ Discovery policy is versioned in Pi's persisted model catalog. After upgrading,
 LiteLLM may load-balance one public `model_name` across deployments with different backends or model versions. The extension reduces `/model/info` rows conservatively before publishing one Pi model. A route group that mixes a chat-style deployment with an explicitly incompatible mode such as embedding is withheld entirely and reported with a bounded diagnostic.
 
 - Responses is selected only when every deployment supports it under the protocol rules above; a deployment that requires Chat keeps the group on Chat.
-- Vision and reasoning are advertised only when every routable deployment resolves them as supported. An explicit `supports_reasoning: false` suppresses all thinking controls, even when catalog or router effort metadata exists. Catalog thinking controls are intersected per level: a level is exposed only when every deployment maps it to the same value, while disagreement or absence becomes an explicit denial. Reasoning selectors additionally require an accepted wire carrier on every deployment. Public effort lists are intersected, explicit LiteLLM denials win, and `xhigh`/`max` need unanimous explicit support. Null or absent standard-effort flags have no opinion.
-- Context and output limits use the minimum resolved value across deployments.
+- Vision and reasoning are advertised only when every routable deployment resolves them as supported. An explicit `supports_reasoning: false` suppresses all thinking controls, even when catalog or router effort metadata exists. Catalog thinking controls are intersected per level: a level is exposed only when every deployment maps it to the same value, while disagreement or absence becomes an explicit denial. Catalog maps are not complete effort lists, so absence has one exception: an omitted `minimal`, `low`, `medium` or `high` counts as Pi's default spelling of that level. An omitted `off`, `xhigh` or `max` still differs from an explicit mapping. Reasoning selectors additionally require an accepted wire carrier on every deployment. Public effort lists are intersected, explicit LiteLLM denials win, and `xhigh`/`max` need unanimous explicit support. Null or absent standard-effort flags have no opinion. A deployment that declares `model_info.reasoning_effort_levels` answers every level from that list, ahead of its per-level flags, as LiteLLM itself reads it.
+- Context and output limits use the minimum resolved value across deployments. Pi catalogue lookup prefers the deployment's provider-specific entry before falling back to the generic vendor entry; Azure and Azure AI therefore retain Azure-specific limits rather than being capped by a different OpenAI catalogue entry. When models.dev supplies a sparse vendor record, missing fields still use the provider-specific Pi catalogue.
 - Each displayed base-price field uses the maximum only when every deployment resolves that field; unresolved fields remain zero and the model name is suffixed with ` (incomplete metadata)`.
 - When complete base pricing comes from the catalog under unanimous catalog authority, tiered pricing is the conservative worst-case envelope. Its usable finite non-negative thresholds are the sorted union of every deployment's thresholds, and each price field at each interval is the maximum applicable rate across all deployments, floored by each deployment's base rate. Differing tier breakpoints are therefore preserved rather than treated as incompatible. An explicit `/model/info` base-price field replaces catalog pricing for that field at every threshold; catalog tiers remain for unaffected fields and are omitted only when every base-price field is explicit.
 - Reduced route groups omit tier ladders when complete base-pricing or catalog authority is unavailable. Wildcard expansion retains every known tier from matching groups even when another match has incomplete metadata: suppressing a complete sibling's higher tier could understate the known worst-case rate. The ` (incomplete metadata)` marker remains because the resulting envelope is only as complete as the available ladders.
@@ -345,7 +356,9 @@ The ` (incomplete metadata)` suffix marks reduced `/model/info` groups, incomple
 
 ### Reasoning controls
 
-Reasoning levels require deployment evidence for the parameter Pi sends: `reasoning_effort` for effort levels or `thinking` for a native thinking switch. Public route names never authorize these controls. Evidence-free fallback models retain catalog presentation metadata but expose no speculative selector.
+Reasoning levels require deployment evidence for the parameter Pi sends: `reasoning_effort` for effort levels or `thinking` for a native thinking switch. Public route names never authorize these controls.
+
+LiteLLM omits `supported_openai_params` for a deployment its model map does not describe. Such a deployment keeps the `reasoning_effort` carrier when its `model_info` explicitly sets `supports_reasoning: true`, which is the operator's opt-in. A `supported_openai_params` list that omits `reasoning_effort` still denies effort levels unless `litellm_params.allowed_openai_params` adds it. Kimi and DeepSeek generations are excluded because their contracts name their own carriers. Evidence-free fallback models retain catalog presentation metadata but expose no speculative selector.
 
 Kimi K2.5/K2.6 use an on/off thinking switch; K2.7 Code/Highspeed is always thinking. Kimi K3 and DeepSeek V4 use public effort evidence with LiteLLM overrides. Responses levels are translated to valid `reasoning.effort` values; Chat-only compatibility fields never reach Responses models. Legacy cached level maps remain usable while discovery refreshes, and stored Moonshot compatibility can restore safe display normalization.
 
@@ -356,6 +369,7 @@ The development probe runs against minimized snapshots with `npm run probe:proxy
 | Symptom | Likely cause |
 |---|---|
 | "no credentials" warning at startup | Env vars not set and no OAuth credential — run `/login litellm` |
+| No models, or no new routes, while `PI_OFFLINE` is set | Pi skips every model network refresh, including the one `/model` starts. Run `/litellm-refresh`, or unset `PI_OFFLINE` (`PI_OFFLINE=0` is not enough) |
 | `No models available` at startup, gone after a restart | A Pi startup race, not discovery — see [`No models available` at startup](#no-models-available-at-startup) |
 | "discovered no models" | Proxy returned an empty list — check pi's startup log and verify `/model/info`, `/v1/models`, or `/health` responds |
 | `/model/info` returning 401/403/404 | Expected behavior with virtual keys — extension falls back to `/v1/models` |
@@ -364,6 +378,7 @@ The development probe runs against minimized snapshots with `npm run probe:proxy
 | A configured or restored model fails to complete but never leaks the credential | Its `api` is absent from the current provider catalog, so Pi used global API fallback with the credential root pinned. Open `/model` against the active proxy to repopulate the protocol — see [Model host enforcement](#model-host-enforcement) |
 | `LiteLLM discovery: ... route group(s) have missing or conflicting deployment provider evidence` | One or more deployments lack resolvable backend authority or resolve to different providers or concrete catalog models. Add consistent `litellm_params.model`, `model_info.base_model`, or adapter metadata; catalog-derived limits, pricing, and reasoning metadata are withheld meanwhile. This may be intentional for cross-host routes such as Vertex/Bedrock Claude. |
 | `LiteLLM discovery: ... route group(s) mix chat-style and explicitly incompatible deployment modes` | A public route can select both a chat-style deployment and an incompatible deployment such as an embedding model. Give the embedding deployment its own `model_name`. |
+| `LiteLLM (...): a fallback served "<route>"` | A successful HTTP response reports a LiteLLM router fallback (`x-litellm-attempted-fallbacks`). `/model/info` does not expose fallbacks, so protocol and model handling follow the requested route's own deployments. If the primary and fallback groups support Chat Completions, set `model_info.supported_endpoints: ["/v1/chat/completions"]` on the primary to avoid Responses-to-Chat bridging. This warning does not change routing or fix upstream bridging defects. Warned once per provider and route until the extension reloads; shown in the UI, or stderr without a UI. |
 | A model is marked ` (incomplete metadata)` | `/model/info` or `/health` did not provide enough authoritative metadata. Explicit fields remain usable, but unknown cost fields are shown as zero and route-name cache enrichment stays disabled. |
 | `401 Token expired` | Set `LITELLM_API_KEY_HELPER`. |
 | No models with gcloud auth | Verify `gcloud auth application-default login` has been run or set `GOOGLE_APPLICATION_CREDENTIALS` to an `authorized_user` ADC file |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-provider-litellm",
-  "version": "3.0.1",
+  "version": "3.2.0",
   "description": "LiteLLM proxy provider extension for Pi",
   "type": "module",
   "license": "MIT",

package/src/backend-identity.ts

@@ -29,6 +29,10 @@ const PROVIDER_FAMILIES: Readonly<Record<string, BackendFamily>> = {
 // Prefixes whose keyword match is trustworthy across the whole "prefix/modelId" string because
 // the prefix itself names a vendor, not just some unrelated string that happens to contain one.
 const KNOWN_VENDOR_PREFIXES = new Set([...Object.keys(PROVIDER_FAMILIES), "openai"]);
+// First segments that are known to be model path, never a provider: Fireworks' account-scoped
+// ids ("accounts/fireworks/models/x"). An allowlist fails closed: any other differing prefix,
+// recognized provider or not, stays a conflict.
+const MODEL_PATH_SEGMENTS = new Set(["accounts"]);
 // `o\d` (OpenAI's o1/o3/o4-mini reasoning models) is only trustworthy at the start of the id or
 // right after a provider path segment — an interior "-o1-" is as likely to be an unrelated
 // product's own version marker (e.g. "custom-o1-clone").
@@ -62,14 +66,20 @@ export function resolveBackendIdentity(row: BackendIdentityRow): BackendIdentity
     wireString(row.model_info?.base_model) ?? wireString(row.litellm_params?.model) ?? wireString(row.model_name);
   if (!raw) return undefined;
 
-  const slash = raw.indexOf("/");
+  let slash = raw.indexOf("/");
   const prefix = slash > 0 ? raw.slice(0, slash).trim().toLowerCase() : undefined;
-  const prefixProvider = prefix && !GENERIC_ADAPTERS.has(prefix) ? prefix : undefined;
+  let prefixProvider = prefix && !GENERIC_ADAPTERS.has(prefix) ? prefix : undefined;
   // `custom_llm_provider` is how LiteLLM routes an unprefixed model. It is provider evidence
-  // in its own right, so a prefix that names a different provider is a conflict, not a tiebreak.
+  // in its own right, so a differing prefix is a conflict, not a tiebreak. The exception is a
+  // known model-path segment: LiteLLM routes by custom_llm_provider, so Fireworks'
+  // "accounts/fireworks/models/x" is not a provider named "accounts".
   const custom = wireString(row.litellm_params?.custom_llm_provider)?.toLowerCase();
   const customProvider = custom && !GENERIC_ADAPTERS.has(custom) ? custom : undefined;
-  if (prefixProvider && customProvider && prefixProvider !== customProvider) return undefined;
+  if (prefixProvider && customProvider && prefixProvider !== customProvider) {
+    if (!MODEL_PATH_SEGMENTS.has(prefixProvider)) return undefined;
+    prefixProvider = undefined;
+    slash = -1;
+  }
   const provider = prefixProvider ?? customProvider;
   const modelId = slash > 0 ? raw.slice(slash + 1) : raw;
   // Scan the full "prefix/modelId" string only when the prefix is itself a whole known vendor

package/src/discover.ts

@@ -8,9 +8,10 @@ import {
   catalogResolution,
   closeSerializerPolicy,
   conservativeCostTiers,
-  DEFAULT_CONTEXT_WINDOW,
   DEFAULT_MAX_TOKENS,
+  defaultContextWindow,
   hasMixedIncompatibleDeploymentModes,
+  isFallbackContextWindow,
   type MessagesBackendCompat,
   meetVendorCompat,
   normalizedMode,
@@ -39,6 +40,7 @@ const DEFAULT_TIMEOUT_MS = 5000;
 const HEALTH_DETAIL_CONCURRENCY = 8;
 const reportedConflictingFamilyRoutes = new Set<string>();
 const reportedWithheldRepairRoutes = new Set<string>();
+const reportedDefaultedContextRoutes = new Set<string>();
 interface HealthDeployment {
   entry: ModelInfoEntry;
   denyLevels: boolean;
@@ -148,16 +150,11 @@ function supportsResponses(entry: ModelInfoEntry): boolean {
     /^azure(?:_ai)?\//.test(configuredModel ?? "") ||
     reportedProvider === "azure" ||
     reportedProvider === "azure_ai";
-  // LiteLLM bridges /v1/responses to chat completions when the provider has no native Responses config
-  // (litellm/responses/main.py, _bridges_to_chat_completions), so generic adapters remain eligible for Responses.
-  if (!azureAdapter) return true;
-
-  const apiVersion = entry.litellm_params?.api_version;
-  if (apiVersion != null && typeof apiVersion !== "string") return false;
-  const version = apiVersion?.trim();
-  if (!version) return true;
-  const date = version.match(/^(\d{4}-\d{2}-\d{2})(?:-preview)?$/)?.[1];
-  return date !== undefined && date >= "2025-03-01";
+  // An Azure API version describes the API surface, not whether this deployment
+  // serves Responses. Require the explicit mode/endpoint evidence above instead
+  // of promoting a working Chat route to an unavailable endpoint.
+  // Generic adapters remain eligible for LiteLLM's Responses-to-Chat bridge.
+  return !azureAdapter;
 }
 
 export function modelProtocol(modelId: string, modeOrEntry?: string | null | ModelInfoEntry): ModelProtocol {
@@ -231,7 +228,9 @@ export function enrichCachedModel(input: Model<Api>): Model<Api> {
     }),
   } as Model<Api>;
   // Reduced deployment groups use a distinct marker; this sentinel remains
-  // exclusive to evidence-free fallback models that may be enriched safely.
+  // exclusive to evidence-free fallback models that may be enriched safely. A window is
+  // fallback evidence when it matches either default, so a cached entry still qualifies
+  // after an operator configures LITELLM_DEFAULT_CONTEXT_WINDOW.
   if (
     !model.name.endsWith(" (no metadata)") ||
     model.reasoning ||
@@ -243,13 +242,15 @@ export function enrichCachedModel(input: Model<Api>): Model<Api> {
     model.cost.cacheRead !== 0 ||
     model.cost.cacheWrite !== 0 ||
     model.cost.tiers !== undefined ||
-    model.contextWindow !== DEFAULT_CONTEXT_WINDOW ||
+    !isFallbackContextWindow(model.contextWindow) ||
     model.maxTokens !== DEFAULT_MAX_TOKENS
   ) {
     return model;
   }
   const catalogModel = findCatalogModel(model.id);
-  if (!catalogModel) return model;
+  // The gate above proved this window is an assumption, so it is re-derived rather than
+  // restored: the cache is seeded offline, where discovery never re-reads the setting.
+  if (!catalogModel) return { ...model, contextWindow: defaultContextWindow() };
   // Match evidence-free discovery: only an explicit Responses catalog transport
   // changes the protocol; all other catalog APIs continue through Chat.
   const api = catalogModel.api === "openai-responses" ? "openai-responses" : "openai-completions";
@@ -399,7 +400,7 @@ function adaptPublicCatalogRecord(
   catalogModelId: string,
   record: PublicCatalogRecord | undefined,
 ): CatalogResolution {
-  const piProvider = adapterCatalogProvider(record?.provider ?? provider);
+  const piProvider = adapterCatalogProvider(record?.piProvider ?? record?.provider ?? provider);
   const resolved = piProvider
     ? (resolveCatalogModel(record?.modelId ?? catalogModelId, piProvider) ??
       resolveCatalogModel(catalogModelId, piProvider))
@@ -597,6 +598,7 @@ function mapFromModelInfoGroup(
     ambiguousRoutes?: string[];
     conflictingFamilyRoutes?: string[];
     withheldRepairRoutes?: string[];
+    defaultedContextRoutes?: Set<string>;
     denyLevels?: boolean;
     allowMessages?: boolean;
   } = {},
@@ -618,6 +620,7 @@ function mapFromModelInfoGroup(
     };
   });
   if (!reduced) return undefined;
+  if (reduced.contextWindowDefaulted) options.defaultedContextRoutes?.add(reduced.id);
   if (reduced.catalogAuthorityAmbiguous) options.ambiguousRoutes?.push(reduced.id);
   if (reduced.deploymentFamilies.includes("conflicting")) options.conflictingFamilyRoutes?.push(reduced.id);
   const protocols = entries.map((entry) => modelProtocol(reduced.id, entry));
@@ -720,7 +723,7 @@ function mapFromModelsList(entry: ModelsListEntry): DiscoveredModel | undefined
     }),
     input: catalogModel?.input ?? ["text"],
     cost: catalogModel?.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
-    contextWindow: catalogModel?.contextWindow ?? DEFAULT_CONTEXT_WINDOW,
+    contextWindow: catalogModel?.contextWindow ?? defaultContextWindow(),
     maxTokens: catalogModel?.maxTokens ?? DEFAULT_MAX_TOKENS,
     api,
     litellmDiscoveryVersion: LITELLM_DISCOVERY_VERSION,
@@ -1034,6 +1037,7 @@ export async function discoverModels(
     const incompatibleModeRoutes: string[] = [];
     const conflictingFamilyRoutes: string[] = [];
     const withheldRepairRoutes: string[] = [];
+    const defaultedContextRoutes = new Set<string>();
     const reducedGroups = [...groups.entries()].map(([route, group]) => {
       if (hasMixedIncompatibleDeploymentModes(group)) incompatibleModeRoutes.push(route);
       return {
@@ -1042,6 +1046,7 @@ export async function discoverModels(
           ambiguousRoutes,
           conflictingFamilyRoutes,
           withheldRepairRoutes,
+          defaultedContextRoutes,
         }),
         deploymentFamilies: group.map(deploymentFamily),
       };
@@ -1097,7 +1102,27 @@ export async function discoverModels(
       }
     }
     reportWithheldToolRepair(withheldRepairRoutes);
-    return { source: "model_info", models: deduplicateModels(models) };
+    models = deduplicateModels(models);
+    if (process.env.LITELLM_VERBOSE_DISCOVERY === "1") {
+      const defaultedWildcards = wildcardRoutes.filter(({ route }) => defaultedContextRoutes.has(route));
+      // Exact routes override wildcard templates; a tighter measured parent can also win.
+      const defaulted = models.filter((model) =>
+        groups.has(model.id)
+          ? defaultedContextRoutes.has(model.id)
+          : defaultedWildcards.some(
+              ({ route, model: parent }) =>
+                parent?.contextWindow === model.contextWindow && wildcardMatches(route, model.id),
+            ),
+      );
+      reportBoundedRoutes(
+        reportedDefaultedContextRoutes,
+        defaulted.map((model) => JSON.stringify(model.id)),
+        (count) =>
+          `LiteLLM discovery: ${count} route(s) default contextWindow to ${defaultContextWindow()}; ` +
+          "set model_info.max_input_tokens on every deployment in each route",
+      );
+    }
+    return { source: "model_info", models };
   }
   if (![401, 403, 404].includes(infoResult.status)) {
     throw new Error(`/model/info returned ${infoResult.status}`);