llmshim 0.7.2__tar.gz → 0.8.0__tar.gz

{llmshim-0.7.2 → llmshim-0.8.0}/Cargo.lock

@@ -1180,7 +1180,7 @@ checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
 
 [[package]]
 name = "llmshim"
-version = "0.7.2"
+version = "0.8.0"
 dependencies = [
  "async-stream",
  "async-trait",

{llmshim-0.7.2 → llmshim-0.8.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llmshim"
-version = "0.7.2"
+version = "0.8.0"
 edition = "2021"
 description = "Blazing fast LLM API translation layer in pure Rust"
 license = "MIT OR Apache-2.0"

{llmshim-0.7.2 → llmshim-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmshim
-Version: 0.7.2
+Version: 0.8.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{llmshim-0.7.2 → llmshim-0.8.0}/README.md

@@ -100,6 +100,18 @@ Then address the served model as `sglang/<served-model>` or `vllm/<served-model>
 (e.g. `sglang/Qwen/Qwen3.6-35B-A3B-FP8`). Server-specific knobs go under
 `x-vllm` / `x-sglang`.
 
+A server that also serves `/v1/responses` (SGLang does) can be spoken to on
+that wire with `SGLANG_WIRE=responses` (or `VLLM_WIRE=responses`). Reasoning
+then comes back as an item with its own id rather than bare `reasoning_content`,
+and is replayed as that item. Replay is gated on a known model family on every
+wire, and the public catalog does not know a served model: declare it once in
+`.llmshim/models.toml` —
+
+```toml
+[models."sglang/<served-model>"]
+family = "qwen"
+```
+
 Or persist them to the config file (used by all three surfaces):
 
 ```bash

{llmshim-0.7.2 → llmshim-0.8.0}/docs/src/reference/providers.md

@@ -11,8 +11,8 @@ different native API and translates only the fields that API understands.
 | Google Gemini | `generateContent` / `streamGenerateContent` | `gemini*` | `x-gemini` |
 | xAI | Responses API | `grok*` | none |
 | OpenRouter | Chat Completions (aggregator) | none — address as `openrouter/<vendor>/<model>` | `x-openrouter` |
-| vLLM | Chat Completions (self-hosted, `VLLM_BASE_URL`) | none — address as `vllm/<served-model>` | `x-vllm` |
-| SGLang | Chat Completions (self-hosted, `SGLANG_BASE_URL`) | none — address as `sglang/<served-model>` | `x-sglang` |
+| vLLM | Chat Completions (self-hosted, `VLLM_BASE_URL`); Responses API with `VLLM_WIRE=responses` | none — address as `vllm/<served-model>` | `x-vllm` |
+| SGLang | Chat Completions (self-hosted, `SGLANG_BASE_URL`); Responses API with `SGLANG_WIRE=responses` | none — address as `sglang/<served-model>` | `x-sglang` |
 
 An explicit address such as `anthropic/claude-sonnet-5` avoids inference.
 The named provider must be registered in the Router—that normally means its

{llmshim-0.7.2 → llmshim-0.8.0}/src/breaker.rs

@@ -304,6 +304,7 @@ mod tests {
         ShimError::ProviderError {
             status: 503,
             body: "down".into(),
+            retry_after: None,
         }
     }
 
@@ -327,17 +328,20 @@ mod tests {
         assert!(!counts_toward_health(&ShimError::ProviderError {
             status: 429,
             body: "slow down".into(),
+            retry_after: None,
         }));
         for status in [500, 502, 503, 504, 529] {
             assert!(counts_toward_health(&ShimError::ProviderError {
                 status,
                 body: String::new(),
+                retry_after: None,
             }));
         }
         for status in [400, 401, 403, 404, 422] {
             assert!(!counts_toward_health(&ShimError::ProviderError {
                 status,
                 body: String::new(),
+                retry_after: None,
             }));
         }
         assert!(!counts_toward_health(&ShimError::MissingModel));

{llmshim-0.7.2 → llmshim-0.8.0}/src/cache.rs

@@ -91,6 +91,7 @@ fn invalid(message: &str) -> ShimError {
     ShimError::ProviderError {
         status: 400,
         body: format!("invalid x-cache: {message}"),
+        retry_after: None,
     }
 }
 fn policy(request: &Value) -> Result<Option<CachePolicy>> {

{llmshim-0.7.2 → llmshim-0.8.0}/src/client.rs

@@ -169,10 +169,15 @@ impl ShimClient {
                         tokio::time::sleep(wait).await;
                         continue;
                     }
+                    // Read before the body is consumed: the header is the
+                    // server's own wait, and a caller with its own backoff
+                    // above this client gets to honour it too.
+                    let retry_after = parse_retry_after(resp.headers());
                     let body = resp.text().await.unwrap_or_default();
                     return Err(ShimError::ProviderError {
                         status: status_code,
                         body,
+                        retry_after,
                     });
                 }
                 // Transport errors carry no headers: always jittered backoff.

{llmshim-0.7.2 → llmshim-0.8.0}/src/error.rs

@@ -19,8 +19,16 @@ pub enum ShimError {
     #[error("JSON error: {0}")]
     Json(#[from] serde_json::Error),
 
+    /// A non-success response from the provider. `retry_after` is the
+    /// server's own `Retry-After` (delay-seconds or HTTP-date, parsed at
+    /// receipt), so a caller with its own backoff can wait what was asked
+    /// instead of guessing; `None` when the header was absent or unparseable.
     #[error("provider error ({status}): {body}")]
-    ProviderError { status: u16, body: String },
+    ProviderError {
+        status: u16,
+        body: String,
+        retry_after: Option<std::time::Duration>,
+    },
 
     #[error("stream error: {0}")]
     Stream(String),

{llmshim-0.7.2 → llmshim-0.8.0}/src/gateway/http.rs

@@ -55,7 +55,9 @@ pub struct RealDispatch {
 impl RealDispatch {
     fn map_err(err: ShimError) -> DispatchError {
         match err {
-            ShimError::ProviderError { status: 429, body } => DispatchError {
+            ShimError::ProviderError {
+                status: 429, body, ..
+            } => DispatchError {
                 message: body,
                 retry_after: Some(penalty_duration()),
             },
@@ -373,6 +375,7 @@ impl GatewayState {
                          key to run it uncharged.\",\"type\":\"invalid_request_error\",\
                          \"param\":\"model\",\"code\":\"unpriceable_under_budget\"}}}}"
                     ),
+                    retry_after: None,
                 }))
             }
         }
@@ -425,10 +428,12 @@ fn gateway_err_to_api(state: &GatewayState, err: GatewayError) -> ApiError {
         GatewayError::Shutdown => ApiError::from(ShimError::ProviderError {
             status: 503,
             body: "gateway shutting down".to_string(),
+            retry_after: None,
         }),
         GatewayError::Upstream(message) => ApiError::from(ShimError::ProviderError {
             status: 502,
             body: message,
+            retry_after: None,
         }),
     }
 }

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/anthropic.rs

@@ -382,6 +382,7 @@ fn transform_response_to_openai(model: &str, resp: &Value) -> Result<Value> {
             return Err(ShimError::ProviderError {
                 status: 502,
                 body: "Anthropic response has no supported terminal stop reason".into(),
+                retry_after: None,
             })
         }
     };
@@ -643,7 +644,7 @@ impl Provider for Anthropic {
             if let Some(thinking) = body_obj.get("thinking") {
                 if thinking["type"] != "adaptive" {
                     return Err(ShimError::ProviderError { status: 400, body:
-                        "Claude Fable requires adaptive thinking; use reasoning_effort to control depth".into() });
+                        "Claude Fable requires adaptive thinking; use reasoning_effort to control depth".into(), retry_after: None });
                 }
             }
             if body_obj
@@ -653,7 +654,7 @@ impl Provider for Anthropic {
                 .is_some_and(|message| message["role"] == "assistant")
             {
                 return Err(ShimError::ProviderError { status: 400, body:
-                    "Claude Fable does not support assistant prefill; end the request with a user turn".into() });
+                    "Claude Fable does not support assistant prefill; end the request with a user turn".into(), retry_after: None });
             }
         }
         if model == "claude-fable-5-1"
@@ -663,7 +664,7 @@ impl Provider for Anthropic {
                 .is_some_and(|kind| matches!(kind, "any" | "tool"))
         {
             return Err(ShimError::ProviderError { status: 400, body:
-                "Claude Fable 5.1 supports only auto or none tool choice; request the desired tool in the prompt".into() });
+                "Claude Fable 5.1 supports only auto or none tool choice; request the desired tool in the prompt".into(), retry_after: None });
         }
 
         // Fast mode support: extract "speed" from the request and apply
@@ -768,6 +769,7 @@ impl Anthropic {
             return Err(ShimError::ProviderError {
                 status: 400,
                 body: msg.to_string(),
+                retry_after: None,
             });
         }
 

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/anthropic_reasoning.rs

@@ -107,5 +107,6 @@ fn invalid() -> ShimError {
     ShimError::ProviderError {
         status: 400,
         body: "max_tokens cannot accommodate the configured reasoning budget".into(),
+        retry_after: None,
     }
 }

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/chatgpt/auth.rs

@@ -23,6 +23,7 @@ pub(super) fn auth_error(status: u16, message: &str) -> ShimError {
     ShimError::ProviderError {
         status,
         body: format!("ChatGPT: {message}"),
+        retry_after: None,
     }
 }
 

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/gemini.rs

@@ -288,6 +288,7 @@ fn transform_response_to_openai(model: &str, resp: &Value) -> Result<Value> {
         .ok_or_else(|| ShimError::ProviderError {
             status: 500,
             body: format!("no candidates in response: {}", resp),
+            retry_after: None,
         })?;
 
     let parts = candidate
@@ -354,6 +355,7 @@ fn transform_response_to_openai(model: &str, resp: &Value) -> Result<Value> {
             return Err(ShimError::ProviderError {
                 status: 502,
                 body: "Gemini response has no supported terminal finish reason".into(),
+                retry_after: None,
             })
         }
     };
@@ -606,6 +608,7 @@ impl Gemini {
             return Err(ShimError::ProviderError {
                 status: code,
                 body: msg.to_string(),
+                retry_after: None,
             });
         }
         transform_response_to_openai(model, &response)

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/openai.rs

@@ -363,7 +363,7 @@ impl Provider for OpenAi {
 }
 
 impl OpenAi {
-    fn transform_response_native(&self, model: &str, response: Value) -> Result<Value> {
+    pub(crate) fn transform_response_native(&self, model: &str, response: Value) -> Result<Value> {
         // Check for error (Responses API returns "error": null on success)
         if let Some(err) = response.get("error") {
             if !err.is_null() {
@@ -374,6 +374,7 @@ impl OpenAi {
                 return Err(ShimError::ProviderError {
                     status: 400,
                     body: msg.to_string(),
+                    retry_after: None,
                 });
             }
         }
@@ -384,6 +385,7 @@ impl OpenAi {
             .ok_or_else(|| ShimError::ProviderError {
                 status: 500,
                 body: "no output in response".to_string(),
+                retry_after: None,
             })?;
 
         // Extract reasoning summary
@@ -449,6 +451,7 @@ impl OpenAi {
                 return Err(ShimError::ProviderError {
                     status: 502,
                     body: "OpenAI response has no supported terminal status".into(),
+                    retry_after: None,
                 })
             }
         };
@@ -471,7 +474,11 @@ impl OpenAi {
 }
 
 impl OpenAi {
-    fn transform_stream_chunk_native(&self, model: &str, chunk: &str) -> Result<Option<String>> {
+    pub(crate) fn transform_stream_chunk_native(
+        &self,
+        model: &str,
+        chunk: &str,
+    ) -> Result<Option<String>> {
         let trimmed = chunk.trim();
         if trimmed.is_empty() || trimmed == "[DONE]" {
             return Ok(None);

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/openai_compat.rs

@@ -1,5 +1,7 @@
 use crate::error::{Result, ShimError};
 use crate::provider::{Provider, ProviderRequest};
+use crate::providers::openai::OpenAi;
+use crate::reasoning::{ReplayTarget, WireFormat};
 use crate::vision;
 use serde_json::{json, Value};
 
@@ -17,10 +19,18 @@ use serde_json::{json, Value};
 /// `name` (e.g. `"vllm"` / `"sglang"`) is both the provider key and the
 /// extension namespace: server-specific params (`chat_template_kwargs`,
 /// `separate_reasoning`, `guided_json`, `top_k`, …) go under `x-<name>`.
+///
+/// The wire is Chat Completions unless [`OpenAiCompatible::with_wire`] selects
+/// the Responses API, which SGLang also serves at `<base>/responses`. On that
+/// wire reasoning comes back as an item with its own id, so it can be replayed
+/// as a keyed block instead of bare `reasoning_content`; the request and
+/// response translation is the OpenAI adapter's, with this server's URL, its
+/// optional auth, and its `x-<name>` namespace.
 pub struct OpenAiCompatible {
     pub name: String,
     pub base_url: String,
     pub api_key: Option<String>,
+    wire: WireFormat,
 }
 
 impl OpenAiCompatible {
@@ -33,7 +43,66 @@ impl OpenAiCompatible {
             name: name.into(),
             base_url: base_url.into(),
             api_key,
+            wire: WireFormat::OpenAiChat,
+        }
+    }
+
+    /// Select the wire this server is spoken to on: `OpenAiChat` (the
+    /// default, `<base>/chat/completions`) or `OpenAiResponses`
+    /// (`<base>/responses`). The other wires are not OpenAI-compatible and
+    /// are a caller error.
+    pub fn with_wire(mut self, wire: WireFormat) -> Self {
+        assert!(
+            matches!(wire, WireFormat::OpenAiChat | WireFormat::OpenAiResponses),
+            "an OpenAI-compatible server speaks Chat Completions or Responses, not {wire:?}"
+        );
+        self.wire = wire;
+        self
+    }
+
+    pub fn wire(&self) -> WireFormat {
+        self.wire
+    }
+
+    /// Auth is optional — self-hosted servers are unauthenticated unless
+    /// launched with --api-key.
+    fn headers(&self) -> Vec<(String, String)> {
+        let mut headers = vec![("Content-Type".to_string(), "application/json".to_string())];
+        if let Some(key) = self.api_key.as_deref().filter(|k| !k.is_empty()) {
+            headers.push(("Authorization".to_string(), format!("Bearer {key}")));
+        }
+        headers
+    }
+
+    /// The OpenAI adapter pointed at this server, for the Responses wire. Its
+    /// translation is reused whole; only the URL, the auth and the extension
+    /// namespace are this server's.
+    fn responses_adapter(&self) -> OpenAi {
+        OpenAi::new(self.api_key.clone().unwrap_or_default())
+            .with_base_url(self.base_url.trim_end_matches('/').to_string())
+    }
+
+    fn transform_request_responses(&self, model: &str, request: &Value) -> Result<ProviderRequest> {
+        // The OpenAI adapter reads its overrides from `x-openai`, and applies
+        // them before the stateless and native-tool passes. Moving `x-<name>`
+        // there keeps that order, so an override cannot re-enable storage.
+        let mut request = request.clone();
+        let namespace = format!("x-{}", self.name);
+        if let Some(ext) = request
+            .as_object_mut()
+            .and_then(|obj| obj.remove(&namespace))
+            .and_then(|ext| ext.as_object().cloned())
+        {
+            let target = request["x-openai"].as_object().cloned().unwrap_or_default();
+            request["x-openai"] = Value::Object(target.into_iter().chain(ext).collect());
         }
+        let mut sent = self.responses_adapter().transform_request_for_target(
+            model,
+            &request,
+            &self.replay_target(model),
+        )?;
+        sent.headers = self.headers();
+        Ok(sent)
     }
 }
 
@@ -76,16 +145,15 @@ impl Provider for OpenAiCompatible {
         &self.name
     }
 
-    fn replay_target(&self, model: &str) -> crate::reasoning::ReplayTarget {
-        crate::reasoning::ReplayTarget::new(
-            self.name(),
-            model,
-            crate::reasoning::WireFormat::OpenAiChat,
-        )
-        .bind_account(&self.base_url, self.api_key.as_deref())
+    fn replay_target(&self, model: &str) -> ReplayTarget {
+        ReplayTarget::new(self.name(), model, self.wire)
+            .bind_account(&self.base_url, self.api_key.as_deref())
     }
 
     fn transform_request(&self, model: &str, request: &Value) -> Result<ProviderRequest> {
+        if self.wire == WireFormat::OpenAiResponses {
+            return self.transform_request_responses(model, request);
+        }
         let request = crate::schema::prepare_request(request);
         let request =
             crate::cache::prepare_request(&request, crate::reasoning::WireFormat::OpenAiChat)?;
@@ -140,14 +208,7 @@ impl Provider for OpenAiCompatible {
             }
         }
 
-        let mut headers = vec![("Content-Type".to_string(), "application/json".to_string())];
-        // Auth is optional — self-hosted servers are unauthenticated unless
-        // launched with --api-key.
-        if let Some(key) = &self.api_key {
-            if !key.is_empty() {
-                headers.push(("Authorization".to_string(), format!("Bearer {key}")));
-            }
-        }
+        let headers = self.headers();
 
         let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/'));
         crate::toolcall::validate_native(&body, &self.replay_target(model))?;
@@ -167,14 +228,26 @@ impl Provider for OpenAiCompatible {
 
     fn transform_response(&self, model: &str, response: Value) -> Result<Value> {
         let native = response.clone();
-        let mut result = self.transform_response_native(model, response)?;
+        let mut result = match self.wire {
+            WireFormat::OpenAiResponses => self
+                .responses_adapter()
+                .transform_response_native(model, response)?,
+            _ => self.transform_response_native(model, response)?,
+        };
+        // Captured against this server's target, not the OpenAI adapter's, so
+        // the block's origin names the provider that actually issued it.
         crate::reasoning::capture_response(&self.replay_target(model), &native, &mut result);
         crate::toolcall::capture_response(&self.replay_target(model), &native, &mut result)?;
         Ok(result)
     }
 
     fn transform_stream_chunk(&self, model: &str, chunk: &str) -> Result<Option<String>> {
-        let result = self.transform_stream_chunk_native(model, chunk)?;
+        let result = match self.wire {
+            WireFormat::OpenAiResponses => self
+                .responses_adapter()
+                .transform_stream_chunk_native(model, chunk)?,
+            _ => self.transform_stream_chunk_native(model, chunk)?,
+        };
         let native: Value = match serde_json::from_str(chunk) {
             Ok(v) => v,
             Err(_) => return Ok(result),
@@ -189,6 +262,7 @@ impl OpenAiCompatible {
             return Err(ShimError::ProviderError {
                 status: 502,
                 body: "invalid upstream response shape".into(),
+                retry_after: None,
             });
         }
         if let Some(err) = response.get("error") {
@@ -202,6 +276,7 @@ impl OpenAiCompatible {
                 return Err(ShimError::ProviderError {
                     status,
                     body: message,
+                    retry_after: None,
                 });
             }
         }

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/openrouter.rs

@@ -267,6 +267,7 @@ impl OpenRouter {
             return Err(ShimError::ProviderError {
                 status: 502,
                 body: "invalid upstream response shape".into(),
+                retry_after: None,
             });
         }
         // Non-stream errors usually surface via HTTP status, but a body-level
@@ -282,6 +283,7 @@ impl OpenRouter {
                 return Err(ShimError::ProviderError {
                     status,
                     body: message,
+                    retry_after: None,
                 });
             }
         }

{llmshim-0.7.2 → llmshim-0.8.0}/src/providers/xai.rs

@@ -370,6 +370,7 @@ impl Xai {
                 return Err(ShimError::ProviderError {
                     status: 400,
                     body: msg.to_string(),
+                    retry_after: None,
                 });
             }
         }
@@ -380,6 +381,7 @@ impl Xai {
             .ok_or_else(|| ShimError::ProviderError {
                 status: 500,
                 body: "no output in response".to_string(),
+                retry_after: None,
             })?;
 
         let mut text_content: Option<String> = None;
@@ -440,6 +442,7 @@ impl Xai {
                 return Err(ShimError::ProviderError {
                     status: 502,
                     body: "xAI response has no supported terminal status".into(),
+                    retry_after: None,
                 });
             }
         };

{llmshim-0.7.2 → llmshim-0.8.0}/src/proxy/error.rs

@@ -72,7 +72,7 @@ impl IntoResponse for ApiError {
                 "unknown_provider",
                 format!("Unknown provider or model: {}", p),
             ),
-            crate::error::ShimError::ProviderError { status, body } => {
+            crate::error::ShimError::ProviderError { status, body, .. } => {
                 let http_status = StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY);
                 let code = if *status == 400 {
                     "invalid_request"

{llmshim-0.7.2 → llmshim-0.8.0}/src/reasoning/normalize.rs

@@ -50,14 +50,30 @@ fn structured_block(
     Some(b)
 }
 
+/// The readable text of a Responses `reasoning` item. OpenAI's hosted models
+/// put it in `summary[]`; a server returning the model's own reasoning puts it
+/// in `content[]` as `reasoning_text` and leaves the summary empty. The summary
+/// is preferred when both exist; the content is the fallback, or the item's
+/// completed snapshot would replace every streamed delta with nothing.
 fn summary_text(payload: &Value) -> String {
-    payload["summary"]
-        .as_array()
-        .into_iter()
-        .flatten()
-        .filter_map(|p| p["text"].as_str())
-        .collect::<Vec<_>>()
-        .join("\n")
+    let joined = |field: &str, kind: Option<&str>| {
+        payload[field]
+            .as_array()
+            .into_iter()
+            .flatten()
+            .filter(|p| kind.is_none_or(|kind| p["type"] == kind))
+            .filter_map(|p| p["text"].as_str())
+            .collect::<Vec<_>>()
+            .join("\n")
+    };
+    // Every summary part is read, as before; only the content fallback is
+    // typed, because a content part can be something other than reasoning.
+    let summary = joined("summary", None);
+    if summary.is_empty() {
+        joined("content", Some("reasoning_text"))
+    } else {
+        summary
+    }
 }
 
 pub(super) fn chat_blocks(message: &Value, origin: &ReasoningOrigin) -> Vec<ReasoningBlock> {
@@ -352,11 +368,21 @@ pub struct ReasoningAccumulator {
 }
 impl ReasoningAccumulator {
     pub fn push(&mut self, message: &Value) {
-        for fragment in message["reasoning"].as_array().into_iter().flatten() {
-            let key = fragment["item_id"]
-                .as_str()
-                .map(|s| format!("item:{s}"))
-                .unwrap_or_else(|| format!("index:{}", fragment.get("index").unwrap_or(&json!(0))));
+        for (position, fragment) in message["reasoning"]
+            .as_array()
+            .into_iter()
+            .flatten()
+            .enumerate()
+        {
+            // A fragment carrying neither key is a whole block already: a buffered
+            // answer's blocks lose their stream index when they are assembled. Its
+            // position in this message keys it, or every unkeyed block would merge
+            // into one and an encrypted block could swallow a readable one.
+            let key = match (fragment["item_id"].as_str(), fragment.get("index")) {
+                (Some(id), _) => format!("item:{id}"),
+                (None, Some(index)) => format!("index:{index}"),
+                (None, None) => format!("index:{position}"),
+            };
             if !self.parts.contains_key(&key) {
                 self.order.push(key.clone());
             }

{llmshim-0.7.2 → llmshim-0.8.0}/src/reasoning.rs

@@ -571,6 +571,7 @@ pub(crate) fn enforce_stateless(body: &mut Value) -> crate::error::Result<()> {
             .ok_or_else(|| crate::error::ShimError::ProviderError {
                 status: 400,
                 body: "include must be an array".into(),
+                retry_after: None,
             })?;
     if !include.iter().any(|v| v == "reasoning.encrypted_content") {
         include.push(json!("reasoning.encrypted_content"));