llmshim 0.5.0__tar.gz → 0.6.0__tar.gz

{llmshim-0.5.0 → llmshim-0.6.0}/Cargo.lock

@@ -1180,7 +1180,7 @@ checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
 
 [[package]]
 name = "llmshim"
-version = "0.5.0"
+version = "0.6.0"
 dependencies = [
  "async-stream",
  "async-trait",

{llmshim-0.5.0 → llmshim-0.6.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llmshim"
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 description = "Blazing fast LLM API translation layer in pure Rust"
 license = "MIT OR Apache-2.0"

{llmshim-0.5.0 → llmshim-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmshim
-Version: 0.5.0
+Version: 0.6.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{llmshim-0.5.0 → llmshim-0.6.0}/docs/src/proxy/scaling.md

@@ -107,9 +107,30 @@ buckets. A gateway key's identity may carry `budget_usd` and an optional
 ```
 
 Cost is only knowable after a response, so the cap is checked before dispatch
-and charged after: one in-flight request can overshoot. A response the catalog
-cannot price (`cost_usd: null`) is **not** charged — recording zero would let an
-unpriced model run forever under a budget, so a hard cap requires priced models.
+and charged after: one in-flight request can overshoot.
+
+A response the catalog cannot price is **not** charged — recording zero would let
+an unpriced model run forever under a budget. So that a cap cannot silently stop
+binding, a request whose target has **no catalog price is refused before it runs**
+when a budget is set:
+
+```
+400 {"error":{"code":"unpriceable_under_budget","param":"model", …}}
+```
+
+It is deliberately not a `429`: retrying never clears it. Three ways forward —
+use a priced model, add a local price override in the catalog, or accept the risk
+explicitly per key:
+
+```json
+{"sk-example": {"tenant": "acme", "budget_usd": 100, "budget_allow_unpriced": true}}
+```
+
+`budget_allow_unpriced` defaults to `false`. With it set, those requests run and
+are not charged, and each one logs a warning and increments
+`llmshim_gateway_unpriced_under_cap_total{provider,model}` — a non-zero counter
+means the budget is not binding for that target. An accepted risk should stay
+measurable rather than become an assumption.
 
 ## One replica or a coordinated fleet
 

{llmshim-0.5.0 → llmshim-0.6.0}/src/cost.rs

@@ -80,6 +80,14 @@ pub fn cost_usd(provider: &str, model: &str, usage: &Value) -> Option<f64> {
     price(usage, &for_target(provider, model)?)
 }
 
+/// Whether this target can be priced at all, asked *before* the request runs.
+///
+/// Pricing after the fact cannot enforce a budget: by then the money is spent.
+/// A cap therefore needs this question answered up front.
+pub fn is_priceable(provider: &str, model: &str) -> bool {
+    for_target(provider, model).is_some()
+}
+
 /// Read a cost already stamped onto a normalized usage object. A stamped
 /// `null` and an unstamped body both mean "not known".
 pub fn stamped(usage: &Value) -> Option<f64> {

{llmshim-0.5.0 → llmshim-0.6.0}/src/gateway/auth.rs

@@ -35,6 +35,15 @@ pub struct Identity {
     /// Window the cap applies to, in seconds. Defaults to one day.
     #[serde(default)]
     pub budget_window_secs: Option<u64>,
+    /// Permit requests the catalog cannot price while a budget is set.
+    ///
+    /// Defaults to **false**, which refuses them. An unpriced response cannot be
+    /// charged, so under a cap it is spend the ledger never sees — the budget
+    /// silently stops binding and nothing says so. Setting this to `true` is an
+    /// operator accepting that risk knowingly; the requests are still counted and
+    /// reported so the hole is visible rather than assumed absent.
+    #[serde(default)]
+    pub budget_allow_unpriced: bool,
 }
 
 /// Authentication failure — both map to HTTP 401.
@@ -103,6 +112,7 @@ impl KeyStore {
                 tpm: None,
                 budget_usd: None,
                 budget_window_secs: None,
+                budget_allow_unpriced: false,
             }),
             KeyStore::Enforced(map) => {
                 let key = bearer_token(headers).ok_or(AuthError::MissingKey)?;
@@ -171,6 +181,7 @@ mod tests {
                 tpm: None,
                 budget_usd: None,
                 budget_window_secs: None,
+                budget_allow_unpriced: false,
             },
         );
         let store = KeyStore::enforced(keys);

{llmshim-0.5.0 → llmshim-0.6.0}/src/gateway/http.rs

@@ -325,16 +325,56 @@ impl GatewayState {
         &self,
         identity: &crate::gateway::auth::Identity,
         provider: &str,
+        model: &str,
     ) -> Result<(), ApiError> {
-        match self.spend.check(identity).await {
-            Ok(()) => Ok(()),
-            Err(retry) => {
+        use crate::gateway::quota::BudgetRefusal;
+        match self.spend.check(identity, provider, model).await {
+            Ok(()) => {
+                // An operator who opted in still gets told, every time. An
+                // accepted risk that stops being visible becomes an assumption.
+                if crate::gateway::quota::SpendCap::is_unpriced_under_cap(identity, provider, model)
+                {
+                    crate::gateway::metrics::incr(
+                        crate::gateway::metrics::UNPRICED_UNDER_CAP,
+                        &[("provider", provider), ("model", model)],
+                    );
+                    eprintln!(
+                        "gateway: tenant {} running {provider}/{model} unpriced under a spend \
+                         cap; this spend is NOT charged against the budget \
+                         (budget_allow_unpriced is set)",
+                        identity.tenant
+                    );
+                }
+                Ok(())
+            }
+            Err(BudgetRefusal::Exhausted(retry)) => {
                 crate::gateway::metrics::incr(
                     crate::gateway::metrics::REJECTED,
                     &[("provider", provider), ("reason", "tenant_budget")],
                 );
                 Err(ApiError::RateLimited(retry))
             }
+            Err(BudgetRefusal::Unpriceable) => {
+                crate::gateway::metrics::incr(
+                    crate::gateway::metrics::REJECTED,
+                    &[
+                        ("provider", provider),
+                        ("reason", "unpriceable_under_budget"),
+                    ],
+                );
+                // Not a 429: retrying never clears this. The catalog has no
+                // price for the target, so the cap cannot be enforced against it.
+                Err(ApiError::Shim(crate::error::ShimError::ProviderError {
+                    status: 400,
+                    body: format!(
+                        "{{\"error\":{{\"message\":\"no catalog price for '{provider}/{model}', \
+                         so it cannot be charged against this key's spend budget. Use a priced \
+                         model, add a local price override, or set budget_allow_unpriced on the \
+                         key to run it uncharged.\",\"type\":\"invalid_request_error\",\
+                         \"param\":\"model\",\"code\":\"unpriceable_under_budget\"}}}}"
+                    ),
+                }))
+            }
         }
     }
 }
@@ -410,7 +450,10 @@ async fn chat(
         .map(str::to_string);
     let (provider_name, gw, identity) = build_request(&state, &headers, &req)?;
     state.enforce_quota(&identity, &provider_name, gw.permits)?;
-    state.enforce_budget(&identity, &provider_name).await?;
+    let (_, budget_model) = state.router.resolve(&req.model)?;
+    state
+        .enforce_budget(&identity, &provider_name, &budget_model)
+        .await?;
 
     // Retry-safety: a repeated Idempotency-Key returns the first result.
     if let Some(key) = &idem_key {
@@ -461,7 +504,14 @@ async fn chat_stream_inner(
     if let Err(e) = state.enforce_quota(&identity, &provider_name, gw.permits) {
         return e.into_response();
     }
-    if let Err(e) = state.enforce_budget(&identity, &provider_name).await {
+    let budget_model = match state.router.resolve(&req.model) {
+        Ok((_, m)) => m,
+        Err(e) => return ApiError::from(e).into_response(),
+    };
+    if let Err(e) = state
+        .enforce_budget(&identity, &provider_name, &budget_model)
+        .await
+    {
         return e.into_response();
     }
 
@@ -673,6 +723,7 @@ mod native_tests {
                 tpm: None,
                 budget_usd: None,
                 budget_window_secs: None,
+                budget_allow_unpriced: false,
             },
         )]);
         let state = Arc::new(GatewayState {

{llmshim-0.5.0 → llmshim-0.6.0}/src/gateway/metrics.rs

@@ -17,6 +17,10 @@ pub const REQUESTS: &str = "llmshim_gateway_requests_total";
 pub const DISPATCHED: &str = "llmshim_gateway_dispatched_total";
 /// Requests that did not dispatch (labels: provider, reason).
 pub const REJECTED: &str = "llmshim_gateway_rejected_total";
+
+/// Requests permitted to run without a price while a spend cap is configured.
+/// Non-zero means a budget is not binding for that provider/model.
+pub const UNPRICED_UNDER_CAP: &str = "llmshim_gateway_unpriced_under_cap_total";
 /// In-flight upstream calls (gauge, label: provider).
 pub const INFLIGHT: &str = "llmshim_gateway_inflight";
 /// Time a job waited in the queue before dispatch (histogram, label: provider).

{llmshim-0.5.0 → llmshim-0.6.0}/src/gateway/quota.rs

@@ -184,6 +184,23 @@ impl Default for SpendCap {
     }
 }
 
+/// Why a budgeted request was refused.
+///
+/// The two are kept apart because the correct answer to the caller differs.
+/// `Exhausted` is a wait — the window resets and the same request succeeds.
+/// `Unpriceable` is a deployment fact: the catalog has no price for this target,
+/// so the cap cannot be enforced against it and no amount of retrying changes
+/// that. Collapsing them into one 429 would tell an operator to wait for a
+/// condition that never clears.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum BudgetRefusal {
+    /// The window's spend has reached the cap. Carries the time until reset.
+    Exhausted(Duration),
+    /// The catalog cannot price this target, and the identity has not opted in
+    /// to running unpriced under a cap.
+    Unpriceable,
+}
+
 impl SpendCap {
     /// A cap governing this instance only.
     pub fn in_memory() -> Self {
@@ -206,28 +223,65 @@ impl SpendCap {
         )
     }
 
-    /// A no-op when the identity carries no budget. `Err(retry_after)` once the
-    /// window's spend has reached the cap, where the wait is the window reset.
-    pub async fn check(&self, identity: &crate::gateway::auth::Identity) -> Result<(), Duration> {
+    /// A no-op when the identity carries no budget.
+    ///
+    /// Two ways a budgeted request is refused, and they are not the same event:
+    /// [`BudgetRefusal::Exhausted`] is temporary and clears at the window reset;
+    /// [`BudgetRefusal::Unpriceable`] never clears on its own, because the
+    /// catalog has no price for the target and retrying changes nothing.
+    pub async fn check(
+        &self,
+        identity: &crate::gateway::auth::Identity,
+        provider: &str,
+        model: &str,
+    ) -> Result<(), BudgetRefusal> {
         // `budget_usd: 0` freezes the key rather than unlimiting it: an
         // operator typing zero means "spend nothing", and the opposite reading
         // is the expensive one to be wrong about.
         let Some(budget) = identity.budget_usd else {
             return Ok(());
         };
+
+        // A response the catalog cannot price is spend the ledger never sees.
+        // Allowing it under a cap does not merely lose one charge — it makes the
+        // cap stop binding for every later request too, silently. Refusing is the
+        // only outcome that keeps "a budget is set" and "the budget is enforced"
+        // the same statement.
+        if !crate::cost::is_priceable(provider, model) && !identity.budget_allow_unpriced {
+            return Err(BudgetRefusal::Unpriceable);
+        }
+
         let window = Self::window(identity);
         if self.store.spent(&identity.tenant, window).await >= budget {
-            return Err(window_reset(window));
+            return Err(BudgetRefusal::Exhausted(window_reset(window)));
         }
         Ok(())
     }
 
+    /// Whether this request is about to run unpriced under a cap.
+    ///
+    /// True only when the operator opted in. Callers report it so an accepted
+    /// risk stays measurable instead of becoming an assumption.
+    #[must_use]
+    pub fn is_unpriced_under_cap(
+        identity: &crate::gateway::auth::Identity,
+        provider: &str,
+        model: &str,
+    ) -> bool {
+        identity.budget_usd.is_some() && !crate::cost::is_priceable(provider, model)
+    }
+
     /// Charge a completed response.
     ///
-    /// `None` means the catalog could not price the model. That spend is
+    /// `None` means the catalog could not price the model, and that spend is
     /// **not** recorded — charging zero would quietly let an unpriced model run
-    /// forever under a budget. Operators who need a hard cap must ensure their
-    /// models are priced; `cost_usd: null` in the response is the signal.
+    /// forever under a budget.
+    ///
+    /// This used to be the whole story, and it was a hole: the cap silently
+    /// stopped binding and the only signal was a `null` in a response body an
+    /// operator had to notice. [`SpendCap::check`] now refuses unpriceable
+    /// targets under a budget before they run, so reaching here with `None`
+    /// means the operator set `budget_allow_unpriced` and accepted it.
     pub async fn record(&self, identity: &crate::gateway::auth::Identity, usd: Option<f64>) {
         let Some(usd) = usd.filter(|u| *u > 0.0) else {
             return;
@@ -280,27 +334,115 @@ mod tests {
             tpm: None,
             budget_usd: Some(budget),
             budget_window_secs: Some(3600),
+            budget_allow_unpriced: false,
         }
     }
 
+    /// A target the catalog prices. Asserted rather than assumed: if the catalog
+    /// stops pricing it, these tests must fail loudly rather than quietly start
+    /// exercising the unpriceable path instead of the budget path.
+    const PRICED: (&str, &str) = ("anthropic", "claude-sonnet-4-6");
+
+    /// A model the catalog does not price. Deliberately implausible so it cannot
+    /// start being priced by a catalog refresh and quietly neuter these tests.
+    const UNPRICED: &str = "definitely-not-a-model-xyz";
+
     #[tokio::test]
     async fn a_budget_trips_once_the_window_is_spent() {
         let cap = SpendCap::in_memory();
         let acme = capped("acme", 1.0);
 
-        assert!(cap.check(&acme).await.is_ok(), "under budget admits");
+        assert!(
+            crate::cost::is_priceable(PRICED.0, PRICED.1),
+            "test fixture must be priced or this tests the wrong path"
+        );
+
+        let (p, m) = PRICED;
+        assert!(cap.check(&acme, p, m).await.is_ok(), "under budget admits");
         cap.record(&acme, Some(0.75)).await;
-        assert!(cap.check(&acme).await.is_ok(), "still under budget");
+        assert!(cap.check(&acme, p, m).await.is_ok(), "still under budget");
         cap.record(&acme, Some(0.30)).await;
 
-        let retry = cap.check(&acme).await.expect_err("over budget must reject");
+        match cap.check(&acme, p, m).await {
+            Err(BudgetRefusal::Exhausted(retry)) => assert!(
+                retry > Duration::ZERO,
+                "rejection must say when to come back"
+            ),
+            other => panic!("over budget must reject as Exhausted, got {other:?}"),
+        }
+
+        // Another tenant's ledger is its own.
+        assert!(cap.check(&capped("other", 1.0), p, m).await.is_ok());
+    }
+
+    /// A target with no catalog price cannot be charged. Under a cap that is not
+    /// a lost charge, it is a cap that stops binding — so it must be refused, and
+    /// refused *differently* from being out of budget.
+    #[tokio::test]
+    async fn an_unpriceable_model_is_refused_under_a_budget() {
+        let cap = SpendCap::in_memory();
+        let acme = capped("acme", 100.0);
+        assert!(
+            !crate::cost::is_priceable("anthropic", UNPRICED),
+            "fixture must be unpriced or this tests nothing"
+        );
+
+        let refusal = cap
+            .check(&acme, "anthropic", UNPRICED)
+            .await
+            .expect_err("an unpriceable model under a cap must be refused");
+        assert_eq!(
+            refusal,
+            BudgetRefusal::Unpriceable,
+            "must not masquerade as Exhausted: retrying never clears this"
+        );
+
+        // The budget is nowhere near spent — the refusal is about priceability.
+        assert!(cap.check(&acme, PRICED.0, PRICED.1).await.is_ok());
+    }
+
+    /// Opting in is allowed, because a new model can outrun the catalog. It is
+    /// explicit, per-key, and defaults to off.
+    #[tokio::test]
+    async fn an_operator_can_opt_in_to_running_unpriced() {
+        let cap = SpendCap::in_memory();
+        let mut acme = capped("acme", 100.0);
+        acme.budget_allow_unpriced = true;
+
+        assert!(
+            cap.check(&acme, "anthropic", UNPRICED).await.is_ok(),
+            "explicit opt-in must admit"
+        );
         assert!(
-            retry > Duration::ZERO,
-            "rejection must say when to come back"
+            SpendCap::is_unpriced_under_cap(&acme, "anthropic", UNPRICED),
+            "and the caller must be able to see it, so the hole stays measurable"
         );
+        assert!(
+            !SpendCap::is_unpriced_under_cap(&acme, PRICED.0, PRICED.1),
+            "a priced target is not a hole"
+        );
+    }
 
-        // Another tenant's ledger is its own.
-        assert!(cap.check(&capped("other", 1.0)).await.is_ok());
+    /// With no cap there is nothing to enforce, so priceability is irrelevant.
+    /// Refusing here would break every unbudgeted key the moment a model is new.
+    #[tokio::test]
+    async fn without_a_budget_an_unpriceable_model_is_fine() {
+        let cap = SpendCap::in_memory();
+        let free = crate::gateway::auth::Identity {
+            tenant: "free".into(),
+            tier: 0,
+            rpm: None,
+            tpm: None,
+            budget_usd: None,
+            budget_window_secs: None,
+            budget_allow_unpriced: false,
+        };
+        assert!(cap.check(&free, "anthropic", UNPRICED).await.is_ok());
+        assert!(!SpendCap::is_unpriced_under_cap(
+            &free,
+            "anthropic",
+            UNPRICED
+        ));
     }
 
     #[tokio::test]
@@ -308,7 +450,10 @@ mod tests {
         let cap = SpendCap::in_memory();
         let frozen = capped("frozen", 0.0);
         assert!(
-            cap.check(&frozen).await.is_err(),
+            matches!(
+                cap.check(&frozen, PRICED.0, PRICED.1).await,
+                Err(BudgetRefusal::Exhausted(_))
+            ),
             "budget_usd: 0 must mean spend nothing, not spend anything"
         );
     }
@@ -323,10 +468,11 @@ mod tests {
             tpm: None,
             budget_usd: None,
             budget_window_secs: None,
+            budget_allow_unpriced: false,
         };
         for _ in 0..100 {
             cap.record(&uncapped, Some(1_000.0)).await;
-            assert!(cap.check(&uncapped).await.is_ok());
+            assert!(cap.check(&uncapped, PRICED.0, PRICED.1).await.is_ok());
         }
 
         // An unpriceable response cannot be charged; it must not read as free
@@ -337,7 +483,7 @@ mod tests {
             cap.store.spent("acme", Duration::from_secs(3600)).await,
             0.0
         );
-        assert!(cap.check(&acme).await.is_ok());
+        assert!(cap.check(&acme, PRICED.0, PRICED.1).await.is_ok());
     }
 
     #[tokio::test(start_paused = true)]