llmshim 0.6.0__tar.gz → 0.7.0__tar.gz

{llmshim-0.6.0 → llmshim-0.7.0}/Cargo.lock

@@ -1180,7 +1180,7 @@ checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
 
 [[package]]
 name = "llmshim"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "async-stream",
  "async-trait",

{llmshim-0.6.0 → llmshim-0.7.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llmshim"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 description = "Blazing fast LLM API translation layer in pure Rust"
 license = "MIT OR Apache-2.0"

{llmshim-0.6.0 → llmshim-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llmshim
-Version: 0.6.0
+Version: 0.7.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{llmshim-0.6.0 → llmshim-0.7.0}/docs/src/proxy/scaling.md

@@ -106,11 +106,18 @@ buckets. A gateway key's identity may carry `budget_usd` and an optional
 {"sk-example": {"tenant": "acme", "tier": 1, "budget_usd": 100, "budget_window_secs": 86400}}
 ```
 
-Cost is only knowable after a response, so the cap is checked before dispatch
-and charged after: one in-flight request can overshoot.
-
-A response the catalog cannot price is **not** charged — recording zero would let
-an unpriced model run forever under a budget. So that a cap cannot silently stop
+Cost is only knowable after a response, so the cap is checked before dispatch and
+charged after. Everything admitted between the last charge and the next check
+passes, so the overshoot bound is **admitted concurrency × the most expensive
+request**, multiplied again across replicas that have not yet shared their
+ledger. Size a cap with that headroom in mind rather than as a hard ceiling.
+
+A response the catalog cannot price at all is **not** charged — recording zero
+would let an unpriced model run forever under a budget. A model that prices only
+*some* token classes is charged at its highest published rate for the rest, so a
+partial price bounds the charge from above instead of voiding it: 2,537 of the
+7,461 priced models in the catalog publish no `cache_read` rate, and voiding
+those would have reopened this same hole one layer down. So that a cap cannot silently stop
 binding, a request whose target has **no catalog price is refused before it runs**
 when a budget is set:
 

{llmshim-0.6.0 → llmshim-0.7.0}/src/cost.rs

@@ -29,6 +29,27 @@ fn charge(tokens: u64, rate: Option<f64>) -> Option<f64> {
     Some(tokens as f64 * rate? / PER_MILLION)
 }
 
+/// The highest rate this model publishes, used where a class has none of its own.
+///
+/// Catalogs price partially and often: **2,537 of 7,461 priced models in the
+/// vendored snapshot carry no `cache_read` rate and 5,892 no `cache_write`**,
+/// including `gpt-5-pro` and `o3-pro`, priced `{input, output}` only. Returning
+/// `None` for those made the whole response unpriceable the moment it reported a
+/// cached token — and a discarded charge is a spend cap that silently stops
+/// binding, which is the failure this crate exists to prevent.
+///
+/// Substituting the highest published rate can only over-estimate, never under.
+/// For a cap that is the safe direction: it spends a budget slightly early, where
+/// under-estimating spends it forever.
+fn highest_published_rate(cost: &Cost) -> Option<f64> {
+    [cost.input, cost.output, cost.cache_read, cost.cache_write]
+        .into_iter()
+        .flatten()
+        .fold(None, |best: Option<f64>, rate| {
+            Some(best.map_or(rate, |b| b.max(rate)))
+        })
+}
+
 /// Price one normalized usage object.
 ///
 /// Input is charged on `uncached_input_tokens` — the prompt minus whatever
@@ -53,11 +74,17 @@ pub fn price(usage: &Value, cost: &Cost) -> Option<f64> {
         return None;
     }
 
+    // A class the model does not price falls back to its highest published rate
+    // rather than voiding the total. See `highest_published_rate`: the result is
+    // an upper bound, which is the only safe direction for a budget.
+    let ceiling = highest_published_rate(cost);
+    let rate_for = |own: Option<f64>| own.or(ceiling);
+
     Some(
-        charge(input, cost.input)?
-            + charge(count(usage, "completion_tokens"), cost.output)?
-            + charge(cache_read, cost.cache_read)?
-            + charge(cache_write, cost.cache_write)?,
+        charge(input, rate_for(cost.input))?
+            + charge(count(usage, "completion_tokens"), rate_for(cost.output))?
+            + charge(cache_read, rate_for(cost.cache_read))?
+            + charge(cache_write, rate_for(cost.cache_write))?,
     )
 }
 
@@ -221,14 +248,21 @@ mod tests {
     }
 
     #[test]
-    fn a_missing_cache_rate_poisons_the_total_only_when_it_is_used() {
+    fn a_missing_cache_rate_is_charged_at_the_models_highest_rate() {
+        // This replaced an assertion that a used-but-unpriced class voids the
+        // total. That instinct was right — never under-report — but the remedy
+        // was wrong: `None` is discarded by `SpendCap::record`, so a spend cap
+        // silently stopped binding for the 2,537 catalogued models that publish
+        // no `cache_read` rate. An upper bound honours the same principle
+        // without the hole.
         let partial = Cost {
             input: Some(3.0),
             output: Some(15.0),
             cache_read: None,
             cache_write: None,
         };
-        // No cache tokens: the missing rates are irrelevant.
+
+        // No cache tokens: the missing rates never come into play.
         close(
             price(
                 &json!({"uncached_input_tokens": 1_000_000, "completion_tokens": 0, "cache_read_tokens": 0}),
@@ -236,14 +270,26 @@ mod tests {
             ),
             3.0,
         );
-        // Cache tokens actually used, with no rate to charge them at.
-        assert_eq!(
+
+        // Cache tokens used with no rate of their own: charged at 15.0, the
+        // highest rate this model publishes — an over-estimate, never an under.
+        close(
             price(
-                &json!({"uncached_input_tokens": 1_000_000, "cache_read_tokens": 1}),
+                &json!({"uncached_input_tokens": 1_000_000, "cache_read_tokens": 1_000_000}),
                 &partial,
             ),
-            None,
-            "an unpriced bucket that was used must not round down to a partial sum"
+            18.0,
+        );
+
+        // The bound must never fall below what a complete price would charge.
+        let complete = Cost {
+            cache_read: Some(0.3),
+            ..partial
+        };
+        let usage = json!({"uncached_input_tokens": 1_000_000, "cache_read_tokens": 1_000_000});
+        assert!(
+            price(&usage, &partial).unwrap() >= price(&usage, &complete).unwrap(),
+            "a fallback rate must bound the real one from above, or a cap under-charges"
         );
     }
 

{llmshim-0.6.0 → llmshim-0.7.0}/src/proxy/types.rs

@@ -119,7 +119,12 @@ pub struct Usage {
     pub cache_read_tokens: u64,
     pub cache_write_tokens: u64,
     /// USD charged for this response. `null` means the catalog carries no price
-    /// for the model — it never means free. See `llmshim::cost`.
+    /// for the model at all — it never means free.
+    ///
+    /// Where a model prices some token classes and not others, the unpriced ones
+    /// are charged at its highest published rate, so this is an **upper bound**
+    /// rather than `null`. Under-reporting would let a spend cap stop binding;
+    /// over-reporting merely spends a budget slightly early. See `llmshim::cost`.
     pub cost_usd: Option<f64>,
 }