litellm-haystack 1.1.2__tar.gz → 2.0.0__tar.gz

{litellm_haystack-1.1.2 → litellm_haystack-2.0.0}/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## [integrations/litellm-v1.2.0] - 2026-07-30
+
+### 🚀 Features
+
+- *(litellm)* Include provider cache tokens in usage metadata (#3678)
+
+### 🐛 Bug Fixes
+
+- Cap litellm dependency below 1.92.0 (#3601)
+
+### 🌀 Miscellaneous
+
+
 ## [integrations/litellm-v1.1.1] - 2026-07-15
 
 ### 🧹 Chores

{litellm_haystack-1.1.2 → litellm_haystack-2.0.0}/LICENSE.txt

@@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
 
 To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
 
-Copyright [yyyy] [name of copyright owner]
+Copyright 2023-present deepset GmbH
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

{litellm_haystack-1.1.2 → litellm_haystack-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: litellm-haystack
-Version: 1.1.2
+Version: 2.0.0
 Summary: An integration of LiteLLM AI gateway into the Haystack framework, routing to 100+ LLM providers.
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/litellm#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -12,13 +12,12 @@ Keywords: ai-gateway,haystack,litellm,llm
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: haystack-ai>=2.30.0
 Requires-Dist: litellm<1.92.0,>=1.84.0
 Description-Content-Type: text/markdown

{litellm_haystack-1.1.2 → litellm_haystack-2.0.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = "litellm-haystack"
 dynamic = ["version"]
 description = "An integration of LiteLLM AI gateway into the Haystack framework, routing to 100+ LLM providers."
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 license = "Apache-2.0"
 keywords = ["haystack", "litellm", "llm", "ai-gateway"]
 authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
@@ -15,7 +15,6 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: 3.13",

{litellm_haystack-1.1.2 → litellm_haystack-2.0.0}/src/haystack_integrations/components/generators/litellm/chat/chat_generator.py

@@ -25,7 +25,7 @@ from haystack.tools import (
     flatten_tools_or_toolsets,
     serialize_tools_or_toolset,
 )
-from haystack.utils.auth import Secret, deserialize_secrets_inplace
+from haystack.utils.auth import Secret
 from haystack.utils.callable_serialization import deserialize_callable, serialize_callable
 
 logger = logging.getLogger(__name__)
@@ -149,7 +149,9 @@ class LiteLLMChatGenerator:
         :param messages: Input messages as ChatMessage instances.
             If a string is provided, it is converted to a list containing a ChatMessage with user role.
         :param streaming_callback: Override the streaming callback for this call.
-        :param generation_kwargs: Override generation parameters for this call.
+        :param generation_kwargs: Additional parameters for this call. These are merged per key with the
+            `generation_kwargs` passed at initialization: keys provided here take precedence, keys set only
+            at initialization are kept.
         :param tools: Override tools for this call.
         :returns: A dict with key ``replies`` containing ChatMessage instances.
         """
@@ -186,7 +188,9 @@ class LiteLLMChatGenerator:
         :param messages: Input messages as ChatMessage instances.
             If a string is provided, it is converted to a list containing a ChatMessage with user role.
         :param streaming_callback: Override the streaming callback for this call.
-        :param generation_kwargs: Override generation parameters for this call.
+        :param generation_kwargs: Additional parameters for this call. These are merged per key with the
+            `generation_kwargs` passed at initialization: keys provided here take precedence, keys set only
+            at initialization are kept.
         :param tools: Override tools for this call.
         :returns: A dict with key ``replies`` containing ChatMessage instances.
         """
@@ -246,7 +250,6 @@ class LiteLLMChatGenerator:
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "LiteLLMChatGenerator":
         """Deserialize a component from a dictionary."""
-        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
         deserialize_tools_or_toolset_inplace(data["init_parameters"], key="tools")
         if data["init_parameters"].get("streaming_callback"):
             data["init_parameters"]["streaming_callback"] = deserialize_callable(
@@ -266,16 +269,12 @@ _FINISH_REASON_MAPPING: dict[str, FinishReason] = {
 }
 
 
-def _extract_usage(obj: Any) -> dict[str, int]:
+def _extract_usage(obj: Any) -> dict[str, Any]:
     """Pull token usage off a litellm response or chunk, tolerating its absence."""
     usage = getattr(obj, "usage", None)
-    if not usage:
-        return {}
-    return {
-        "prompt_tokens": getattr(usage, "prompt_tokens", 0),
-        "completion_tokens": getattr(usage, "completion_tokens", 0),
-        "total_tokens": getattr(usage, "total_tokens", 0),
-    }
+    if usage is not None and hasattr(usage, "model_dump"):
+        return usage.model_dump(exclude_none=True)
+    return {}
 
 
 def _build_chat_message(response: Any, choice: Any) -> ChatMessage:

{litellm_haystack-1.1.2 → litellm_haystack-2.0.0}/tests/test_litellm_chat_generator.py

@@ -13,6 +13,7 @@ import pytest
 from haystack.dataclasses import ChatMessage, StreamingChunk
 from haystack.tools import Tool
 from haystack.utils.auth import Secret
+from litellm.types.utils import Usage
 
 from haystack_integrations.components.generators.litellm import LiteLLMChatGenerator
 
@@ -29,14 +30,9 @@ def _make_mock_response(content="Hello!", model="openai/gpt-4o", tool_calls=None
     choice.index = 0
     choice.finish_reason = "stop"
 
-    usage = MagicMock()
-    usage.prompt_tokens = 10
-    usage.completion_tokens = 5
-    usage.total_tokens = 15
-
     resp = MagicMock()
     resp.choices = [choice]
-    resp.usage = usage
+    resp.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
     resp.model = model
     return resp
 
@@ -205,18 +201,20 @@ class TestRun:
             call_kwargs = fake_litellm.completion.call_args
             assert call_kwargs.kwargs["api_base"] == "https://proxy.local"
 
-    def test_generation_kwargs_merged(self):
-        gen = LiteLLMChatGenerator(model="openai/gpt-4o", generation_kwargs={"temperature": 0.5})
+    def test_run_with_generation_kwargs(self):
+        gen = LiteLLMChatGenerator(
+            model="openai/gpt-4o", generation_kwargs={"temperature": 0.5, "max_completion_tokens": 100}
+        )
         mock_resp = _make_mock_response()
 
         fake_litellm = types.ModuleType("litellm")
         fake_litellm.completion = MagicMock(return_value=mock_resp)
 
         with mock.patch.dict(sys.modules, {"litellm": fake_litellm}):
-            gen.run(messages=[ChatMessage.from_user("hi")], generation_kwargs={"max_tokens": 100})
+            gen.run(messages=[ChatMessage.from_user("hi")], generation_kwargs={"temperature": 0.9})
             call_kwargs = fake_litellm.completion.call_args
-            assert call_kwargs.kwargs["temperature"] == 0.5
-            assert call_kwargs.kwargs["max_tokens"] == 100
+            assert call_kwargs.kwargs["temperature"] == 0.9
+            assert call_kwargs.kwargs["max_completion_tokens"] == 100
 
     def test_empty_messages_returns_empty(self):
         gen = LiteLLMChatGenerator()
@@ -235,6 +233,30 @@ class TestRun:
             meta = result["replies"][0].meta
             assert meta["usage"]["prompt_tokens"] == 10
             assert meta["usage"]["completion_tokens"] == 5
+            assert set(meta["usage"]) == {"prompt_tokens", "completion_tokens", "total_tokens"}
+
+    def test_cache_tokens_in_meta(self):
+        gen = LiteLLMChatGenerator(model="anthropic/claude-sonnet-4-20250514")
+        mock_resp = _make_mock_response(model="anthropic/claude-sonnet-4-20250514")
+        mock_resp.usage = Usage(
+            prompt_tokens=2010,
+            completion_tokens=5,
+            total_tokens=2015,
+            cache_creation_input_tokens=1200,
+            cache_read_input_tokens=800,
+        )
+
+        fake_litellm = types.ModuleType("litellm")
+        fake_litellm.completion = MagicMock(return_value=mock_resp)
+
+        with mock.patch.dict(sys.modules, {"litellm": fake_litellm}):
+            usage = gen.run(messages=[ChatMessage.from_user("hi")])["replies"][0].meta["usage"]
+            assert usage["cache_creation_input_tokens"] == 1200
+            assert usage["cache_read_input_tokens"] == 800
+            assert usage["prompt_tokens"] == 2010
+            # litellm also normalizes cache counts into the OpenAI-style nested breakdown
+            assert usage["prompt_tokens_details"]["cached_tokens"] == 800
+            assert usage["prompt_tokens_details"]["cache_creation_tokens"] == 1200
 
     def test_tool_calls_parsed(self):
         tc = MagicMock()
@@ -258,7 +280,7 @@ class TestRun:
 
     def test_streaming_content_accumulated(self):
         gen = LiteLLMChatGenerator(model="openai/gpt-4o")
-        usage = SimpleNamespace(prompt_tokens=10, completion_tokens=5, total_tokens=15)
+        usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
         chunks = _make_mock_stream_chunks("Hi!", usage=usage)
 
         fake_litellm = types.ModuleType("litellm")
@@ -443,6 +465,27 @@ class TestAsync:
             assert len(result["replies"]) == 1
             assert isinstance(result["replies"][0], ChatMessage)
 
+    @pytest.mark.asyncio
+    async def test_run_async_with_generation_kwargs(self):
+        gen = LiteLLMChatGenerator(
+            model="openai/gpt-4o", generation_kwargs={"temperature": 0.5, "max_completion_tokens": 100}
+        )
+        mock_resp = _make_mock_response()
+
+        fake_litellm = types.ModuleType("litellm")
+        captured_kwargs = {}
+
+        async def _acompletion(**kwargs):
+            captured_kwargs.update(kwargs)
+            return mock_resp
+
+        fake_litellm.acompletion = _acompletion
+
+        with mock.patch.dict(sys.modules, {"litellm": fake_litellm}):
+            await gen.run_async(messages=[ChatMessage.from_user("hi")], generation_kwargs={"temperature": 0.9})
+            assert captured_kwargs["temperature"] == 0.9
+            assert captured_kwargs["max_completion_tokens"] == 100
+
     @pytest.mark.asyncio
     async def test_run_async_empty_messages(self):
         gen = LiteLLMChatGenerator(model="openai/gpt-4o")
@@ -452,7 +495,7 @@ class TestAsync:
     @pytest.mark.asyncio
     async def test_run_async_streaming(self):
         gen = LiteLLMChatGenerator(model="openai/gpt-4o")
-        usage = SimpleNamespace(prompt_tokens=10, completion_tokens=5, total_tokens=15)
+        usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
         chunks = _make_mock_stream_chunks("Hi!", usage=usage)
 
         async def _aiter():