livekit-plugins-anthropic 0.2.10__py3-none-any.whl → 0.2.11__py3-none-any.whl

livekit/plugins/anthropic/llm.py

@@ -53,6 +53,8 @@ from .models import (
     ChatModels,
 )
 
+CACHE_CONTROL_EPHEMERAL = anthropic.types.CacheControlEphemeralParam(type="ephemeral")
+
 
 @dataclass
 class LLMOptions:
@@ -61,6 +63,8 @@ class LLMOptions:
     temperature: float | None
     parallel_tool_calls: bool | None
     tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] | None
+    caching: Literal["ephemeral"] | None = None
+    """If set to "ephemeral", the system prompt, tools, and chat history will be cached."""
 
 
 class LLM(llm.LLM):
@@ -75,12 +79,23 @@ class LLM(llm.LLM):
         temperature: float | None = None,
         parallel_tool_calls: bool | None = None,
         tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = "auto",
+        caching: Literal["ephemeral"] | None = None,
     ) -> None:
         """
         Create a new instance of Anthropic LLM.
 
         ``api_key`` must be set to your Anthropic API key, either using the argument or by setting
         the ``ANTHROPIC_API_KEY`` environmental variable.
+
+        model (str | ChatModels): The model to use. Defaults to "claude-3-5-sonnet-20241022".
+        api_key (str | None): The Anthropic API key. Defaults to the ANTHROPIC_API_KEY environment variable.
+        base_url (str | None): The base URL for the Anthropic API. Defaults to None.
+        user (str | None): The user for the Anthropic API. Defaults to None.
+        client (anthropic.AsyncClient | None): The Anthropic client to use. Defaults to None.
+        temperature (float | None): The temperature for the Anthropic API. Defaults to None.
+        parallel_tool_calls (bool | None): Whether to parallelize tool calls. Defaults to None.
+        tool_choice (Union[ToolChoice, Literal["auto", "required", "none"]] | None): The tool choice for the Anthropic API. Defaults to "auto".
+        caching (Literal["ephemeral"] | None): If set to "ephemeral", caching will be enabled for the system prompt, tools, and chat history.
         """
 
         super().__init__(
@@ -101,6 +116,7 @@ class LLM(llm.LLM):
             temperature=temperature,
             parallel_tool_calls=parallel_tool_calls,
             tool_choice=tool_choice,
+            caching=caching,
         )
         self._client = client or anthropic.AsyncClient(
             api_key=api_key,
@@ -138,8 +154,20 @@ class LLM(llm.LLM):
         opts: dict[str, Any] = dict()
         if fnc_ctx and len(fnc_ctx.ai_functions) > 0:
             fncs_desc: list[anthropic.types.ToolParam] = []
-            for fnc in fnc_ctx.ai_functions.values():
-                fncs_desc.append(_build_function_description(fnc))
+            for i, fnc in enumerate(fnc_ctx.ai_functions.values()):
+                # caching last tool will cache all the tools if caching is enabled
+                cache_ctrl = (
+                    CACHE_CONTROL_EPHEMERAL
+                    if (i == len(fnc_ctx.ai_functions) - 1)
+                    and self._opts.caching == "ephemeral"
+                    else None
+                )
+                fncs_desc.append(
+                    _build_function_description(
+                        fnc,
+                        cache_ctrl=cache_ctrl,
+                    )
+                )
 
             opts["tools"] = fncs_desc
             if tool_choice is not None:
@@ -157,13 +185,19 @@ class LLM(llm.LLM):
                 anthropic_tool_choice["disable_parallel_tool_use"] = True
             opts["tool_choice"] = anthropic_tool_choice
 
-        latest_system_message = _latest_system_message(chat_ctx)
-        anthropic_ctx = _build_anthropic_context(chat_ctx.messages, id(self))
+        latest_system_message: anthropic.types.TextBlockParam = _latest_system_message(
+            chat_ctx, caching=self._opts.caching
+        )
+        anthropic_ctx = _build_anthropic_context(
+            chat_ctx.messages,
+            id(self),
+            caching=self._opts.caching,
+        )
         collaped_anthropic_ctx = _merge_messages(anthropic_ctx)
 
         stream = self._client.messages.create(
             max_tokens=opts.get("max_tokens", 1024),
-            system=latest_system_message,
+            system=[latest_system_message],
             messages=collaped_anthropic_ctx,
             model=self._opts.model,
             temperature=temperature or anthropic.NOT_GIVEN,
@@ -209,6 +243,8 @@ class LLMStream(llm.LLMStream):
         self._request_id: str = ""
         self._ignoring_cot = False  # ignore chain of thought
         self._input_tokens = 0
+        self._cache_creation_tokens = 0
+        self._cache_read_tokens = 0
         self._output_tokens = 0
 
     async def _run(self) -> None:
@@ -230,7 +266,12 @@ class LLMStream(llm.LLMStream):
                         usage=llm.CompletionUsage(
                             completion_tokens=self._output_tokens,
                             prompt_tokens=self._input_tokens,
-                            total_tokens=self._input_tokens + self._output_tokens,
+                            total_tokens=self._input_tokens
+                            + self._output_tokens
+                            + self._cache_creation_tokens
+                            + self._cache_read_tokens,
+                            cache_creation_input_tokens=self._cache_creation_tokens,
+                            cache_read_input_tokens=self._cache_read_tokens,
                         ),
                     )
                 )
@@ -253,6 +294,12 @@ class LLMStream(llm.LLMStream):
             self._request_id = event.message.id
             self._input_tokens = event.message.usage.input_tokens
             self._output_tokens = event.message.usage.output_tokens
+            if event.message.usage.cache_creation_input_tokens:
+                self._cache_creation_tokens = (
+                    event.message.usage.cache_creation_input_tokens
+                )
+            if event.message.usage.cache_read_input_tokens:
+                self._cache_read_tokens = event.message.usage.cache_read_input_tokens
         elif event.type == "message_delta":
             self._output_tokens += event.usage.output_tokens
         elif event.type == "content_block_start":
@@ -317,7 +364,9 @@ class LLMStream(llm.LLMStream):
         return None
 
 
-def _latest_system_message(chat_ctx: llm.ChatContext) -> str:
+def _latest_system_message(
+    chat_ctx: llm.ChatContext, caching: Literal["ephemeral"] | None = None
+) -> anthropic.types.TextBlockParam:
     latest_system_message: llm.ChatMessage | None = None
     for m in chat_ctx.messages:
         if m.role == "system":
@@ -332,7 +381,12 @@ def _latest_system_message(chat_ctx: llm.ChatContext) -> str:
             latest_system_str = " ".join(
                 [c for c in latest_system_message.content if isinstance(c, str)]
             )
-    return latest_system_str
+    system_text_block = anthropic.types.TextBlockParam(
+        text=latest_system_str,
+        type="text",
+        cache_control=CACHE_CONTROL_EPHEMERAL if caching == "ephemeral" else None,
+    )
+    return system_text_block
 
 
 def _merge_messages(
@@ -362,18 +416,29 @@ def _merge_messages(
 
 
 def _build_anthropic_context(
-    chat_ctx: List[llm.ChatMessage], cache_key: Any
+    chat_ctx: List[llm.ChatMessage],
+    cache_key: Any,
+    caching: Literal["ephemeral"] | None,
 ) -> List[anthropic.types.MessageParam]:
     result: List[anthropic.types.MessageParam] = []
-    for msg in chat_ctx:
-        a_msg = _build_anthropic_message(msg, cache_key, chat_ctx)
+    for i, msg in enumerate(chat_ctx):
+        # caching last message will cache whole chat history if caching is enabled
+        cache_ctrl = (
+            CACHE_CONTROL_EPHEMERAL
+            if ((i == len(chat_ctx) - 1) and caching == "ephemeral")
+            else None
+        )
+        a_msg = _build_anthropic_message(msg, cache_key, cache_ctrl=cache_ctrl)
+
         if a_msg:
             result.append(a_msg)
     return result
 
 
 def _build_anthropic_message(
-    msg: llm.ChatMessage, cache_key: Any, chat_ctx: List[llm.ChatMessage]
+    msg: llm.ChatMessage,
+    cache_key: Any,
+    cache_ctrl: anthropic.types.CacheControlEphemeralParam | None,
 ) -> anthropic.types.MessageParam | None:
     if msg.role == "user" or msg.role == "assistant":
         a_msg: anthropic.types.MessageParam = {
@@ -386,22 +451,27 @@ def _build_anthropic_message(
         # add content if provided
         if isinstance(msg.content, str) and msg.content:
             a_msg["content"].append(
-                anthropic.types.TextBlock(
+                anthropic.types.TextBlockParam(
                     text=msg.content,
                     type="text",
+                    cache_control=cache_ctrl,
                 )
             )
         elif isinstance(msg.content, list):
             for cnt in msg.content:
                 if isinstance(cnt, str) and cnt:
-                    content: anthropic.types.TextBlock = anthropic.types.TextBlock(
-                        text=cnt,
-                        type="text",
+                    content: anthropic.types.TextBlockParam = (
+                        anthropic.types.TextBlockParam(
+                            text=cnt,
+                            type="text",
+                            cache_control=cache_ctrl,
+                        )
                     )
                     a_content.append(content)
                 elif isinstance(cnt, llm.ChatImage):
-                    a_content.append(_build_anthropic_image_content(cnt, cache_key))
-
+                    a_content.append(
+                        _build_anthropic_image_content(cnt, cache_key, cache_ctrl)
+                    )
         if msg.tool_calls is not None:
             for fnc in msg.tool_calls:
                 tool_use = anthropic.types.ToolUseBlockParam(
@@ -409,6 +479,7 @@ def _build_anthropic_message(
                     type="tool_use",
                     name=fnc.function_info.name,
                     input=fnc.arguments,
+                    cache_control=cache_ctrl,
                 )
                 a_content.append(tool_use)
 
@@ -427,6 +498,7 @@ def _build_anthropic_message(
             type="tool_result",
             content=msg.content,
             is_error=msg.tool_exception is not None,
+            cache_control=cache_ctrl,
         )
         return {
             "role": "user",
@@ -437,7 +509,9 @@ def _build_anthropic_message(
 
 
 def _build_anthropic_image_content(
-    image: llm.ChatImage, cache_key: Any
+    image: llm.ChatImage,
+    cache_key: Any,
+    cache_ctrl: anthropic.types.CacheControlEphemeralParam | None,
 ) -> anthropic.types.ImageBlockParam:
     if isinstance(image.image, str):  # image is a URL
         if not image.image.startswith("data:"):
@@ -463,6 +537,7 @@ def _build_anthropic_image_content(
                         media_type,
                     ),
                 },
+                "cache_control": cache_ctrl,
             }
         except (ValueError, IndexError) as e:
             raise ValueError(
@@ -490,6 +565,7 @@ def _build_anthropic_image_content(
                 "data": image._cache[cache_key],
                 "media_type": "image/jpeg",
             },
+            "cache_control": cache_ctrl,
         }
 
     raise ValueError(
@@ -499,6 +575,7 @@ def _build_anthropic_image_content(
 
 def _build_function_description(
     fnc_info: llm.function_context.FunctionInfo,
+    cache_ctrl: anthropic.types.CacheControlEphemeralParam | None,
 ) -> anthropic.types.ToolParam:
     def build_schema_field(arg_info: llm.function_context.FunctionArgInfo):
         def type2str(t: type) -> str:
@@ -520,7 +597,7 @@ def _build_function_description(
         if arg_info.description:
             p["description"] = arg_info.description
 
-        is_optional, inner_th = _is_optional_type(arg_info.type)
+        _, inner_th = _is_optional_type(arg_info.type)
 
         if get_origin(inner_th) is list:
             inner_type = get_args(inner_th)[0]
@@ -542,8 +619,9 @@ def _build_function_description(
     for arg_info in fnc_info.arguments.values():
         input_schema[arg_info.name] = build_schema_field(arg_info)
 
-    return {
-        "name": fnc_info.name,
-        "description": fnc_info.description,
-        "input_schema": input_schema,
-    }
+    return anthropic.types.ToolParam(
+        name=fnc_info.name,
+        description=fnc_info.description,
+        input_schema=input_schema,
+        cache_control=cache_ctrl,
+    )

livekit/plugins/anthropic/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.2.10"
+__version__ = "0.2.11"

{livekit_plugins_anthropic-0.2.10.dist-info → livekit_plugins_anthropic-0.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: livekit-plugins-anthropic
-Version: 0.2.10
+Version: 0.2.11
 Summary: Agent Framework plugin for services from Anthropic
 Home-page: https://github.com/livekit/agents
 License: Apache-2.0

livekit_plugins_anthropic-0.2.11.dist-info/RECORD

@@ -0,0 +1,10 @@
+livekit/plugins/anthropic/__init__.py,sha256=1WCyNEaR6qBsX54qJQM0SeY-QHIucww16PLXcSnMqRo,1175
+livekit/plugins/anthropic/llm.py,sha256=JgY7x1DlngrQNsXss_Vc0VMNge4jza3D3-tde7WtNwU,23071
+livekit/plugins/anthropic/log.py,sha256=fG1pYSY88AnT738gZrmzF9FO4l4BdGENj3VKHMQB3Yo,72
+livekit/plugins/anthropic/models.py,sha256=wyTr2nl6SL4ylN6s4mHJcqtmgV2mjJysZo89FknWdhI,213
+livekit/plugins/anthropic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/anthropic/version.py,sha256=tQf5iSQsEO0Vr1TimFhAqVDGkoj_Na4Qfj_agnoO3NQ,601
+livekit_plugins_anthropic-0.2.11.dist-info/METADATA,sha256=IY-JVBfw4xdAXlZZfIYqcOqBrBKAPGDUBP34cItva60,1481
+livekit_plugins_anthropic-0.2.11.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+livekit_plugins_anthropic-0.2.11.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
+livekit_plugins_anthropic-0.2.11.dist-info/RECORD,,

livekit_plugins_anthropic-0.2.10.dist-info/RECORD

@@ -1,10 +0,0 @@
-livekit/plugins/anthropic/__init__.py,sha256=1WCyNEaR6qBsX54qJQM0SeY-QHIucww16PLXcSnMqRo,1175
-livekit/plugins/anthropic/llm.py,sha256=xRayMqq_FXFGZwskpqcINKVCbXL-754Jh2CJMWvxJnY,19261
-livekit/plugins/anthropic/log.py,sha256=fG1pYSY88AnT738gZrmzF9FO4l4BdGENj3VKHMQB3Yo,72
-livekit/plugins/anthropic/models.py,sha256=wyTr2nl6SL4ylN6s4mHJcqtmgV2mjJysZo89FknWdhI,213
-livekit/plugins/anthropic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/anthropic/version.py,sha256=vT0ONEJ_8wtOHcTDpZ96O0mQtrdNrO2FOuqDfAOF_bI,601
-livekit_plugins_anthropic-0.2.10.dist-info/METADATA,sha256=8DF-B_YzfayulYZMpOB2_smktfH8s9gKXXXeccz5J4U,1481
-livekit_plugins_anthropic-0.2.10.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-livekit_plugins_anthropic-0.2.10.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
-livekit_plugins_anthropic-0.2.10.dist-info/RECORD,,