livekit-plugins-anthropic 1.0.22__py3-none-any.whl → 1.1.0__py3-none-any.whl

livekit/plugins/anthropic/llm.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 import os
 from collections.abc import Awaitable
 from dataclasses import dataclass
-from typing import Any, Literal
+from typing import Any, Literal, cast
 
 import httpx
 
@@ -25,7 +25,7 @@ import anthropic
 from livekit.agents import APIConnectionError, APIStatusError, APITimeoutError, llm
 from livekit.agents.llm import ToolChoice
 from livekit.agents.llm.chat_context import ChatContext
-from livekit.agents.llm.tool_context import FunctionTool
+from livekit.agents.llm.tool_context import FunctionTool, RawFunctionTool
 from livekit.agents.types import (
     DEFAULT_API_CONNECT_OPTIONS,
     NOT_GIVEN,
@@ -35,7 +35,7 @@ from livekit.agents.types import (
 from livekit.agents.utils import is_given
 
 from .models import ChatModels
-from .utils import to_chat_ctx, to_fnc_ctx
+from .utils import CACHE_CONTROL_EPHEMERAL, to_fnc_ctx
 
 
 @dataclass
@@ -118,7 +118,7 @@ class LLM(llm.LLM):
         self,
         *,
         chat_ctx: ChatContext,
-        tools: list[FunctionTool] | None = None,
+        tools: list[FunctionTool | RawFunctionTool] | None = None,
         conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
         parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
         tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
@@ -141,8 +141,10 @@ class LLM(llm.LLM):
         extra["max_tokens"] = self._opts.max_tokens if is_given(self._opts.max_tokens) else 1024
 
         if tools:
-            extra["tools"] = to_fnc_ctx(tools, self._opts.caching)
-            tool_choice = tool_choice if is_given(tool_choice) else self._opts.tool_choice
+            extra["tools"] = to_fnc_ctx(tools, self._opts.caching or None)
+            tool_choice = (
+                cast(ToolChoice, tool_choice) if is_given(tool_choice) else self._opts.tool_choice
+            )
             if is_given(tool_choice):
                 anthropic_tool_choice: dict[str, Any] | None = {"type": "auto"}
                 if isinstance(tool_choice, dict) and tool_choice.get("type") == "function":
@@ -166,15 +168,38 @@ class LLM(llm.LLM):
                         anthropic_tool_choice["disable_parallel_tool_use"] = not parallel_tool_calls
                     extra["tool_choice"] = anthropic_tool_choice
 
-        anthropic_ctx, system_message = to_chat_ctx(chat_ctx, id(self), caching=self._opts.caching)
-
-        if system_message:
-            extra["system"] = [system_message]
+        anthropic_ctx, extra_data = chat_ctx.to_provider_format(format="anthropic")
+        messages = cast(list[anthropic.types.MessageParam], anthropic_ctx)
+        if extra_data.system_messages:
+            extra["system"] = [
+                anthropic.types.TextBlockParam(text=content, type="text")
+                for content in extra_data.system_messages
+            ]
+
+        # add cache control
+        if self._opts.caching == "ephemeral":
+            if extra.get("system"):
+                extra["system"][-1]["cache_control"] = CACHE_CONTROL_EPHEMERAL
+
+            seen_assistant = False
+            for msg in reversed(messages):
+                if (
+                    msg["role"] == "assistant"
+                    and (content := msg["content"])
+                    and not seen_assistant
+                ):
+                    content[-1]["cache_control"] = CACHE_CONTROL_EPHEMERAL  # type: ignore
+                    seen_assistant = True
+
+                elif msg["role"] == "user" and (content := msg["content"]) and seen_assistant:
+                    content[-1]["cache_control"] = CACHE_CONTROL_EPHEMERAL  # type: ignore
+                    break
 
         stream = self._client.messages.create(
-            messages=anthropic_ctx,
+            messages=messages,
             model=self._opts.model,
             stream=True,
+            timeout=conn_options.timeout,
             **extra,
         )
 
@@ -194,7 +219,7 @@ class LLMStream(llm.LLMStream):
         *,
         anthropic_stream: Awaitable[anthropic.AsyncStream[anthropic.types.RawMessageStreamEvent]],
         chat_ctx: llm.ChatContext,
-        tools: list[FunctionTool],
+        tools: list[FunctionTool | RawFunctionTool],
         conn_options: APIConnectOptions,
     ) -> None:
         super().__init__(llm, chat_ctx=chat_ctx, tools=tools, conn_options=conn_options)
@@ -228,18 +253,20 @@ class LLMStream(llm.LLMStream):
                         self._event_ch.send_nowait(chat_chunk)
                         retryable = False
 
+                # https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#tracking-cache-performance
+                prompt_token = (
+                    self._input_tokens + self._cache_creation_tokens + self._cache_read_tokens
+                )
                 self._event_ch.send_nowait(
                     llm.ChatChunk(
                         id=self._request_id,
                         usage=llm.CompletionUsage(
                             completion_tokens=self._output_tokens,
-                            prompt_tokens=self._input_tokens,
-                            total_tokens=self._input_tokens
-                            + self._output_tokens
-                            + self._cache_creation_tokens
-                            + self._cache_read_tokens,
-                            cache_creation_input_tokens=self._cache_creation_tokens,
-                            cache_read_input_tokens=self._cache_read_tokens,
+                            prompt_tokens=prompt_token,
+                            total_tokens=prompt_token + self._output_tokens,
+                            prompt_cached_tokens=self._cache_read_tokens,
+                            cache_creation_tokens=self._cache_creation_tokens,
+                            cache_read_tokens=self._cache_read_tokens,
                         ),
                     )
                 )

livekit/plugins/anthropic/utils.py

@@ -1,147 +1,54 @@
-import base64
-import json
-from typing import Any, Literal
+from typing import Literal, Optional, Union
 
 import anthropic
 from livekit.agents import llm
-from livekit.agents.llm import FunctionTool
-
+from livekit.agents.llm import FunctionTool, RawFunctionTool
+from livekit.agents.llm.tool_context import (
+    get_raw_function_info,
+    is_function_tool,
+    is_raw_function_tool,
+)
+
+# We can define up to 4 cache breakpoints, we will add them at:
+# - the last tool definition
+# - the last system message
+# - the last assistant message
+# - the last user message before the last assistant message
+# https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#structuring-your-prompt
 CACHE_CONTROL_EPHEMERAL = anthropic.types.CacheControlEphemeralParam(type="ephemeral")
 
-__all__ = ["to_fnc_ctx", "to_chat_ctx"]
+__all__ = ["to_fnc_ctx", "CACHE_CONTROL_EPHEMERAL"]
 
 
 def to_fnc_ctx(
-    fncs: list[FunctionTool], caching: Literal["ephemeral"] | None
+    fncs: list[Union[FunctionTool, RawFunctionTool]], caching: Optional[Literal["ephemeral"]]
 ) -> list[anthropic.types.ToolParam]:
     tools: list[anthropic.types.ToolParam] = []
-    for i, fnc in enumerate(fncs):
-        cache_ctrl = (
-            CACHE_CONTROL_EPHEMERAL if (i == len(fncs) - 1) and caching == "ephemeral" else None
-        )
-        tools.append(_build_anthropic_schema(fnc, cache_ctrl=cache_ctrl))
-
-    return tools
-
-
-def to_chat_ctx(
-    chat_ctx: llm.ChatContext,
-    cache_key: Any,
-    caching: Literal["ephemeral"] | None,
-) -> list[anthropic.types.MessageParam]:
-    messages: list[anthropic.types.MessageParam] = []
-    system_message: anthropic.types.TextBlockParam | None = None
-    current_role: str | None = None
-    content: list[anthropic.types.TextBlockParam] = []
-    for i, msg in enumerate(chat_ctx.items):
-        if msg.type == "message" and msg.role == "system":
-            for content in msg.content:
-                if content and isinstance(content, str):
-                    system_message = anthropic.types.TextBlockParam(
-                        text=content,
-                        type="text",
-                        cache_control=CACHE_CONTROL_EPHEMERAL if caching == "ephemeral" else None,
-                    )
-            continue
-
-        cache_ctrl = (
-            CACHE_CONTROL_EPHEMERAL
-            if (i == len(chat_ctx.items) - 1) and caching == "ephemeral"
-            else None
-        )
-        if msg.type == "message":
-            role = "assistant" if msg.role == "assistant" else "user"
-        elif msg.type == "function_call":
-            role = "assistant"
-        elif msg.type == "function_call_output":
-            role = "user"
-
-        if role != current_role:
-            if current_role is not None and content:
-                messages.append(anthropic.types.MessageParam(role=current_role, content=content))
-            content = []
-            current_role = role
+    for fnc in fncs:
+        tools.append(_build_anthropic_schema(fnc))
 
-        if msg.type == "message":
-            for c in msg.content:
-                if c and isinstance(c, str):
-                    content.append(
-                        anthropic.types.TextBlockParam(
-                            text=c, type="text", cache_control=cache_ctrl
-                        )
-                    )
-                elif isinstance(c, llm.ImageContent):
-                    content.append(_to_image_content(c, cache_key, cache_ctrl=cache_ctrl))
-        elif msg.type == "function_call":
-            content.append(
-                anthropic.types.ToolUseBlockParam(
-                    id=msg.call_id,
-                    type="tool_use",
-                    name=msg.name,
-                    input=json.loads(msg.arguments or "{}"),
-                    cache_control=cache_ctrl,
-                )
-            )
-        elif msg.type == "function_call_output":
-            content.append(
-                anthropic.types.ToolResultBlockParam(
-                    tool_use_id=msg.call_id,
-                    type="tool_result",
-                    content=msg.output,
-                    cache_control=cache_ctrl,
-                )
-            )
+    if tools and caching == "ephemeral":
+        tools[-1]["cache_control"] = CACHE_CONTROL_EPHEMERAL
 
-    if current_role is not None and content:
-        messages.append(anthropic.types.MessageParam(role=current_role, content=content))
-
-    # ensure the messages starts with a "user" message
-    if not messages or messages[0]["role"] != "user":
-        messages.insert(
-            0,
-            anthropic.types.MessageParam(
-                role="user",
-                content=[anthropic.types.TextBlockParam(text="(empty)", type="text")],
-            ),
-        )
-
-    return messages, system_message
-
-
-def _to_image_content(
-    image: llm.ImageContent,
-    cache_key: Any,
-    cache_ctrl: anthropic.types.CacheControlEphemeralParam | None,
-) -> anthropic.types.ImageBlockParam:
-    img = llm.utils.serialize_image(image)
-    if img.external_url:
-        return {
-            "type": "image",
-            "source": {"type": "url", "url": img.external_url},
-            "cache_control": cache_ctrl,
-        }
-    if cache_key not in image._cache:
-        image._cache[cache_key] = img.data_bytes
-    b64_data = base64.b64encode(image._cache[cache_key]).decode("utf-8")
-    return {
-        "type": "image",
-        "source": {
-            "type": "base64",
-            "data": f"data:{img.mime_type};base64,{b64_data}",
-            "media_type": img.mime_type,
-        },
-        "cache_control": cache_ctrl,
-    }
+    return tools
 
 
 def _build_anthropic_schema(
-    function_tool: FunctionTool,
-    cache_ctrl: anthropic.types.CacheControlEphemeralParam | None = None,
+    function_tool: Union[FunctionTool, RawFunctionTool],
 ) -> anthropic.types.ToolParam:
-    fnc = llm.utils.build_legacy_openai_schema(function_tool, internally_tagged=True)
-    return anthropic.types.ToolParam(
-        name=fnc["name"],
-        description=fnc["description"] or "",
-        input_schema=fnc["parameters"],
-        cache_control=cache_ctrl,
-    )
+    if is_function_tool(function_tool):
+        fnc = llm.utils.build_legacy_openai_schema(function_tool, internally_tagged=True)
+        return anthropic.types.ToolParam(
+            name=fnc["name"],
+            description=fnc["description"] or "",
+            input_schema=fnc["parameters"],
+        )
+    elif is_raw_function_tool(function_tool):
+        info = get_raw_function_info(function_tool)
+        return anthropic.types.ToolParam(
+            name=info.name,
+            description=info.raw_schema.get("description", ""),
+            input_schema=info.raw_schema.get("parameters", {}),
+        )
+    else:
+        raise ValueError("Invalid function tool")

livekit/plugins/anthropic/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.0.22"
+__version__ = "1.1.0"

{livekit_plugins_anthropic-1.0.22.dist-info → livekit_plugins_anthropic-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: livekit-plugins-anthropic
-Version: 1.0.22
+Version: 1.1.0
 Summary: Agent Framework plugin for services from Anthropic
 Project-URL: Documentation, https://docs.livekit.io
 Project-URL: Website, https://livekit.io/
@@ -19,7 +19,8 @@ Classifier: Topic :: Multimedia :: Video
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0
 Requires-Dist: anthropic>=0.41
-Requires-Dist: livekit-agents>=1.0.22
+Requires-Dist: httpx
+Requires-Dist: livekit-agents>=1.1.0
 Description-Content-Type: text/markdown
 
 # Anthropic plugin for LiveKit Agents

livekit_plugins_anthropic-1.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+livekit/plugins/anthropic/__init__.py,sha256=oZc3LY5BrjXy-hYMb0wyvXgqXRW-ikAlo5xGh2gM0Nc,1304
+livekit/plugins/anthropic/llm.py,sha256=FhzN2ualvDu48iktJIm7MFNR_i9T06QPX8USAtDBBuY,14217
+livekit/plugins/anthropic/log.py,sha256=fG1pYSY88AnT738gZrmzF9FO4l4BdGENj3VKHMQB3Yo,72
+livekit/plugins/anthropic/models.py,sha256=wyTr2nl6SL4ylN6s4mHJcqtmgV2mjJysZo89FknWdhI,213
+livekit/plugins/anthropic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/anthropic/utils.py,sha256=CojebB7CWjX6JyObIrM8eRLgli07NGnp84mGDPrLQ1s,1907
+livekit/plugins/anthropic/version.py,sha256=7SjyflIFTjH0djSotKGIRoRykPCqMpVYetIlvHMFuh0,600
+livekit_plugins_anthropic-1.1.0.dist-info/METADATA,sha256=u9CAQed356V3xbZxbjRKH-02tAq2UZEMhfHjfr-0HY0,1452
+livekit_plugins_anthropic-1.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+livekit_plugins_anthropic-1.1.0.dist-info/RECORD,,

livekit_plugins_anthropic-1.0.22.dist-info/RECORD

@@ -1,10 +0,0 @@
-livekit/plugins/anthropic/__init__.py,sha256=oZc3LY5BrjXy-hYMb0wyvXgqXRW-ikAlo5xGh2gM0Nc,1304
-livekit/plugins/anthropic/llm.py,sha256=Q0U5ZIufGtR5abMdfMkZrANz7Ri4RxZLQYocDZOhK1Y,12884
-livekit/plugins/anthropic/log.py,sha256=fG1pYSY88AnT738gZrmzF9FO4l4BdGENj3VKHMQB3Yo,72
-livekit/plugins/anthropic/models.py,sha256=wyTr2nl6SL4ylN6s4mHJcqtmgV2mjJysZo89FknWdhI,213
-livekit/plugins/anthropic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/anthropic/utils.py,sha256=Nfl9dGCZGDEJAHj_f-TmePr8bKJrc8IwM6Houjev4DE,5158
-livekit/plugins/anthropic/version.py,sha256=-8dkOE2vDSF9WN8VoBrSwU2sb5YBGFuwPnSQXQ-uaYM,601
-livekit_plugins_anthropic-1.0.22.dist-info/METADATA,sha256=k3W_EpA4q1RQSlsDN-3CSa-VcqJ45upY7eV5jhI8xgY,1433
-livekit_plugins_anthropic-1.0.22.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-livekit_plugins_anthropic-1.0.22.dist-info/RECORD,,