livekit-plugins-anthropic 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

livekit/plugins/anthropic/__init__.py

@@ -35,3 +35,12 @@ class AnthropicPlugin(Plugin):
 
 
 Plugin.register_plugin(AnthropicPlugin())
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit/plugins/anthropic/llm.py

@@ -23,7 +23,13 @@ from typing import Any, Awaitable, List, Tuple, get_args, get_origin
 
 import httpx
 from livekit import rtc
-from livekit.agents import llm, utils
+from livekit.agents import (
+    APIConnectionError,
+    APIStatusError,
+    APITimeoutError,
+    llm,
+    utils,
+)
 
 import anthropic
 
@@ -37,6 +43,7 @@ from .models import (
 class LLMOptions:
     model: str | ChatModels
     user: str | None
+    temperature: float | None
 
 
 class LLM(llm.LLM):
@@ -48,6 +55,7 @@ class LLM(llm.LLM):
         base_url: str | None = None,
         user: str | None = None,
         client: anthropic.AsyncClient | None = None,
+        temperature: float | None = None,
     ) -> None:
         """
         Create a new instance of Anthropic LLM.
@@ -55,13 +63,14 @@ class LLM(llm.LLM):
         ``api_key`` must be set to your Anthropic API key, either using the argument or by setting
         the ``ANTHROPIC_API_KEY`` environmental variable.
         """
+        super().__init__()
 
         # throw an error on our end
         api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
         if api_key is None:
             raise ValueError("Anthropic API key is required")
 
-        self._opts = LLMOptions(model=model, user=user)
+        self._opts = LLMOptions(model=model, user=user, temperature=temperature)
         self._client = client or anthropic.AsyncClient(
             api_key=api_key,
             base_url=base_url,
@@ -85,6 +94,9 @@ class LLM(llm.LLM):
         n: int | None = 1,
         parallel_tool_calls: bool | None = None,
     ) -> "LLMStream":
+        if temperature is None:
+            temperature = self._opts.temperature
+
         opts: dict[str, Any] = dict()
         if fnc_ctx and len(fnc_ctx.ai_functions) > 0:
             fncs_desc: list[anthropic.types.ToolParam] = []
@@ -99,8 +111,9 @@ class LLM(llm.LLM):
         latest_system_message = _latest_system_message(chat_ctx)
         anthropic_ctx = _build_anthropic_context(chat_ctx.messages, id(self))
         collaped_anthropic_ctx = _merge_messages(anthropic_ctx)
+
         stream = self._client.messages.create(
-            max_tokens=opts.get("max_tokens", 1000),
+            max_tokens=opts.get("max_tokens", 1024),
             system=latest_system_message,
             messages=collaped_anthropic_ctx,
             model=self._opts.model,
@@ -110,12 +123,15 @@ class LLM(llm.LLM):
             **opts,
         )
 
-        return LLMStream(anthropic_stream=stream, chat_ctx=chat_ctx, fnc_ctx=fnc_ctx)
+        return LLMStream(
+            self, anthropic_stream=stream, chat_ctx=chat_ctx, fnc_ctx=fnc_ctx
+        )
 
 
 class LLMStream(llm.LLMStream):
     def __init__(
         self,
+        llm: LLM,
         *,
         anthropic_stream: Awaitable[
             anthropic.AsyncStream[anthropic.types.RawMessageStreamEvent]
@@ -123,7 +139,7 @@ class LLMStream(llm.LLMStream):
         chat_ctx: llm.ChatContext,
         fnc_ctx: llm.FunctionContext | None,
     ) -> None:
-        super().__init__(chat_ctx=chat_ctx, fnc_ctx=fnc_ctx)
+        super().__init__(llm, chat_ctx=chat_ctx, fnc_ctx=fnc_ctx)
         self._awaitable_anthropic_stream = anthropic_stream
         self._anthropic_stream: (
             anthropic.AsyncStream[anthropic.types.RawMessageStreamEvent] | None
@@ -134,89 +150,113 @@ class LLMStream(llm.LLMStream):
         self._fnc_name: str | None = None
         self._fnc_raw_arguments: str | None = None
 
-    async def aclose(self) -> None:
-        if self._anthropic_stream:
-            await self._anthropic_stream.close()
-
-        return await super().aclose()
-
-    async def __anext__(self):
-        if not self._anthropic_stream:
-            self._anthropic_stream = await self._awaitable_anthropic_stream
-
-        fn_calling_enabled = self._fnc_ctx is not None
-        ignore = False
-
-        async for event in self._anthropic_stream:
-            if event.type == "message_start":
-                pass
-            elif event.type == "message_delta":
-                pass
-            elif event.type == "message_stop":
-                pass
-            elif event.type == "content_block_start":
-                if event.content_block.type == "tool_use":
-                    self._tool_call_id = event.content_block.id
-                    self._fnc_raw_arguments = ""
-                    self._fnc_name = event.content_block.name
-            elif event.type == "content_block_delta":
-                delta = event.delta
-                if delta.type == "text_delta":
-                    text = delta.text
-
-                    # Anthropic seems to add a prompt when tool calling is enabled
-                    # where responses always start with a "<thinking>" block containing
-                    # the LLM's chain of thought. It's very verbose and not useful for voice
-                    # applications.
-                    if fn_calling_enabled:
-                        if text.startswith("<thinking>"):
-                            ignore = True
-
-                        if "</thinking>" in text:
-                            text = text.split("</thinking>")[-1]
-                            ignore = False
-
-                    if ignore:
-                        continue
-
-                    return llm.ChatChunk(
-                        choices=[
-                            llm.Choice(
-                                delta=llm.ChoiceDelta(content=text, role="assistant")
-                            )
-                        ]
-                    )
-                elif delta.type == "input_json_delta":
-                    assert self._fnc_raw_arguments is not None
-                    self._fnc_raw_arguments += delta.partial_json
-
-            elif event.type == "content_block_stop":
-                if self._tool_call_id is not None and self._fnc_ctx:
-                    assert self._fnc_name is not None
-                    assert self._fnc_raw_arguments is not None
-                    fnc_info = _create_ai_function_info(
-                        self._fnc_ctx,
-                        self._tool_call_id,
-                        self._fnc_name,
-                        self._fnc_raw_arguments,
+        self._request_id: str = ""
+        self._ignoring_cot = False  # ignore chain of thought
+        self._input_tokens = 0
+        self._output_tokens = 0
+
+    async def _main_task(self) -> None:
+        try:
+            if not self._anthropic_stream:
+                self._anthropic_stream = await self._awaitable_anthropic_stream
+
+            async with self._anthropic_stream as stream:
+                async for event in stream:
+                    chat_chunk = self._parse_event(event)
+                    if chat_chunk is not None:
+                        self._event_ch.send_nowait(chat_chunk)
+
+                self._event_ch.send_nowait(
+                    llm.ChatChunk(
+                        request_id=self._request_id,
+                        usage=llm.CompletionUsage(
+                            completion_tokens=self._output_tokens,
+                            prompt_tokens=self._input_tokens,
+                            total_tokens=self._input_tokens + self._output_tokens,
+                        ),
                     )
-                    self._function_calls_info.append(fnc_info)
-                    chunk = llm.ChatChunk(
-                        choices=[
-                            llm.Choice(
-                                delta=llm.ChoiceDelta(
-                                    role="assistant", tool_calls=[fnc_info]
-                                ),
-                                index=0,
-                            )
-                        ]
-                    )
-                    self._tool_call_id = None
-                    self._fnc_raw_arguments = None
-                    self._fnc_name = None
-                    return chunk
+                )
+        except anthropic.APITimeoutError:
+            raise APITimeoutError()
+        except anthropic.APIStatusError as e:
+            raise APIStatusError(
+                e.message,
+                status_code=e.status_code,
+                request_id=e.request_id,
+                body=e.body,
+            )
+        except Exception as e:
+            raise APIConnectionError() from e
+
+    def _parse_event(
+        self, event: anthropic.types.RawMessageStreamEvent
+    ) -> llm.ChatChunk | None:
+        if event.type == "message_start":
+            self._request_id = event.message.id
+            self._input_tokens = event.message.usage.input_tokens
+            self._output_tokens = event.message.usage.output_tokens
+        elif event.type == "message_delta":
+            self._output_tokens += event.usage.output_tokens
+        elif event.type == "content_block_start":
+            if event.content_block.type == "tool_use":
+                self._tool_call_id = event.content_block.id
+                self._fnc_name = event.content_block.name
+                self._fnc_raw_arguments = ""
+        elif event.type == "content_block_delta":
+            delta = event.delta
+            if delta.type == "text_delta":
+                text = delta.text
+
+                if self._fnc_ctx is not None:
+                    # anthropic may inject COC when using functions
+                    if text.startswith("<thinking>"):
+                        self._ignoring_cot = True
+                    elif self._ignoring_cot and "</thinking>" in text:
+                        text = text.split("</thinking>")[-1]
+                        self._ignoring_cot = False
+
+                if self._ignoring_cot:
+                    return None
+
+                return llm.ChatChunk(
+                    request_id=self._request_id,
+                    choices=[
+                        llm.Choice(
+                            delta=llm.ChoiceDelta(content=text, role="assistant")
+                        )
+                    ],
+                )
+            elif delta.type == "input_json_delta":
+                assert self._fnc_raw_arguments is not None
+                self._fnc_raw_arguments += delta.partial_json
+
+        elif event.type == "content_block_stop":
+            if self._tool_call_id is not None and self._fnc_ctx:
+                assert self._fnc_name is not None
+                assert self._fnc_raw_arguments is not None
+
+                fnc_info = _create_ai_function_info(
+                    self._fnc_ctx,
+                    self._tool_call_id,
+                    self._fnc_name,
+                    self._fnc_raw_arguments,
+                )
+                self._function_calls_info.append(fnc_info)
+
+                chat_chunk = llm.ChatChunk(
+                    request_id=self._request_id,
+                    choices=[
+                        llm.Choice(
+                            delta=llm.ChoiceDelta(
+                                role="assistant", tool_calls=[fnc_info]
+                            ),
+                        )
+                    ],
+                )
+                self._tool_call_id = self._fnc_raw_arguments = self._fnc_name = None
+                return chat_chunk
 
-        raise StopAsyncIteration
+        return None
 
 
 def _latest_system_message(chat_ctx: llm.ChatContext) -> str:
@@ -286,7 +326,7 @@ def _build_anthropic_message(
         a_content = a_msg["content"]
 
         # add content if provided
-        if isinstance(msg.content, str):
+        if isinstance(msg.content, str) and msg.content:
             a_msg["content"].append(
                 anthropic.types.TextBlock(
                     text=msg.content,
@@ -295,7 +335,7 @@ def _build_anthropic_message(
             )
         elif isinstance(msg.content, list):
             for cnt in msg.content:
-                if isinstance(cnt, str):
+                if isinstance(cnt, str) and cnt:
                     content: anthropic.types.TextBlock = anthropic.types.TextBlock(
                         text=cnt,
                         type="text",

livekit/plugins/anthropic/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.2.1"
+__version__ = "0.2.3"

{livekit_plugins_anthropic-0.2.1.dist-info → livekit_plugins_anthropic-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: livekit-plugins-anthropic
-Version: 0.2.1
+Version: 0.2.3
 Summary: Agent Framework plugin for services from Anthropic
 Home-page: https://github.com/livekit/agents
 License: Apache-2.0
@@ -19,8 +19,8 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
-Requires-Dist: livekit-agents ~=0.8
-Requires-Dist: anthropic ~=0.34
+Requires-Dist: livekit-agents >=0.11
+Requires-Dist: anthropic >=0.34
 
 # LiveKit Plugins Anthropic
 

livekit_plugins_anthropic-0.2.3.dist-info/RECORD

@@ -0,0 +1,10 @@
+livekit/plugins/anthropic/__init__.py,sha256=1WCyNEaR6qBsX54qJQM0SeY-QHIucww16PLXcSnMqRo,1175
+livekit/plugins/anthropic/llm.py,sha256=6rNtLJOfvB72gmmkFk8qN-N8fKnm-vhflpiBKTU_gqM,18921
+livekit/plugins/anthropic/log.py,sha256=fG1pYSY88AnT738gZrmzF9FO4l4BdGENj3VKHMQB3Yo,72
+livekit/plugins/anthropic/models.py,sha256=AVEhrEtKfWxsd-R03u7R74hcKjJq4oDVSTukvoPQGb0,179
+livekit/plugins/anthropic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/anthropic/version.py,sha256=A5eBQ2sWvNCBt2aefaSvtvK0XRwtGa-rgF4340O8cjk,600
+livekit_plugins_anthropic-0.2.3.dist-info/METADATA,sha256=dttEt-3lS_LDVl0XZkgnINXr560oPngIyXyzeJvih6c,1265
+livekit_plugins_anthropic-0.2.3.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+livekit_plugins_anthropic-0.2.3.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
+livekit_plugins_anthropic-0.2.3.dist-info/RECORD,,

{livekit_plugins_anthropic-0.2.1.dist-info → livekit_plugins_anthropic-0.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

livekit_plugins_anthropic-0.2.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-livekit/plugins/anthropic/__init__.py,sha256=g6KUqOfZo9DIBwBD98u6QOWY7pr8ZYJJ61fk3AWpoa4,1006
-livekit/plugins/anthropic/llm.py,sha256=s6Xt_5kuvypG1ZLUq_aYaYO_ZaTgagyM4b1c5iI2WUo,17652
-livekit/plugins/anthropic/log.py,sha256=fG1pYSY88AnT738gZrmzF9FO4l4BdGENj3VKHMQB3Yo,72
-livekit/plugins/anthropic/models.py,sha256=AVEhrEtKfWxsd-R03u7R74hcKjJq4oDVSTukvoPQGb0,179
-livekit/plugins/anthropic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/anthropic/version.py,sha256=tb-eucPKT0bz3vQDaI1ltUZEhGKRFOFhN3U5ZIv_-xY,600
-livekit_plugins_anthropic-0.2.1.dist-info/METADATA,sha256=-m1Q9gn3H1gRoVjFb1uIDfyJ5TP6p1HEMrh-zDyGeQs,1264
-livekit_plugins_anthropic-0.2.1.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-livekit_plugins_anthropic-0.2.1.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
-livekit_plugins_anthropic-0.2.1.dist-info/RECORD,,