livekit-plugins-aws 0.1.0__py3-none-any.whl

livekit/plugins/aws/__init__.py

@@ -0,0 +1,30 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .llm import LLM
+from .stt import STT, SpeechStream
+from .tts import TTS, ChunkedStream
+from .version import __version__
+
+__all__ = ["STT", "SpeechStream", "TTS", "ChunkedStream", "LLM", "__version__"]
+
+from livekit.agents import Plugin
+
+
+class AWSPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__)
+
+
+Plugin.register_plugin(AWSPlugin())

livekit/plugins/aws/_utils.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import base64
+import inspect
+import json
+import os
+from typing import Any, Dict, List, Optional, Tuple, get_args, get_origin
+
+import boto3
+from livekit import rtc
+from livekit.agents import llm, utils
+from livekit.agents.llm.function_context import _is_optional_type
+
+__all__ = ["_build_aws_ctx", "_build_tools", "_get_aws_credentials"]
+
+
+def _get_aws_credentials(
+    api_key: Optional[str], api_secret: Optional[str], region: Optional[str]
+):
+    region = region or os.environ.get("AWS_DEFAULT_REGION")
+    if not region:
+        raise ValueError(
+            "AWS_DEFAULT_REGION must be set using the argument or by setting the AWS_DEFAULT_REGION environment variable."
+        )
+
+    # If API key and secret are provided, create a session with them
+    if api_key and api_secret:
+        session = boto3.Session(
+            aws_access_key_id=api_key,
+            aws_secret_access_key=api_secret,
+            region_name=region,
+        )
+    else:
+        session = boto3.Session(region_name=region)
+
+    credentials = session.get_credentials()
+    if not credentials or not credentials.access_key or not credentials.secret_key:
+        raise ValueError("No valid AWS credentials found.")
+    return credentials.access_key, credentials.secret_key
+
+
+JSON_SCHEMA_TYPE_MAP: Dict[type, str] = {
+    str: "string",
+    int: "integer",
+    float: "number",
+    bool: "boolean",
+    dict: "object",
+    list: "array",
+}
+
+
+def _build_parameters(arguments: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    properties: Dict[str, dict] = {}
+    required: List[str] = []
+
+    for arg_name, arg_info in arguments.items():
+        prop = {}
+        if hasattr(arg_info, "description") and arg_info.description:
+            prop["description"] = arg_info.description
+
+        _, py_type = _is_optional_type(arg_info.type)
+        origin = get_origin(py_type)
+        if origin is list:
+            item_type = get_args(py_type)[0]
+            if item_type not in JSON_SCHEMA_TYPE_MAP:
+                raise ValueError(f"Unsupported type: {item_type}")
+            prop["type"] = "array"
+            prop["items"] = {"type": JSON_SCHEMA_TYPE_MAP[item_type]}
+
+            if hasattr(arg_info, "choices") and arg_info.choices:
+                prop["items"]["enum"] = list(arg_info.choices)
+        else:
+            if py_type not in JSON_SCHEMA_TYPE_MAP:
+                raise ValueError(f"Unsupported type: {py_type}")
+
+            prop["type"] = JSON_SCHEMA_TYPE_MAP[py_type]
+
+            if arg_info.choices:
+                prop["enum"] = list(arg_info.choices)
+
+        properties[arg_name] = prop
+
+        if arg_info.default is inspect.Parameter.empty:
+            required.append(arg_name)
+
+    if properties:
+        parameters = {"json": {"type": "object", "properties": properties}}
+        if required:
+            parameters["json"]["required"] = required
+
+        return parameters
+
+    return None
+
+
+def _build_tools(fnc_ctx: Any) -> List[dict]:
+    tools: List[dict] = []
+    for fnc_info in fnc_ctx.ai_functions.values():
+        parameters = _build_parameters(fnc_info.arguments)
+
+        func_decl = {
+            "toolSpec": {
+                "name": fnc_info.name,
+                "description": fnc_info.description,
+                "inputSchema": parameters
+                if parameters
+                else {"json": {"type": "object", "properties": {}}},
+            }
+        }
+
+        tools.append(func_decl)
+    return tools
+
+
+def _build_image(image: llm.ChatImage, cache_key: Any) -> dict:
+    if isinstance(image.image, str):
+        if image.image.startswith("data:image/jpeg;base64,"):
+            base64_data = image.image.split(",", 1)[1]
+            try:
+                image_bytes = base64.b64decode(base64_data)
+            except Exception as e:
+                raise ValueError("Invalid base64 data in image URL") from e
+
+            return {"image": {"format": "jpeg", "source": {"bytes": image_bytes}}}
+        else:
+            return {"image": {"format": "jpeg", "source": {"uri": image.image}}}
+
+    elif isinstance(image.image, rtc.VideoFrame):
+        if cache_key not in image._cache:
+            opts = utils.images.EncodeOptions()
+            if image.inference_width and image.inference_height:
+                opts.resize_options = utils.images.ResizeOptions(
+                    width=image.inference_width,
+                    height=image.inference_height,
+                    strategy="scale_aspect_fit",
+                )
+            image._cache[cache_key] = utils.images.encode(image.image, opts)
+
+        return {
+            "image": {
+                "format": "jpeg",
+                "source": {
+                    "bytes": image._cache[cache_key],
+                },
+            }
+        }
+    raise ValueError(f"Unsupported image type: {type(image.image)}")
+
+
+def _build_aws_ctx(
+    chat_ctx: llm.ChatContext, cache_key: Any
+) -> Tuple[List[dict], Optional[dict]]:
+    messages: List[dict] = []
+    system: Optional[dict] = None
+    current_role: Optional[str] = None
+    current_content: List[dict] = []
+
+    for msg in chat_ctx.messages:
+        if msg.role == "system":
+            if isinstance(msg.content, str):
+                system = {"text": msg.content}
+            continue
+
+        if msg.role == "assistant":
+            role = "assistant"
+        else:
+            role = "user"
+
+        if role != current_role:
+            if current_role is not None and current_content:
+                messages.append({"role": current_role, "content": current_content})
+            current_role = role
+            current_content = []
+
+        if msg.tool_calls:
+            for fnc in msg.tool_calls:
+                current_content.append(
+                    {
+                        "toolUse": {
+                            "toolUseId": fnc.tool_call_id,
+                            "name": fnc.function_info.name,
+                            "input": fnc.arguments,
+                        }
+                    }
+                )
+
+        if msg.role == "tool":
+            tool_response: dict = {
+                "toolResult": {
+                    "toolUseId": msg.tool_call_id,
+                    "content": [],
+                    "status": "success",
+                }
+            }
+            if isinstance(msg.content, dict):
+                tool_response["toolResult"]["content"].append({"json": msg.content})
+            elif isinstance(msg.content, str):
+                tool_response["toolResult"]["content"].append({"text": msg.content})
+            current_content.append(tool_response)
+        else:
+            if msg.content:
+                if isinstance(msg.content, str):
+                    current_content.append({"text": msg.content})
+                elif isinstance(msg.content, dict):
+                    current_content.append({"text": json.dumps(msg.content)})
+                elif isinstance(msg.content, list):
+                    for item in msg.content:
+                        if isinstance(item, str):
+                            current_content.append({"text": item})
+                        elif isinstance(item, llm.ChatImage):
+                            current_content.append(_build_image(item, cache_key))
+
+    if current_role is not None and current_content:
+        messages.append({"role": current_role, "content": current_content})
+
+    return messages, system

livekit/plugins/aws/llm.py

@@ -0,0 +1,350 @@
+# Copyright 2023 LiveKit, Inc.
+#
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import asyncio
+import os
+from dataclasses import dataclass
+from typing import Any, Literal, MutableSet, Union
+
+import boto3
+from livekit.agents import (
+    APIConnectionError,
+    APIStatusError,
+    llm,
+)
+from livekit.agents.llm import LLMCapabilities, ToolChoice, _create_ai_function_info
+from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS, APIConnectOptions
+
+from ._utils import _build_aws_ctx, _build_tools, _get_aws_credentials
+from .log import logger
+
+TEXT_MODEL = Literal["anthropic.claude-3-5-sonnet-20241022-v2:0"]
+DEFAULT_REGION = "us-east-1"
+
+
+@dataclass
+class LLMOptions:
+    model: TEXT_MODEL | str
+    temperature: float | None
+    tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = "auto"
+    max_output_tokens: int | None = None
+    top_p: float | None = None
+    additional_request_fields: dict[str, Any] | None = None
+
+
+class LLM(llm.LLM):
+    def __init__(
+        self,
+        *,
+        model: TEXT_MODEL | str = "anthropic.claude-3-5-sonnet-20240620-v1:0",
+        api_key: str | None = None,
+        api_secret: str | None = None,
+        region: str = "us-east-1",
+        temperature: float = 0.8,
+        max_output_tokens: int | None = None,
+        top_p: float | None = None,
+        tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = "auto",
+        additional_request_fields: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Create a new instance of AWS Bedrock LLM.
+
+        ``api_key``  and ``api_secret`` must be set to your AWS Access key id and secret access key, either using the argument or by setting the
+        ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environmental variables.
+
+        See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock-runtime/client/converse_stream.html for more details on the the AWS Bedrock Runtime API.
+
+        Args:
+            model (TEXT_MODEL, optional): model or inference profile arn to use(https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-use.html). Defaults to 'anthropic.claude-3-5-sonnet-20240620-v1:0'.
+            api_key(str, optional): AWS access key id.
+            api_secret(str, optional): AWS secret access key
+            region (str, optional): The region to use for AWS API requests. Defaults value is "us-east-1".
+            temperature (float, optional): Sampling temperature for response generation. Defaults to 0.8.
+            max_output_tokens (int, optional): Maximum number of tokens to generate in the output. Defaults to None.
+            top_p (float, optional): The nucleus sampling probability for response generation. Defaults to None.
+            tool_choice (ToolChoice or Literal["auto", "required", "none"], optional): Specifies whether to use tools during response generation. Defaults to "auto".
+            additional_request_fields (dict[str, Any], optional): Additional request fields to send to the AWS Bedrock Converse API. Defaults to None.
+        """
+        super().__init__(
+            capabilities=LLMCapabilities(
+                supports_choices_on_int=True,
+                requires_persistent_functions=True,
+            )
+        )
+        self._api_key, self._api_secret = _get_aws_credentials(
+            api_key, api_secret, region
+        )
+
+        self._model = model or os.environ.get("BEDROCK_INFERENCE_PROFILE_ARN")
+        if not self._model:
+            raise ValueError(
+                "model or inference profile arn must be set using the argument or by setting the BEDROCK_INFERENCE_PROFILE_ARN environment variable."
+            )
+        self._opts = LLMOptions(
+            model=self._model,
+            temperature=temperature,
+            tool_choice=tool_choice,
+            max_output_tokens=max_output_tokens,
+            top_p=top_p,
+            additional_request_fields=additional_request_fields,
+        )
+        self._region = region
+        self._running_fncs: MutableSet[asyncio.Task[Any]] = set()
+
+    def chat(
+        self,
+        *,
+        chat_ctx: llm.ChatContext,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+        fnc_ctx: llm.FunctionContext | None = None,
+        temperature: float | None = None,
+        n: int | None = 1,
+        parallel_tool_calls: bool | None = None,
+        tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]]
+        | None = None,
+    ) -> "LLMStream":
+        if tool_choice is None:
+            tool_choice = self._opts.tool_choice
+
+        if temperature is None:
+            temperature = self._opts.temperature
+
+        return LLMStream(
+            self,
+            model=self._opts.model,
+            aws_access_key_id=self._api_key,
+            aws_secret_access_key=self._api_secret,
+            region_name=self._region,
+            max_output_tokens=self._opts.max_output_tokens,
+            top_p=self._opts.top_p,
+            additional_request_fields=self._opts.additional_request_fields,
+            chat_ctx=chat_ctx,
+            fnc_ctx=fnc_ctx,
+            conn_options=conn_options,
+            temperature=temperature,
+            tool_choice=tool_choice,
+        )
+
+
+class LLMStream(llm.LLMStream):
+    def __init__(
+        self,
+        llm: LLM,
+        *,
+        model: str | TEXT_MODEL,
+        aws_access_key_id: str | None,
+        aws_secret_access_key: str | None,
+        region_name: str,
+        chat_ctx: llm.ChatContext,
+        conn_options: APIConnectOptions,
+        fnc_ctx: llm.FunctionContext | None,
+        temperature: float | None,
+        max_output_tokens: int | None,
+        top_p: float | None,
+        tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]],
+        additional_request_fields: dict[str, Any] | None,
+    ) -> None:
+        super().__init__(
+            llm, chat_ctx=chat_ctx, fnc_ctx=fnc_ctx, conn_options=conn_options
+        )
+        self._client = boto3.client(
+            "bedrock-runtime",
+            region_name=region_name,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+        self._model = model
+        self._llm: LLM = llm
+        self._max_output_tokens = max_output_tokens
+        self._top_p = top_p
+        self._temperature = temperature
+        self._tool_choice = tool_choice
+        self._additional_request_fields = additional_request_fields
+
+    async def _run(self) -> None:
+        self._tool_call_id: str | None = None
+        self._fnc_name: str | None = None
+        self._fnc_raw_arguments: str | None = None
+        self._text: str = ""
+        retryable = True
+
+        try:
+            opts: dict[str, Any] = {}
+            messages, system_instruction = _build_aws_ctx(self._chat_ctx, id(self))
+            messages = _merge_messages(messages)
+
+            def _get_tool_config() -> dict[str, Any] | None:
+                if not (self._fnc_ctx and self._fnc_ctx.ai_functions):
+                    return None
+
+                tools = _build_tools(self._fnc_ctx)
+                config: dict[str, Any] = {"tools": tools}
+
+                if isinstance(self._tool_choice, ToolChoice):
+                    config["toolChoice"] = {"tool": {"name": self._tool_choice.name}}
+                elif self._tool_choice == "required":
+                    config["toolChoice"] = {"any": {}}
+                elif self._tool_choice == "auto":
+                    config["toolChoice"] = {"auto": {}}
+                else:
+                    return None
+
+                return config
+
+            tool_config = _get_tool_config()
+            if tool_config:
+                opts["toolConfig"] = tool_config
+
+            if self._additional_request_fields:
+                opts["additionalModelRequestFields"] = _strip_nones(
+                    self._additional_request_fields
+                )
+            if system_instruction:
+                opts["system"] = [system_instruction]
+
+            inference_config = _strip_nones(
+                {
+                    "maxTokens": self._max_output_tokens,
+                    "temperature": self._temperature,
+                    "topP": self._top_p,
+                }
+            )
+            response = self._client.converse_stream(
+                modelId=self._model,
+                messages=messages,
+                inferenceConfig=inference_config,
+                **_strip_nones(opts),
+            )  # type: ignore
+
+            request_id = response["ResponseMetadata"]["RequestId"]
+            if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
+                raise APIStatusError(
+                    f"aws bedrock llm: error generating content: {response}",
+                    retryable=False,
+                    request_id=request_id,
+                )
+
+            for chunk in response["stream"]:
+                chat_chunk = self._parse_chunk(request_id, chunk)
+                if chat_chunk is not None:
+                    retryable = False
+                    self._event_ch.send_nowait(chat_chunk)
+
+                # Let other coroutines run
+                await asyncio.sleep(0)
+
+        except Exception as e:
+            raise APIConnectionError(
+                f"aws bedrock llm: error generating content: {e}",
+                retryable=retryable,
+            ) from e
+
+    def _parse_chunk(self, request_id: str, chunk: dict) -> llm.ChatChunk | None:
+        if "contentBlockStart" in chunk:
+            tool_use = chunk["contentBlockStart"]["start"]["toolUse"]
+            self._tool_call_id = tool_use["toolUseId"]
+            self._fnc_name = tool_use["name"]
+            self._fnc_raw_arguments = ""
+        elif "contentBlockDelta" in chunk:
+            delta = chunk["contentBlockDelta"]["delta"]
+            if "toolUse" in delta:
+                self._fnc_raw_arguments += delta["toolUse"]["input"]
+            elif "text" in delta:
+                self._text += delta["text"]
+        elif "contentBlockStop" in chunk:
+            if self._text:
+                chat_chunk = llm.ChatChunk(
+                    request_id=request_id,
+                    choices=[
+                        llm.Choice(
+                            delta=llm.ChoiceDelta(content=self._text, role="assistant"),
+                            index=chunk["contentBlockStop"]["contentBlockIndex"],
+                        )
+                    ],
+                )
+                self._text = ""
+                return chat_chunk
+            elif self._tool_call_id:
+                return self._try_build_function(request_id, chunk)
+
+        return None
+
+    def _try_build_function(self, request_id: str, chunk: dict) -> llm.ChatChunk | None:
+        if self._tool_call_id is None:
+            logger.warning("aws bedrock llm: no tool call id in the response")
+            return None
+        if self._fnc_name is None:
+            logger.warning("aws bedrock llm: no function name in the response")
+            return None
+        if self._fnc_raw_arguments is None:
+            logger.warning("aws bedrock llm: no function arguments in the response")
+            return None
+        if self._fnc_ctx is None:
+            logger.warning(
+                "aws bedrock llm: stream tried to run function without function context"
+            )
+            return None
+
+        fnc_info = _create_ai_function_info(
+            self._fnc_ctx,
+            self._tool_call_id,
+            self._fnc_name,
+            self._fnc_raw_arguments,
+        )
+
+        self._tool_call_id = self._fnc_name = self._fnc_raw_arguments = None
+        self._function_calls_info.append(fnc_info)
+
+        return llm.ChatChunk(
+            request_id=request_id,
+            choices=[
+                llm.Choice(
+                    delta=llm.ChoiceDelta(
+                        role="assistant",
+                        tool_calls=[fnc_info],
+                    ),
+                    index=chunk["contentBlockStop"]["contentBlockIndex"],
+                )
+            ],
+        )
+
+
+def _merge_messages(
+    messages: list[dict],
+) -> list[dict]:
+    # Anthropic enforces alternating messages
+    combined_messages: list[dict] = []
+    for m in messages:
+        if len(combined_messages) == 0 or m["role"] != combined_messages[-1]["role"]:
+            combined_messages.append(m)
+            continue
+        last_message = combined_messages[-1]
+        if not isinstance(last_message["content"], list) or not isinstance(
+            m["content"], list
+        ):
+            logger.error("message content is not a list")
+            continue
+
+        last_message["content"].extend(m["content"])
+
+    if len(combined_messages) == 0 or combined_messages[0]["role"] != "user":
+        combined_messages.insert(0, {"role": "user", "content": [{"text": "(empty)"}]})
+
+    return combined_messages
+
+
+def _strip_nones(d: dict[str, Any]) -> dict[str, Any]:
+    return {k: v for k, v in d.items() if v is not None}

livekit/plugins/aws/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.aws")

livekit/plugins/aws/models.py

@@ -0,0 +1,48 @@
+from typing import Literal
+
+TTS_SPEECH_ENGINE = Literal["standard", "neural", "long-form", "generative"]
+TTS_LANGUAGE = Literal[
+    "arb",
+    "cmn-CN",
+    "cy-GB",
+    "da-DK",
+    "de-DE",
+    "en-AU",
+    "en-GB",
+    "en-GB-WLS",
+    "en-IN",
+    "en-US",
+    "es-ES",
+    "es-MX",
+    "es-US",
+    "fr-CA",
+    "fr-FR",
+    "is-IS",
+    "it-IT",
+    "ja-JP",
+    "hi-IN",
+    "ko-KR",
+    "nb-NO",
+    "nl-NL",
+    "pl-PL",
+    "pt-BR",
+    "pt-PT",
+    "ro-RO",
+    "ru-RU",
+    "sv-SE",
+    "tr-TR",
+    "en-NZ",
+    "en-ZA",
+    "ca-ES",
+    "de-AT",
+    "yue-CN",
+    "ar-AE",
+    "fi-FI",
+    "en-IE",
+    "nl-BE",
+    "fr-BE",
+    "cs-CZ",
+    "de-CH",
+]
+
+TTS_OUTPUT_FORMAT = Literal["pcm", "mp3"]

livekit/plugins/aws/stt.py

@@ -0,0 +1,218 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Optional
+
+from amazon_transcribe.client import TranscribeStreamingClient
+from amazon_transcribe.model import Result, TranscriptEvent
+from livekit import rtc
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectOptions,
+    stt,
+    utils,
+)
+
+from ._utils import _get_aws_credentials
+from .log import logger
+
+
+@dataclass
+class STTOptions:
+    speech_region: str
+    sample_rate: int
+    language: str
+    encoding: str
+    vocabulary_name: Optional[str]
+    session_id: Optional[str]
+    vocab_filter_method: Optional[str]
+    vocab_filter_name: Optional[str]
+    show_speaker_label: Optional[bool]
+    enable_channel_identification: Optional[bool]
+    number_of_channels: Optional[int]
+    enable_partial_results_stabilization: Optional[bool]
+    partial_results_stability: Optional[str]
+    language_model_name: Optional[str]
+
+
+class STT(stt.STT):
+    def __init__(
+        self,
+        *,
+        speech_region: str = "us-east-1",
+        api_key: str | None = None,
+        api_secret: str | None = None,
+        sample_rate: int = 48000,
+        language: str = "en-US",
+        encoding: str = "pcm",
+        vocabulary_name: Optional[str] = None,
+        session_id: Optional[str] = None,
+        vocab_filter_method: Optional[str] = None,
+        vocab_filter_name: Optional[str] = None,
+        show_speaker_label: Optional[bool] = None,
+        enable_channel_identification: Optional[bool] = None,
+        number_of_channels: Optional[int] = None,
+        enable_partial_results_stabilization: Optional[bool] = None,
+        partial_results_stability: Optional[str] = None,
+        language_model_name: Optional[str] = None,
+    ):
+        super().__init__(
+            capabilities=stt.STTCapabilities(streaming=True, interim_results=True)
+        )
+
+        self._api_key, self._api_secret = _get_aws_credentials(
+            api_key, api_secret, speech_region
+        )
+        self._config = STTOptions(
+            speech_region=speech_region,
+            language=language,
+            sample_rate=sample_rate,
+            encoding=encoding,
+            vocabulary_name=vocabulary_name,
+            session_id=session_id,
+            vocab_filter_method=vocab_filter_method,
+            vocab_filter_name=vocab_filter_name,
+            show_speaker_label=show_speaker_label,
+            enable_channel_identification=enable_channel_identification,
+            number_of_channels=number_of_channels,
+            enable_partial_results_stabilization=enable_partial_results_stabilization,
+            partial_results_stability=partial_results_stability,
+            language_model_name=language_model_name,
+        )
+
+    async def _recognize_impl(
+        self,
+        buffer: utils.AudioBuffer,
+        *,
+        language: str | None,
+        conn_options: APIConnectOptions,
+    ) -> stt.SpeechEvent:
+        raise NotImplementedError(
+            "Amazon Transcribe does not support single frame recognition"
+        )
+
+    def stream(
+        self,
+        *,
+        language: str | None = None,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> "SpeechStream":
+        return SpeechStream(
+            stt=self,
+            conn_options=conn_options,
+            opts=self._config,
+        )
+
+
+class SpeechStream(stt.SpeechStream):
+    def __init__(
+        self,
+        stt: STT,
+        opts: STTOptions,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> None:
+        super().__init__(
+            stt=stt, conn_options=conn_options, sample_rate=opts.sample_rate
+        )
+        self._opts = opts
+        self._client = TranscribeStreamingClient(region=self._opts.speech_region)
+
+    async def _run(self) -> None:
+        stream = await self._client.start_stream_transcription(
+            language_code=self._opts.language,
+            media_sample_rate_hz=self._opts.sample_rate,
+            media_encoding=self._opts.encoding,
+            vocabulary_name=self._opts.vocabulary_name,
+            session_id=self._opts.session_id,
+            vocab_filter_method=self._opts.vocab_filter_method,
+            vocab_filter_name=self._opts.vocab_filter_name,
+            show_speaker_label=self._opts.show_speaker_label,
+            enable_channel_identification=self._opts.enable_channel_identification,
+            number_of_channels=self._opts.number_of_channels,
+            enable_partial_results_stabilization=self._opts.enable_partial_results_stabilization,
+            partial_results_stability=self._opts.partial_results_stability,
+            language_model_name=self._opts.language_model_name,
+        )
+
+        @utils.log_exceptions(logger=logger)
+        async def input_generator():
+            async for frame in self._input_ch:
+                if isinstance(frame, rtc.AudioFrame):
+                    await stream.input_stream.send_audio_event(
+                        audio_chunk=frame.data.tobytes()
+                    )
+            await stream.input_stream.end_stream()
+
+        @utils.log_exceptions(logger=logger)
+        async def handle_transcript_events():
+            async for event in stream.output_stream:
+                if isinstance(event, TranscriptEvent):
+                    self._process_transcript_event(event)
+
+        tasks = [
+            asyncio.create_task(input_generator()),
+            asyncio.create_task(handle_transcript_events()),
+        ]
+        try:
+            await asyncio.gather(*tasks)
+        finally:
+            await utils.aio.gracefully_cancel(*tasks)
+
+    def _process_transcript_event(self, transcript_event: TranscriptEvent):
+        stream = transcript_event.transcript.results
+        for resp in stream:
+            if resp.start_time and resp.start_time == 0.0:
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
+                )
+
+            if resp.end_time and resp.end_time > 0.0:
+                if resp.is_partial:
+                    self._event_ch.send_nowait(
+                        stt.SpeechEvent(
+                            type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                            alternatives=[
+                                _streaming_recognize_response_to_speech_data(resp)
+                            ],
+                        )
+                    )
+
+                else:
+                    self._event_ch.send_nowait(
+                        stt.SpeechEvent(
+                            type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                            alternatives=[
+                                _streaming_recognize_response_to_speech_data(resp)
+                            ],
+                        )
+                    )
+
+            if not resp.is_partial:
+                self._event_ch.send_nowait(
+                    stt.SpeechEvent(type=stt.SpeechEventType.END_OF_SPEECH)
+                )
+
+
+def _streaming_recognize_response_to_speech_data(resp: Result) -> stt.SpeechData:
+    data = stt.SpeechData(
+        language="en-US",
+        start_time=resp.start_time if resp.start_time else 0.0,
+        end_time=resp.end_time if resp.end_time else 0.0,
+        confidence=0.0,
+        text=resp.alternatives[0].transcript if resp.alternatives else "",
+    )
+
+    return data

livekit/plugins/aws/tts.py

@@ -0,0 +1,202 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Any, Callable, Optional
+
+import aiohttp
+from aiobotocore.session import AioSession, get_session
+from livekit import rtc
+from livekit.agents import (
+    APIConnectionError,
+    APIConnectOptions,
+    APIStatusError,
+    APITimeoutError,
+    tts,
+    utils,
+)
+
+from ._utils import _get_aws_credentials
+from .models import TTS_LANGUAGE, TTS_OUTPUT_FORMAT, TTS_SPEECH_ENGINE
+
+TTS_NUM_CHANNELS: int = 1
+DEFAULT_OUTPUT_FORMAT: TTS_OUTPUT_FORMAT = "pcm"
+DEFAULT_SPEECH_ENGINE: TTS_SPEECH_ENGINE = "generative"
+DEFAULT_SPEECH_REGION = "us-east-1"
+DEFAULT_VOICE = "Ruth"
+DEFAULT_SAMPLE_RATE = 16000
+
+
+@dataclass
+class _TTSOptions:
+    # https://docs.aws.amazon.com/polly/latest/dg/API_SynthesizeSpeech.html
+    voice: str | None
+    output_format: TTS_OUTPUT_FORMAT
+    speech_engine: TTS_SPEECH_ENGINE
+    speech_region: str
+    sample_rate: int
+    language: TTS_LANGUAGE | str | None
+
+
+class TTS(tts.TTS):
+    def __init__(
+        self,
+        *,
+        voice: str | None = DEFAULT_VOICE,
+        language: TTS_LANGUAGE | str | None = None,
+        output_format: TTS_OUTPUT_FORMAT = DEFAULT_OUTPUT_FORMAT,
+        speech_engine: TTS_SPEECH_ENGINE = DEFAULT_SPEECH_ENGINE,
+        sample_rate: int = DEFAULT_SAMPLE_RATE,
+        speech_region: str = DEFAULT_SPEECH_REGION,
+        api_key: str | None = None,
+        api_secret: str | None = None,
+        session: AioSession | None = None,
+    ) -> None:
+        """
+        Create a new instance of AWS Polly TTS.
+
+        ``api_key``  and ``api_secret`` must be set to your AWS Access key id and secret access key, either using the argument or by setting the
+        ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environmental variables.
+
+        See https://docs.aws.amazon.com/polly/latest/dg/API_SynthesizeSpeech.html for more details on the the AWS Polly TTS.
+
+        Args:
+            Voice (TTSModels, optional): Voice ID to use for the synthesis. Defaults to "Ruth".
+            language (TTS_LANGUAGE, optional): language code for the Synthesize Speech request. This is only necessary if using a bilingual voice, such as Aditi, which can be used for either Indian English (en-IN) or Hindi (hi-IN).
+            output_format(TTS_OUTPUT_FORMAT, optional): The format in which the returned output will be encoded. Defaults to "pcm".
+            sample_rate(int, optional): The audio frequency specified in Hz. Defaults to 16000.
+            speech_engine(TTS_SPEECH_ENGINE, optional): The engine to use for the synthesis. Defaults to "generative".
+            speech_region(str, optional): The region to use for the synthesis. Defaults to "us-east-1".
+            api_key(str, optional): AWS access key id.
+            api_secret(str, optional): AWS secret access key.
+        """
+        super().__init__(
+            capabilities=tts.TTSCapabilities(
+                streaming=False,
+            ),
+            sample_rate=sample_rate,
+            num_channels=TTS_NUM_CHANNELS,
+        )
+
+        self._api_key, self._api_secret = _get_aws_credentials(
+            api_key, api_secret, speech_region
+        )
+
+        self._opts = _TTSOptions(
+            voice=voice,
+            output_format=output_format,
+            speech_engine=speech_engine,
+            speech_region=speech_region,
+            language=language,
+            sample_rate=sample_rate,
+        )
+        self._session = session or get_session()
+
+    def _get_client(self):
+        return self._session.create_client(
+            "polly",
+            region_name=self._opts.speech_region,
+            aws_access_key_id=self._api_key,
+            aws_secret_access_key=self._api_secret,
+        )
+
+    def synthesize(
+        self,
+        text: str,
+        *,
+        conn_options: Optional[APIConnectOptions] = None,
+    ) -> "ChunkedStream":
+        return ChunkedStream(
+            tts=self,
+            text=text,
+            conn_options=conn_options,
+            opts=self._opts,
+            get_client=self._get_client,
+        )
+
+
+class ChunkedStream(tts.ChunkedStream):
+    def __init__(
+        self,
+        *,
+        tts: TTS,
+        text: str,
+        conn_options: Optional[APIConnectOptions] = None,
+        opts: _TTSOptions,
+        get_client: Callable[[], Any],
+    ) -> None:
+        super().__init__(tts=tts, input_text=text, conn_options=conn_options)
+        self._opts = opts
+        self._get_client = get_client
+        self._segment_id = utils.shortuuid()
+
+    async def _run(self):
+        request_id = utils.shortuuid()
+
+        try:
+            async with self._get_client() as client:
+                params = {
+                    "Text": self._input_text,
+                    "OutputFormat": self._opts.output_format,
+                    "Engine": self._opts.speech_engine,
+                    "VoiceId": self._opts.voice,
+                    "TextType": "text",
+                    "SampleRate": str(self._opts.sample_rate),
+                    "LanguageCode": self._opts.language,
+                }
+                response = await client.synthesize_speech(**_strip_nones(params))
+                if "AudioStream" in response:
+                    decoder = utils.codecs.Mp3StreamDecoder()
+                    async with response["AudioStream"] as resp:
+                        async for data, _ in resp.content.iter_chunks():
+                            if self._opts.output_format == "mp3":
+                                frames = decoder.decode_chunk(data)
+                                for frame in frames:
+                                    self._event_ch.send_nowait(
+                                        tts.SynthesizedAudio(
+                                            request_id=request_id,
+                                            segment_id=self._segment_id,
+                                            frame=frame,
+                                        )
+                                    )
+                            else:
+                                self._event_ch.send_nowait(
+                                    tts.SynthesizedAudio(
+                                        request_id=request_id,
+                                        segment_id=self._segment_id,
+                                        frame=rtc.AudioFrame(
+                                            data=data,
+                                            sample_rate=self._opts.sample_rate,
+                                            num_channels=1,
+                                            samples_per_channel=len(data) // 2,
+                                        ),
+                                    )
+                                )
+
+        except asyncio.TimeoutError as e:
+            raise APITimeoutError() from e
+        except aiohttp.ClientResponseError as e:
+            raise APIStatusError(
+                message=e.message,
+                status_code=e.status,
+                request_id=request_id,
+                body=None,
+            ) from e
+        except Exception as e:
+            raise APIConnectionError() from e
+
+
+def _strip_nones(d: dict[str, Any]) -> dict[str, Any]:
+    return {k: v for k, v in d.items() if v is not None}

livekit/plugins/aws/version.py

@@ -0,0 +1,15 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.1.0"

livekit_plugins_aws-0.1.0.dist-info/METADATA

@@ -0,0 +1,53 @@
+Metadata-Version: 2.2
+Name: livekit-plugins-aws
+Version: 0.1.0
+Summary: LiveKit Agents Plugin for services from AWS
+Home-page: https://github.com/livekit/agents
+License: Apache-2.0
+Project-URL: Documentation, https://docs.livekit.io
+Project-URL: Website, https://livekit.io/
+Project-URL: Source, https://github.com/livekit/agents
+Keywords: webrtc,realtime,audio,video,livekit,aws
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.9.0
+Description-Content-Type: text/markdown
+Requires-Dist: livekit-agents>=0.12.0
+Requires-Dist: aiobotocore==2.19.0
+Requires-Dist: boto3==1.36.3
+Requires-Dist: amazon-transcribe>=0.6.2
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# LiveKit Plugins AWS
+
+Agent Framework plugin for services from AWS.
+
+- aws polly for tts
+- aws transcribe for stt
+- aws bedrock for llm
+
+## Installation
+
+```bash
+pip install livekit-plugins-aws
+```
+
+## Pre-requisites
+
+You'll need to specify an AWS Access Key and a Deployment Region. They can be set as environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_DEFAULT_REGION`, respectively.

livekit_plugins_aws-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+livekit/plugins/aws/__init__.py,sha256=Ea-hK7QdutnwdZvvs9K2fiR8RWJqz2JcONxXnV1kXF0,977
+livekit/plugins/aws/_utils.py,sha256=iuDuQpPta4wLtgW1Wc2rHspZWoa7KZI76tujQIPY898,7411
+livekit/plugins/aws/llm.py,sha256=yUAiBCtb2jRB1_S9BNrILTMmDffvKOpDod802kYnPVM,13527
+livekit/plugins/aws/log.py,sha256=jFief0Xhv0n_F6sp6UFu9VKxs2bXNVGAfYGmEYfR_2Q,66
+livekit/plugins/aws/models.py,sha256=wb7AfN-z7qgtKMZnUbQsELi6wN8ha5exI3DH8z6Gz3M,711
+livekit/plugins/aws/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/aws/stt.py,sha256=eH7gKtdCjwki20Th6PrCsjjtH-zjXa8ZWu-cu_KaT80,7935
+livekit/plugins/aws/tts.py,sha256=miUYrhstJ7tcLkvJ-8Cpv1UCQxRSdOqaSC2tvHBh9WI,7800
+livekit/plugins/aws/version.py,sha256=vQH9cItKAVYAmrLbOntkbLqmxrUZrPiKb1TjkZ8jRKQ,600
+livekit_plugins_aws-0.1.0.dist-info/METADATA,sha256=FUzLRO0YcUvcIidEEq_EK7Lbp6yPYKjzT_BkclYNGhM,1686
+livekit_plugins_aws-0.1.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+livekit_plugins_aws-0.1.0.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
+livekit_plugins_aws-0.1.0.dist-info/RECORD,,

livekit_plugins_aws-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

livekit_plugins_aws-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+livekit