livekit-plugins-aliyun 1.1.0.post4__tar.gz

livekit_plugins_aliyun-1.1.0.post4/.gitignore

@@ -0,0 +1,183 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# tests
+test.py
+
+.python-version
+
+*.ipynb
+config.yaml
+uv.lock

livekit_plugins_aliyun-1.1.0.post4/PKG-INFO

@@ -0,0 +1,65 @@
+Metadata-Version: 2.4
+Name: livekit-plugins-aliyun
+Version: 1.1.0.post4
+Summary: LiveKit Agent Plugins for Alibaba Cloud
+Author-email: wangmengdi <790990241@qq.com>
+Keywords: audio,livekit,realtime,video,webrtc
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: dashscope
+Requires-Dist: livekit-agents~=1.1.0
+Requires-Dist: osc-data
+Description-Content-Type: text/markdown
+
+# livekit-plugins-aliyun
+
+适配[阿里云百炼](https://bailian.console.aliyun.com/?spm=5176.29597918.J_SEsSjsNv72yRuRFS2VknO.2.6b887b08kVpe2w&tab=model#/model-market)的[livekit-agent](https://github.com/livekit/agents)框架插件。目前支持[TTS](https://bailian.console.aliyun.com/?spm=5176.29597918.J_SEsSjsNv72yRuRFS2VknO.2.6b887b08kVpe2w&tab=model#/model-market?capabilities=%5B%22TTS%22%5D&z_type_=%7B%22capabilities%22%3A%22array%22%7D), [LLM](https://bailian.console.aliyun.com/?tab=model#/model-market), [STT](https://bailian.console.aliyun.com/?spm=5176.29597918.J_SEsSjsNv72yRuRFS2VknO.2.6b887b08kVpe2w&tab=model#/model-market?capabilities=%5B%22ASR%22%5D&z_type_=%7B%22capabilities%22%3A%22array%22%7D)。
+
+## 安装
+```python
+pip install livekit-plugins-aliyun
+```
+
+## 环境变量
+
+- LLM, STT, TTS: `DASHSCOPE_API_KEY`
+
+## 使用示例
+
+以下代码展示了如何在`livekit-agent`中使用`livekit-plugins-aliyun`插件。
+
+```python
+from livekit.agents import Agent, AgentSession, JobContext, cli, WorkerOptions
+from livekit.plugins import aliyun
+from dotenv import load_dotenv
+
+
+async def entry_point(ctx: JobContext):
+    
+    await ctx.connect()
+    
+    agent = Agent(instructions="You are a helpful assistant.")
+
+    session = AgentSession(
+        stt=aliyun.STT(model="paraformer-realtime-v2"),
+        tts=aliyun.TTS(model="cosyvoice-v2", voice="longcheng_v2"),
+        llm=aliyun.LLM(model="qwen-plus"),
+    )
+    
+    await session.start(agent=agent, room=ctx.room)
+    
+    await session.generate_reply()
+
+if __name__ == "__main__":
+    load_dotenv()
+    cli.run_app(WorkerOptions(entrypoint_fnc=entry_point))
+```
+

livekit_plugins_aliyun-1.1.0.post4/README.md

@@ -0,0 +1,44 @@
+# livekit-plugins-aliyun
+
+适配[阿里云百炼](https://bailian.console.aliyun.com/?spm=5176.29597918.J_SEsSjsNv72yRuRFS2VknO.2.6b887b08kVpe2w&tab=model#/model-market)的[livekit-agent](https://github.com/livekit/agents)框架插件。目前支持[TTS](https://bailian.console.aliyun.com/?spm=5176.29597918.J_SEsSjsNv72yRuRFS2VknO.2.6b887b08kVpe2w&tab=model#/model-market?capabilities=%5B%22TTS%22%5D&z_type_=%7B%22capabilities%22%3A%22array%22%7D), [LLM](https://bailian.console.aliyun.com/?tab=model#/model-market), [STT](https://bailian.console.aliyun.com/?spm=5176.29597918.J_SEsSjsNv72yRuRFS2VknO.2.6b887b08kVpe2w&tab=model#/model-market?capabilities=%5B%22ASR%22%5D&z_type_=%7B%22capabilities%22%3A%22array%22%7D)。
+
+## 安装
+```python
+pip install livekit-plugins-aliyun
+```
+
+## 环境变量
+
+- LLM, STT, TTS: `DASHSCOPE_API_KEY`
+
+## 使用示例
+
+以下代码展示了如何在`livekit-agent`中使用`livekit-plugins-aliyun`插件。
+
+```python
+from livekit.agents import Agent, AgentSession, JobContext, cli, WorkerOptions
+from livekit.plugins import aliyun
+from dotenv import load_dotenv
+
+
+async def entry_point(ctx: JobContext):
+    
+    await ctx.connect()
+    
+    agent = Agent(instructions="You are a helpful assistant.")
+
+    session = AgentSession(
+        stt=aliyun.STT(model="paraformer-realtime-v2"),
+        tts=aliyun.TTS(model="cosyvoice-v2", voice="longcheng_v2"),
+        llm=aliyun.LLM(model="qwen-plus"),
+    )
+    
+    await session.start(agent=agent, room=ctx.room)
+    
+    await session.generate_reply()
+
+if __name__ == "__main__":
+    load_dotenv()
+    cli.run_app(WorkerOptions(entrypoint_fnc=entry_point))
+```
+

livekit_plugins_aliyun-1.1.0.post4/livekit/plugins/aliyun/__init__.py

@@ -0,0 +1,27 @@
+from .llm import LLM
+from .stt import STT
+from .tts import TTS
+from .version import __version__
+
+__all__ = ["TTS", "LLM", "STT", "__version__"]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class AliyunPlugin(Plugin):
+    def __init__(self):
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(AliyunPlugin())
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit_plugins_aliyun-1.1.0.post4/livekit/plugins/aliyun/llm.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any
+import time
+
+import httpx
+import openai
+from openai.types.chat import ChatCompletionChunk, ChatCompletionToolChoiceOptionParam
+from openai.types.chat.chat_completion_chunk import Choice
+
+from livekit.agents import APIConnectionError, APIStatusError, APITimeoutError, llm
+from livekit.agents.llm import ToolChoice
+from livekit.agents.llm.chat_context import ChatContext
+from livekit.agents.llm.tool_context import FunctionTool
+from livekit.agents.types import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    NOT_GIVEN,
+    APIConnectOptions,
+    NotGivenOr,
+)
+from livekit.agents.utils import is_given
+
+from .log import logger
+from .utils import to_chat_ctx, to_fnc_ctx
+
+
+@dataclass
+class _LLMOptions:
+    model: str
+    user: NotGivenOr[str]
+    temperature: NotGivenOr[float]
+    parallel_tool_calls: NotGivenOr[bool]
+    tool_choice: NotGivenOr[ToolChoice]
+    store: NotGivenOr[bool]
+    metadata: NotGivenOr[dict[str, str]]
+
+
+class LLM(llm.LLM):
+    def __init__(
+        self,
+        *,
+        model: str = "qwen-plus",
+        api_key: NotGivenOr[str] = NOT_GIVEN,
+        client: openai.AsyncClient | None = None,
+        user: NotGivenOr[str] = NOT_GIVEN,
+        temperature: NotGivenOr[float] = NOT_GIVEN,
+        parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
+        tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        store: NotGivenOr[bool] = NOT_GIVEN,
+        metadata: NotGivenOr[dict[str, str]] = NOT_GIVEN,
+        timeout: httpx.Timeout | None = None,
+    ) -> None:
+        """
+        Create a new instance of LLM.
+
+        Args:
+            model: The model to use for the LLM, end with -v1.
+            api_key: The API key to use for the LLM.
+            base_url: The base URL to use for the LLM.
+        """
+        super().__init__()
+        self._opts = _LLMOptions(
+            model=model,
+            user=user,
+            temperature=temperature,
+            parallel_tool_calls=parallel_tool_calls,
+            tool_choice=tool_choice,
+            store=store,
+            metadata=metadata,
+        )
+        api_key = api_key if is_given(api_key) else os.getenv("DASHSCOPE_API_KEY")
+        if api_key is None:
+            raise ValueError("environment variable DASHSCOPE_API_KEY is not set")
+        self._client = client or openai.AsyncClient(
+            api_key=api_key if is_given(api_key) else None,
+            base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
+            max_retries=0,
+            http_client=httpx.AsyncClient(
+                timeout=timeout
+                if timeout
+                else httpx.Timeout(connect=15.0, read=5.0, write=5.0, pool=5.0),
+                follow_redirects=True,
+                limits=httpx.Limits(
+                    max_connections=50,
+                    max_keepalive_connections=50,
+                    keepalive_expiry=120,
+                ),
+            ),
+        )
+
+    def chat(
+        self,
+        *,
+        chat_ctx: ChatContext,
+        tools: list[FunctionTool] | None = None,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+        parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
+        tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
+        extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
+    ) -> LLMStream:
+        extra = {}
+
+        if is_given(extra_kwargs):
+            extra.update(extra_kwargs)
+
+        if is_given(self._opts.metadata):
+            extra["metadata"] = self._opts.metadata
+
+        if is_given(self._opts.user):
+            extra["user"] = self._opts.user
+
+        parallel_tool_calls = (
+            parallel_tool_calls
+            if is_given(parallel_tool_calls)
+            else self._opts.parallel_tool_calls
+        )
+        if is_given(parallel_tool_calls):
+            extra["parallel_tool_calls"] = parallel_tool_calls
+
+        tool_choice = tool_choice if is_given(tool_choice) else self._opts.tool_choice  # type: ignore
+        if is_given(tool_choice):
+            oai_tool_choice: ChatCompletionToolChoiceOptionParam
+            if isinstance(tool_choice, dict):
+                oai_tool_choice = {
+                    "type": "function",
+                    "function": {"name": tool_choice["function"]["name"]},
+                }
+                extra["tool_choice"] = oai_tool_choice
+            elif tool_choice in ("auto", "required", "none"):
+                oai_tool_choice = tool_choice
+                extra["tool_choice"] = oai_tool_choice
+        logger.info("llm start", extra={"model": self._opts.model})
+        return LLMStream(
+            self,
+            model=self._opts.model,
+            client=self._client,
+            chat_ctx=chat_ctx,
+            tools=tools or [],
+            conn_options=conn_options,
+            extra_kwargs=extra,
+        )
+
+
+class LLMStream(llm.LLMStream):
+    def __init__(
+        self,
+        llm: LLM,
+        *,
+        model: str,
+        client: openai.AsyncClient,
+        chat_ctx: llm.ChatContext,
+        tools: list[FunctionTool],
+        conn_options: APIConnectOptions,
+        extra_kwargs: dict[str, Any],
+    ) -> None:
+        super().__init__(llm, chat_ctx=chat_ctx, tools=tools, conn_options=conn_options)
+        self._model = model
+        self._client = client
+        self._llm = llm
+        self._extra_kwargs = extra_kwargs
+
+    async def _run(self) -> None:
+        # current function call that we're waiting for full completion (args are streamed)
+        # (defined inside the _run method to make sure the state is reset for each run/attempt)
+        self._oai_stream: openai.AsyncStream[ChatCompletionChunk] | None = None
+        self._tool_call_id: str | None = None
+        self._fnc_name: str | None = None
+        self._fnc_raw_arguments: str | None = None
+        self._tool_index: int | None = None
+        retryable = True
+        first_response = True
+        start = time.perf_counter()
+        try:
+            stream: openai.AsyncStream[
+                ChatCompletionChunk
+            ] = await self._client.chat.completions.create(
+                messages=to_chat_ctx(self._chat_ctx, id(self._llm)),
+                tools=to_fnc_ctx(self._tools) if self._tools else openai.NOT_GIVEN,
+                model=self._model,
+                stream_options={"include_usage": True},
+                stream=True,
+                **self._extra_kwargs,
+            )
+
+            async with stream:
+                async for chunk in stream:
+                    for choice in chunk.choices:
+                        chat_chunk = self._parse_choice(chunk.id, choice)
+                        if chat_chunk is not None:
+                            retryable = False
+                            self._event_ch.send_nowait(chat_chunk)
+                        if first_response:
+                            spent = time.perf_counter() - start
+                            logger.info(
+                                "llm first response", extra={"spent": round(spent, 4)}
+                            )
+                            first_response = False
+
+                    if chunk.usage is not None:
+                        retryable = False
+                        chunk = llm.ChatChunk(
+                            id=chunk.id,
+                            usage=llm.CompletionUsage(
+                                completion_tokens=chunk.usage.completion_tokens,
+                                prompt_tokens=chunk.usage.prompt_tokens,
+                                total_tokens=chunk.usage.total_tokens,
+                            ),
+                        )
+                        self._event_ch.send_nowait(chunk)
+            logger.info("llm end")
+
+        except openai.APITimeoutError:
+            raise APITimeoutError(retryable=retryable)  # noqa: B904
+        except openai.APIStatusError as e:
+            raise APIStatusError(  # noqa: B904
+                e.message,
+                status_code=e.status_code,
+                request_id=e.request_id,
+                body=e.body,
+                retryable=retryable,
+            )
+        except Exception as e:
+            raise APIConnectionError(retryable=retryable) from e
+
+    def _parse_choice(self, id: str, choice: Choice) -> llm.ChatChunk | None:
+        delta = choice.delta
+
+        if delta is None:
+            return None
+
+        if delta.tool_calls:
+            for tool in delta.tool_calls:
+                if not tool.function:
+                    continue
+
+                call_chunk = None
+                if self._tool_call_id and tool.id and tool.index != self._tool_index:
+                    call_chunk = llm.ChatChunk(
+                        id=id,
+                        delta=llm.ChoiceDelta(
+                            role="assistant",
+                            content=delta.content,
+                            tool_calls=[
+                                llm.FunctionToolCall(
+                                    arguments=self._fnc_raw_arguments or "",
+                                    name=self._fnc_name or "",
+                                    call_id=self._tool_call_id or "",
+                                )
+                            ],
+                        ),
+                    )
+                    self._tool_call_id = self._fnc_name = self._fnc_raw_arguments = None
+
+                if tool.function.name:
+                    self._tool_index = tool.index
+                    self._tool_call_id = tool.id
+                    self._fnc_name = tool.function.name
+                    self._fnc_raw_arguments = tool.function.arguments or ""
+                elif tool.function.arguments:
+                    self._fnc_raw_arguments += tool.function.arguments  # type: ignore
+
+                if call_chunk is not None:
+                    return call_chunk
+
+        if choice.finish_reason in ("tool_calls", "stop") and self._tool_call_id:
+            call_chunk = llm.ChatChunk(
+                id=id,
+                delta=llm.ChoiceDelta(
+                    role="assistant",
+                    content=delta.content,
+                    tool_calls=[
+                        llm.FunctionToolCall(
+                            arguments=self._fnc_raw_arguments or "",
+                            name=self._fnc_name or "",
+                            call_id=self._tool_call_id or "",
+                        )
+                    ],
+                ),
+            )
+            self._tool_call_id = self._fnc_name = self._fnc_raw_arguments = None
+            return call_chunk
+
+        return llm.ChatChunk(
+            id=id,
+            delta=llm.ChoiceDelta(content=delta.content, role="assistant"),
+        )

livekit_plugins_aliyun-1.1.0.post4/livekit/plugins/aliyun/log.py

@@ -0,0 +1,3 @@
+from logging import getLogger
+
+logger = getLogger("livekit.plugins.aliyun")

livekit_plugins_aliyun-1.1.0.post4/livekit/plugins/aliyun/stt.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+import os
+from dataclasses import dataclass
+from typing import List
+
+import asyncio
+from dashscope.audio.asr import Recognition, RecognitionCallback, RecognitionResult
+
+from livekit import rtc
+from livekit.agents import stt, utils, APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS
+from livekit.agents.types import (
+    NOT_GIVEN,
+    NotGivenOr,
+)
+from .log import logger
+
+
+@dataclass
+class STTOptions:
+    api_key: str | None
+    language: str | None
+    detect_language: bool
+    interim_results: bool
+    punctuate: bool
+    model: str
+    smart_format: bool
+    endpointing: int | None
+    sample_rate: int = 16000
+
+
+class STT(stt.STT):
+    def __init__(
+        self,
+        *,
+        language="zh",
+        detect_language: bool = False,
+        interim_results: bool = True,
+        punctuate: bool = True,
+        smart_format: bool = True,
+        model: str = "paraformer-realtime-v2",
+        api_key: str | None = None,
+        min_silence_duration: int = 500,
+    ) -> None:
+        super().__init__(
+            capabilities=stt.STTCapabilities(
+                streaming=True, interim_results=interim_results
+            )
+        )
+        api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
+        if api_key is None:
+            raise ValueError("DASHSCOPE API key is required")
+        self._opts = STTOptions(
+            api_key=api_key,
+            language=language,
+            detect_language=detect_language,
+            interim_results=interim_results,
+            punctuate=punctuate,
+            model=model,
+            smart_format=smart_format,
+            endpointing=min_silence_duration,
+        )
+
+    async def _recognize_impl(
+        self,
+        buffer: utils.AudioBuffer,
+        *,
+        language: NotGivenOr[str] = NOT_GIVEN,
+        conn_options: APIConnectOptions,
+    ) -> stt.SpeechEvent:
+        raise NotImplementedError("not implemented")
+
+    def stream(
+        self,
+        *,
+        language: str | None = None,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> "SpeechStream":
+        return SpeechStream(stt=self, opts=self._opts, conn_options=conn_options)
+
+
+class SpeechStream(stt.SpeechStream):
+    def __init__(
+        self,
+        stt: STT,
+        opts: STTOptions,
+        conn_options: APIConnectOptions,
+    ) -> None:
+        super().__init__(stt=stt, conn_options=conn_options)
+
+        if opts.language is None:
+            raise ValueError("language detection is not supported in streaming mode")
+        self._opts: STTOptions = opts
+        self._config = opts
+        self._speaking = False
+        self.recognition = Recognition(
+            model=opts.model,
+            format="pcm",
+            sample_rate=opts.sample_rate,
+            callback=Callback(self),
+            disfluency_removal_enabled=True,
+            semantic_punctuation_enabled=False,
+            max_sentence_silence=opts.endpointing,
+            language_hints=[opts.language],
+        )
+        self._closed = False
+        self._request_id = utils.shortuuid()
+        self._reconnect_event = asyncio.Event()
+
+    async def _run(self) -> None:
+        self.recognition.start()
+        samples_100ms = self._opts.sample_rate // 10
+        audio_bstream = utils.audio.AudioByteStream(
+            sample_rate=self._opts.sample_rate,
+            num_channels=1,
+            samples_per_channel=samples_100ms,
+        )
+        while True:
+            try:
+                has_ended = False
+                async for data in self._input_ch:
+                    frames: list[rtc.AudioFrame] = []
+                    if isinstance(data, rtc.AudioFrame):
+                        frames.extend(audio_bstream.write(data.data.tobytes()))
+                    elif isinstance(data, self._FlushSentinel):
+                        frames.extend(audio_bstream.flush())
+                        has_ended = True
+                    for frame in frames:
+                        self.recognition.send_audio_frame(frame.data.tobytes())
+                        if has_ended:
+                            self.recognition.stop()
+            finally:
+                self.recognition.stop()
+
+
+def live_transcription_to_speech_data(
+    language: str,
+    data,
+) -> List[stt.SpeechData]:
+    return [
+        stt.SpeechData(
+            language=language,
+            start_time=data["begin_time"],
+            end_time=data["end_time"],
+            confidence=0.0,
+            text=data["text"],
+        )
+    ]
+
+
+class Callback(RecognitionCallback):
+    def __init__(self, _stt: SpeechStream):
+        self._stt = _stt
+
+    def on_event(self, result: RecognitionResult) -> None:
+        sentence = result.get_sentence()
+        dg_alts = live_transcription_to_speech_data(
+            self._stt._config.language, sentence
+        )
+        if not result.is_sentence_end(sentence):
+            interim_event = stt.SpeechEvent(
+                type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                alternatives=dg_alts,
+            )
+            self._stt._event_ch.send_nowait(interim_event)
+            logger.info("transcription start")
+        else:
+            final_event = stt.SpeechEvent(
+                type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                alternatives=dg_alts,
+            )
+            self._stt._event_ch.send_nowait(final_event)
+            logger.info(
+                "transcription end", extra={"text": final_event.alternatives[0].text}
+            )

livekit_plugins_aliyun-1.1.0.post4/livekit/plugins/aliyun/tts.py

@@ -0,0 +1,260 @@
+import os
+from dataclasses import dataclass
+from typing import AsyncIterable, Optional, Dict
+import time
+import aiohttp
+import asyncio
+import json
+
+from livekit.agents import tts, APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, utils
+from osc_data.text_stream import TextStreamSentencizer
+
+from .log import logger
+
+
+STREAM_EOS = "EOS"
+
+
+@dataclass
+class TTSOptions:
+    api_key: str
+    model: str
+    # 语速，取值范围：0.5~2。
+    rate: float
+    # 音色
+    voice: str
+    # 合成音频的语速，取值范围：0.5~2。
+    speech_rate: int
+    # 合成音频的音量，取值范围：0~100。
+    volume: int
+    # 采样率，取值范围：8000, 16000, 22050, 24000, 44100, 48000
+    sample_rate: int
+    # 音调，取值范围：0.5~2。
+    pitch: float = 1.0
+
+    def get_ws_url(self) -> str:
+        return "wss://dashscope.aliyuncs.com/api-ws/v1/inference"
+
+    def get_ws_header(self) -> Dict[str, str]:
+        return {
+            "Authorization": f"bearer {self.api_key}",
+            "X-DashScope-DataInspection": "enable",
+        }
+
+    def get_run_task_params(self) -> Dict[str, str]:
+        params = {
+            "header": {
+                "action": "run-task",
+                "task_id": utils.shortuuid(),
+                "streaming": "duplex",
+            },
+            "payload": {
+                "task_group": "audio",
+                "task": "tts",
+                "function": "SpeechSynthesizer",
+                "model": self.model,
+                "parameters": {
+                    "text_type": "PlainText",
+                    "voice": "longxiaochun_v2",
+                    "format": "pcm",
+                    "sample_rate": self.sample_rate,
+                    "volume": self.volume,
+                    "rate": self.rate,
+                    "pitch": self.pitch,
+                },
+                "input": {},
+            },
+        }
+        return params
+
+    def get_continue_task_params(self, text: str) -> Dict[str, str]:
+        params = {
+            "header": {
+                "action": "continue-task",
+                "task_id": utils.shortuuid(),
+                "streaming": "duplex",
+            },
+            "payload": {
+                "input": {
+                    "text": text,
+                }
+            },
+        }
+        return params
+
+    def get_finish_task_params(self) -> Dict[str, str]:
+        params = {
+            "header": {
+                "action": "finish-task",
+                "task_id": utils.shortuuid(),
+                "streaming": "duplex",
+            },
+            "payload": {"input": {}},
+        }
+        return params
+
+
+class TTS(tts.TTS):
+    def __init__(
+        self,
+        *,
+        api_key: Optional[str] = None,
+        sample_rate: int = 24000,
+        voice: str = "longcheng",
+        model: str = "cosyvoice-v2",
+        speech_rate: int = 1,
+        volume: int = 100,
+        rate: float = 1.0,
+        pitch: float = 1.0,
+        http_session: aiohttp.ClientSession | None = None,
+        max_session_duration: float = 600,
+    ) -> None:
+        super().__init__(
+            capabilities=tts.TTSCapabilities(streaming=True),
+            sample_rate=sample_rate,
+            num_channels=1,
+        )
+        api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
+        if not api_key:
+            raise ValueError("DASHSCOPE_API_KEY must be set")
+        self._session = http_session
+        self._opts = TTSOptions(
+            model=model,
+            api_key=api_key,
+            voice=voice,
+            speech_rate=speech_rate,
+            volume=volume,
+            sample_rate=sample_rate,
+            rate=rate,
+            pitch=pitch,
+        )
+        self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse](
+            connect_cb=self._connect_ws,
+            close_cb=self._close_ws,
+            max_session_duration=max_session_duration,
+            mark_refreshed_on_get=True,
+        )
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if self._session is None:
+            self._session = utils.http_context.http_session()
+
+        return self._session
+
+    async def _connect_ws(self, timeout: float) -> aiohttp.ClientWebSocketResponse:
+        session = self._ensure_session()
+        url = self._opts.get_ws_url()
+        headers = self._opts.get_ws_header()
+        return await asyncio.wait_for(
+            session.ws_connect(url, headers=headers),
+            timeout=timeout,
+        )
+
+    async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse):
+        await ws.close()
+
+    def synthesize(
+        self,
+        text: str,
+    ) -> AsyncIterable[tts.SynthesizedAudio]:
+        raise NotImplementedError
+
+    def stream(
+        self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
+    ) -> "SynthesizeStream":
+        return SynthesizeStream(tts=self, opts=self._opts, conn_options=conn_options)
+
+
+class SynthesizeStream(tts.SynthesizeStream):
+    def __init__(
+        self,
+        *,
+        tts: TTS,
+        opts: TTSOptions,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ):
+        super().__init__(tts=tts, conn_options=conn_options)
+        self._opts = opts
+
+    async def _run(self, emitter: tts.AudioEmitter) -> None:
+        request_id = utils.shortuuid()
+        emitter.initialize(
+            request_id=request_id,
+            sample_rate=self._opts.sample_rate,
+            mime_type="audio/pcm",
+            stream=True,
+            num_channels=1,
+            frame_size_ms=200,
+        )
+
+        async def _send_task(sentence: str, ws: aiohttp.ClientWebSocketResponse):
+            run_task_params = self._opts.get_run_task_params()
+            await ws.send_json(run_task_params)
+            continue_task_params = self._opts.get_continue_task_params(text=sentence)
+            await ws.send_json(continue_task_params)
+            finish_task_params = self._opts.get_finish_task_params()
+            await ws.send_json(finish_task_params)
+
+        async def _recv_task(ws: aiohttp.ClientWebSocketResponse):
+            is_first_response = True
+            start_time = time.perf_counter()
+            while True:
+                try:
+                    msg = await ws.receive()
+                except Exception as e:
+                    logger.warning(f"Error while receiving bytes: {e}")
+                    break
+                if msg.type == aiohttp.WSMsgType.BINARY:
+                    if is_first_response:
+                        elapsed_time = time.perf_counter() - start_time
+                        logger.info(
+                            "tts first response",
+                            extra={"spent": round(elapsed_time, 4)},
+                        )
+                        is_first_response = False
+                    emitter.push(data=msg.data)
+                elif msg.type == aiohttp.WSMsgType.TEXT:
+                    msg_json = json.loads(msg.data)
+                    if "header" in msg_json:
+                        header = msg_json["header"]
+                        if "event" in header:
+                            event = header["event"]
+
+                            if event == "task-finished":
+                                break
+
+                            if event == "task-failed":
+                                error_msg = msg_json.get("error_message", "未知错误")
+                                logger.error(f"任务失败: {error_msg}")
+
+        splitter = TextStreamSentencizer()
+        is_first_sentence = True
+        start_time = time.perf_counter()
+        async for token in self._input_ch:
+            if isinstance(token, self._FlushSentinel):
+                sentences = splitter.flush()
+            else:
+                sentences = splitter.push(text=token)
+            for sentence in sentences:
+                if is_first_sentence:
+                    first_sentence_spend = time.perf_counter() - start_time
+                    logger.info(
+                        "llm first sentence",
+                        extra={"spent": str(first_sentence_spend)},
+                    )
+                    is_first_sentence = False
+                logger.info("tts start", extra={"sentence": sentence})
+                emitter.start_segment(segment_id=utils.shortuuid())
+                async with self._tts._pool.connection(
+                    timeout=self._conn_options.timeout
+                ) as ws:
+                    assert not ws.closed, "WebSocket connection is closed"
+                    tasks = [
+                        asyncio.create_task(_send_task(sentence=sentence, ws=ws)),
+                        asyncio.create_task(_recv_task(ws=ws)),
+                    ]
+                    await asyncio.gather(*tasks)
+                    emitter.end_segment()
+                    logger.info("tts end", extra={"sentence": sentence})
+                    self._pushed_text = self._pushed_text.replace(sentence, "")
+                    await utils.aio.gracefully_cancel(*tasks)

livekit_plugins_aliyun-1.1.0.post4/livekit/plugins/aliyun/utils.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import base64
+import os
+from collections import OrderedDict
+from collections.abc import Awaitable
+from typing import Any, Callable, Union
+
+from openai.types.chat import (
+    ChatCompletionContentPartParam,
+    ChatCompletionMessageParam,
+    ChatCompletionToolParam,
+)
+
+from livekit.agents import llm
+
+AsyncAzureADTokenProvider = Callable[[], Union[str, Awaitable[str]]]
+
+
+def get_base_url(base_url: str | None) -> str:
+    if not base_url:
+        base_url = os.getenv(
+            "ZHIPU_LLM_BASE_URL", "https://open.bigmodel.cn/api/paas/v4/"
+        )
+    return base_url
+
+
+def to_fnc_ctx(fnc_ctx: list[llm.FunctionTool]) -> list[ChatCompletionToolParam]:
+    return [llm.utils.build_strict_openai_schema(fnc) for fnc in fnc_ctx]
+
+
+def to_chat_ctx(
+    chat_ctx: llm.ChatContext, cache_key: Any
+) -> list[ChatCompletionMessageParam]:
+    # group the message and function_calls
+    item_groups: dict[str, list[llm.ChatItem]] = OrderedDict()
+    for item in chat_ctx.items:
+        if (
+            item.type == "message" and item.role == "assistant"
+        ) or item.type == "function_call":
+            group_id = item.id.split("/")[0]
+            if group_id not in item_groups:
+                item_groups[group_id] = []
+            item_groups[group_id].append(item)
+        else:
+            item_groups[item.id] = [item]
+
+    return [_group_to_chat_item(items, cache_key) for items in item_groups.values()]
+
+
+def _group_to_chat_item(
+    items: list[llm.ChatItem], cache_key: Any
+) -> ChatCompletionMessageParam:
+    if len(items) == 1:
+        return _to_chat_item(items[0], cache_key)
+    else:
+        msg = {"role": "assistant", "tool_calls": []}
+        for item in items:
+            if item.type == "message":
+                assert item.role == "assistant", (
+                    "only assistant messages can be grouped"
+                )
+                assert "content" not in msg, (
+                    "only one assistant message is allowed in a group"
+                )
+
+                msg.update(_to_chat_item(item, cache_key))
+            elif item.type == "function_call":
+                msg["tool_calls"].append(
+                    {
+                        "id": item.call_id,
+                        "type": "function",
+                        "function": {"name": item.name, "arguments": item.arguments},
+                    }
+                )
+        return msg
+
+
+def _to_chat_item(msg: llm.ChatItem, cache_key: Any) -> ChatCompletionMessageParam:
+    if msg.type == "message":
+        list_content: list[ChatCompletionContentPartParam] = []
+        text_content = ""
+        for content in msg.content:
+            if isinstance(content, str):
+                if text_content:
+                    text_content += "\n"
+                text_content += content
+            elif isinstance(content, llm.ImageContent):
+                list_content.append(_to_image_content(content, cache_key))
+
+        if not list_content:
+            # certain providers require text-only content in a string vs a list.
+            # for max-compatibility, we will combine all text content into a single string.
+            return {
+                "role": msg.role,  # type: ignore
+                "content": text_content,
+            }
+
+        if text_content:
+            list_content.append({"type": "text", "text": text_content})
+
+        return {
+            "role": msg.role,  # type: ignore
+            "content": list_content,
+        }
+
+    elif msg.type == "function_call":
+        return {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "id": msg.call_id,
+                    "type": "function",
+                    "function": {
+                        "name": msg.name,
+                        "arguments": msg.arguments,
+                    },
+                }
+            ],
+        }
+
+    elif msg.type == "function_call_output":
+        return {
+            "role": "tool",
+            "tool_call_id": msg.call_id,
+            "content": msg.output,
+        }
+
+
+def _to_image_content(
+    image: llm.ImageContent, cache_key: Any
+) -> ChatCompletionContentPartParam:
+    img = llm.utils.serialize_image(image)
+    if img.external_url:
+        return {
+            "type": "image_url",
+            "image_url": {
+                "url": img.external_url,
+                "detail": img.inference_detail,
+            },
+        }
+    if cache_key not in image._cache:
+        image._cache[cache_key] = img.data_bytes
+    b64_data = base64.b64encode(image._cache[cache_key]).decode("utf-8")
+    return {
+        "type": "image_url",
+        "image_url": {
+            "url": f"data:{img.mime_type};base64,{b64_data}",
+            "detail": img.inference_detail,
+        },
+    }

livekit_plugins_aliyun-1.1.0.post4/livekit/plugins/aliyun/version.py

@@ -0,0 +1 @@
+__version__ = "1.1.0.post4"

livekit_plugins_aliyun-1.1.0.post4/pyproject.toml

@@ -0,0 +1,42 @@
+[project]
+name = "livekit-plugins-aliyun"
+dynamic = ["version"]
+description = "LiveKit Agent Plugins for Alibaba Cloud"
+readme = "README.md"
+authors = [
+    { name = "wangmengdi", email = "790990241@qq.com" }
+]
+keywords = ["webrtc", "realtime", "audio", "video", "livekit"]
+requires-python = ">=3.9"
+dependencies = [
+    "livekit-agents~=1.1.0",
+    "dashscope",
+    "osc-data"
+]
+
+classifiers = [
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Topic :: Multimedia :: Sound/Audio",
+    "Topic :: Multimedia :: Video",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3 :: Only",
+]
+
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "livekit/plugins/aliyun/version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["livekit"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/livekit"]