agentscope-runtime 0.2.0b2__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

agentscope_runtime/tools/realtime_clients/modelstudio_tts_client.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+# pylint:disable=arguments-renamed
+
+import json
+import logging
+import time
+from typing import Optional, Callable, Any
+
+from dashscope.audio.tts_v2.speech_synthesizer import (
+    SpeechSynthesizer,
+    ResultCallback,
+    AudioFormat,
+)
+from pydantic import BaseModel
+
+from .realtime_tool import (
+    RealtimeState,
+)
+from .tts_client import TtsClient
+from ...engine.schemas.realtime import ModelstudioTtsConfig
+
+logger = logging.getLogger(__name__)
+
+
+class ModelstudioTtsCallbacks(BaseModel):
+    on_open: Optional[Callable] = None
+    on_complete: Optional[Callable] = None
+    on_error: Optional[Callable] = None
+    on_close: Optional[Callable] = None
+    on_event: Optional[Callable] = None
+    on_data: Optional[Callable] = None
+
+
+class ModelstudioTtsClient(TtsClient, ResultCallback):
+    def __init__(
+        self,
+        config: ModelstudioTtsConfig,
+        callbacks: ModelstudioTtsCallbacks,
+    ):
+        super().__init__(config, callbacks)
+        self.tts_request_id = None
+        self.first_request_time = None
+        self.is_first_audio_data = True
+        self.data_index = 0
+        self.synthesizer = SpeechSynthesizer(
+            model=config.model,
+            voice=config.voice,
+            format=self.to_format(config.sample_rate),
+            callback=self,
+        )
+        self.callbacks = callbacks
+        self.state = RealtimeState.IDLE
+
+        logger.info(
+            f"modelstudio_tts_config: {json.dumps(self.config.model_dump())}",
+        )
+
+    def start(self, **kwargs: Any) -> None:
+        logger.info("tts_start")
+        self.tts_request_id = None
+        self.first_request_time = None
+        self.is_first_audio_data = True
+        self.data_index = 0
+
+        self.synthesizer.streaming_call(" ")
+
+    def stop(self, **kwargs: Any) -> None:
+        if self.state == RealtimeState.IDLE:
+            return
+
+        logger.info(f"tts_stop: tts_request_id={self.tts_request_id}")
+
+        self.synthesizer.streaming_complete()
+
+    def async_stop(self, **kwargs: Any) -> None:
+        if self.state == RealtimeState.IDLE:
+            return
+
+        logger.info(f"tts_async_stop: tts_request_id={self.tts_request_id}")
+
+        self.synthesizer.async_streaming_complete()
+
+    def close(self, **kwargs: Any) -> None:
+        self.callbacks = None
+        if self.state == RealtimeState.IDLE:
+            return
+
+        self.state = RealtimeState.IDLE
+
+        logger.info(f"tts_close: tts_request_id={self.tts_request_id}")
+
+        self.synthesizer.streaming_cancel()
+
+    def send_text_data(self, text: str) -> None:
+        if self.state == RealtimeState.IDLE:
+            return
+
+        if self.first_request_time is None:
+            self.first_request_time = int(round(time.time() * 1000))
+
+        self.synthesizer.streaming_call(text)
+
+    def on_open(self) -> None:
+        self.state = RealtimeState.RUNNING
+
+        logger.info(
+            f"tts_on_open: tts_request_id={self.tts_request_id},"
+            f" chat_id={self.config.chat_id}",
+        )
+
+        if self.callbacks and self.callbacks.on_open:
+            self.callbacks.on_open()
+
+    def on_complete(self) -> None:
+        self.state = RealtimeState.IDLE
+
+        logger.info(
+            f"tts_on_complete: tts_request_id={self.tts_request_id},"
+            f" chat_id={self.config.chat_id}",
+        )
+
+        if self.callbacks and self.callbacks.on_complete:
+            self.callbacks.on_complete(self.config.chat_id)
+
+    def on_error(self, message: str) -> None:
+        self.state = RealtimeState.IDLE
+
+        logger.error(
+            f"tts_on_on_error: tts_request_id={self.tts_request_id}, "
+            f"message={message}, chat_id={self.config.chat_id}",
+        )
+
+        if self.callbacks and self.callbacks.on_error:
+            self.callbacks.on_error(message)
+
+    def on_close(self) -> None:
+        self.state = RealtimeState.IDLE
+
+        logger.info(
+            f"tts_on_close: tts_request_id={self.tts_request_id},"
+            f" chat_id={self.config.chat_id}",
+        )
+
+        if self.callbacks and self.callbacks.on_close:
+            self.callbacks.on_close()
+
+    def on_event(self, message: str) -> None:
+        # logger.info(
+        #     f"tts_on_event: tts_request_id={self.tts_request_id}, "
+        #     f"message={message}, chat_id={self.config.chat_id}",
+        # )
+
+        event = json.loads(message)
+        if "header" in event and "task_id" in event["header"]:
+            tts_request_id = event["header"]["task_id"]
+            if self.tts_request_id != tts_request_id:
+                self.tts_request_id = tts_request_id
+
+        if self.callbacks and self.callbacks.on_event:
+            self.callbacks.on_event(message)
+
+    def on_data(self, data: bytes) -> None:
+        # logger.info(
+        #     f"tts_on_data: tts_request_id={self.tts_request_id}, "
+        #     f"data_size={len(data)}, chat_id={self.config.chat_id},
+        #     data_index={self.data_index}",
+        # )
+
+        if (
+            self.is_first_audio_data is True
+            and self.first_request_time is not None
+        ):
+            logger.info(
+                f"tts_first_delay: "
+                f"{int(round(time.time() * 1000)) - self.first_request_time}",
+            )
+            self.is_first_audio_data = False
+
+        if self.callbacks and self.callbacks.on_data:
+            self.callbacks.on_data(data, self.config.chat_id, self.data_index)
+
+        self.data_index += 1
+
+    @staticmethod
+    def to_format(sample_rate: int) -> AudioFormat:
+        if sample_rate == 8000:
+            return AudioFormat.PCM_8000HZ_MONO_16BIT
+        elif sample_rate == 16000:
+            return AudioFormat.PCM_16000HZ_MONO_16BIT
+        elif sample_rate == 22050:
+            return AudioFormat.PCM_22050HZ_MONO_16BIT
+        elif sample_rate == 24000:
+            return AudioFormat.PCM_24000HZ_MONO_16BIT
+        elif sample_rate == 44100:
+            return AudioFormat.PCM_44100HZ_MONO_16BIT
+        elif sample_rate == 48000:
+            return AudioFormat.PCM_48000HZ_MONO_16BIT
+        else:
+            raise ValueError("invalid sample rate")

agentscope_runtime/tools/realtime_clients/realtime_tool.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+from abc import abstractmethod
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class RealtimeState(Enum):
+    IDLE = "idle"
+    RUNNING = "running"
+
+
+class RealtimeType(Enum):
+    TTS = "tts"
+    ASR = "asr"
+    VOICE_CHAT = "voice_chat"
+    VIDEO_CHAT = "video_chat"
+
+
+class RealtimeComponent:
+    """
+    Base class for realtime_client components
+    """
+
+    realtime_type: RealtimeType
+    config: BaseModel
+    callbacks: BaseModel
+    state: RealtimeState
+
+    def __init__(
+        self,
+        realtime_type: RealtimeType,
+        config: BaseModel,
+        callbacks: BaseModel,
+    ):
+        self.realtime_type = realtime_type
+        self.config = config
+        self.callbacks = callbacks
+        self.state = RealtimeState.IDLE
+
+    @abstractmethod
+    def start(self, **kwargs: Any) -> None:
+        pass
+
+    @abstractmethod
+    def stop(self, **kwargs: Any) -> None:
+        pass
+
+    @abstractmethod
+    def close(self, **kwargs: Any) -> None:
+        pass
+
+    def get_state(self) -> RealtimeState:
+        return self.state

agentscope_runtime/tools/realtime_clients/tts_client.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+from .realtime_tool import (
+    RealtimeComponent,
+    RealtimeType,
+)
+from ...engine.schemas.realtime import TtsConfig
+
+
+class TtsClient(RealtimeComponent):
+    def __init__(self, config: TtsConfig, callbacks: BaseModel):
+        super().__init__(RealtimeType.TTS, config, callbacks)
+
+    def start(self, **kwargs: Any) -> None:
+        pass
+
+    def stop(self, **kwargs: Any) -> None:
+        pass
+
+    def async_stop(self, **kwargs: Any) -> None:
+        pass
+
+    def close(self, **kwargs: Any) -> None:
+        pass
+
+    def send_text_data(self, data: str) -> None:
+        pass
+
+    def set_chat_id(self, chat_id: Optional[str] = None) -> None:
+        self.config.chat_id = chat_id

agentscope_runtime/tools/searches/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+from .modelstudio_search import *
+from .modelstudio_search_lite import *