kuavo-humanoid-sdk 1.2.1b3284__tar.gz → 1.3.1b98__tar.gz

{kuavo_humanoid_sdk-1.2.1b3284 → kuavo_humanoid_sdk-1.3.1b98}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kuavo_humanoid_sdk
-Version: 1.2.1b3284
+Version: 1.3.1b98
 Summary: A Python SDK for kuavo humanoid robot.
 Home-page: https://gitee.com/leju-robot/kuavo-ros-opensource/
 Author: ['lejurobot']

kuavo_humanoid_sdk-1.3.1b98/kuavo_humanoid_sdk/kuavo/core/llm_doubao_lib/audio_manager.py

@@ -0,0 +1,255 @@
+import asyncio
+import uuid
+import queue
+import threading
+import time
+import random
+from typing import Optional, Dict, Any
+import wave
+import pyaudio
+import signal
+from dataclasses import dataclass
+
+from kuavo_humanoid_sdk.common.logger import SDKLogger
+from kuavo_humanoid_sdk.kuavo.core.llm_doubao_lib import config
+from kuavo_humanoid_sdk.kuavo.core.llm_doubao_lib.realtime_dialog_client import RealtimeDialogClient
+
+
+@dataclass
+class AudioConfig:
+    """音频配置数据类"""
+    format: str
+    bit_size: int
+    channels: int
+    sample_rate: int
+    chunk: int
+
+
+class AudioDeviceManager:
+    """音频设备管理类，处理音频输入输出"""
+
+    def __init__(self, input_config: AudioConfig, output_config: AudioConfig):
+        self.input_config = input_config
+        self.output_config = output_config
+        self.pyaudio = pyaudio.PyAudio()
+        self.input_stream: Optional[pyaudio.Stream] = None
+        self.output_stream: Optional[pyaudio.Stream] = None
+
+    def open_input_stream(self) -> pyaudio.Stream:
+        """打开音频输入流"""
+        # p = pyaudio.PyAudio()
+        self.input_stream = self.pyaudio.open(
+            format=self.input_config.bit_size,
+            channels=self.input_config.channels,
+            rate=self.input_config.sample_rate,
+            input=True,
+            frames_per_buffer=self.input_config.chunk
+        )
+        return self.input_stream
+
+    def open_output_stream(self) -> pyaudio.Stream:
+        """打开音频输出流"""
+        self.output_stream = self.pyaudio.open(
+            format=self.output_config.bit_size,
+            channels=self.output_config.channels,
+            rate=self.output_config.sample_rate,
+            output=True,
+            frames_per_buffer=self.output_config.chunk
+        )
+        return self.output_stream
+
+    def cleanup(self) -> None:
+        """清理音频设备资源"""
+        for stream in [self.input_stream, self.output_stream]:
+            if stream:
+                stream.stop_stream()
+                stream.close()
+        self.pyaudio.terminate()
+
+
+class DialogSession:
+    """对话会话管理类"""
+
+    def __init__(self, ws_config: Dict[str, Any], enable_signal_handler: bool = True):
+        self.session_id = str(uuid.uuid4())
+        self.client = RealtimeDialogClient(config=ws_config, session_id=self.session_id)
+        self.audio_device = AudioDeviceManager(
+            AudioConfig(**config.input_audio_config),
+            AudioConfig(**config.output_audio_config)
+        )
+
+        self.is_running = True
+        self.is_session_finished = False
+        self.is_user_querying = False
+        self.is_sending_chat_tts_text = False
+        self.audio_buffer = b''
+
+        # 只在主线程中设置信号处理器
+        if enable_signal_handler:
+            try:
+                signal.signal(signal.SIGINT, self._keyboard_signal)
+            except ValueError as e:
+                # 如果不在主线程中，忽略这个错误
+                SDKLogger.warning(f"Warning: Cannot set signal handler (not in main thread): {e}")
+        
+        # 初始化音频队列和输出流
+        self.audio_queue = queue.Queue()
+        self.output_stream = self.audio_device.open_output_stream()
+        # 启动播放线程
+        self.is_recording = True
+        self.is_playing = True
+        self.player_thread = threading.Thread(target=self._audio_player_thread)
+        self.player_thread.daemon = True
+        self.player_thread.start()
+
+    def _audio_player_thread(self):
+        """音频播放线程"""
+        while self.is_playing:
+            try:
+                # 从队列获取音频数据
+                audio_data = self.audio_queue.get(timeout=1.0)
+                if audio_data is not None:
+                    self.output_stream.write(audio_data)
+            except queue.Empty:
+                # 队列为空时等待一小段时间
+                time.sleep(0.1)
+            except Exception as e:
+                SDKLogger.error(f"音频播放错误: {e}")
+                time.sleep(0.1)
+
+    def handle_server_response(self, response: Dict[str, Any]) -> None:
+        if response == {}:
+            return
+        """处理服务器响应"""
+        if response['message_type'] == 'SERVER_ACK' and isinstance(response.get('payload_msg'), bytes):
+            # SDKLogger.info(f"\n接收到音频数据: {len(response['payload_msg'])} 字节")
+            if self.is_sending_chat_tts_text:
+                return
+            audio_data = response['payload_msg']
+            self.audio_queue.put(audio_data)
+            self.audio_buffer += audio_data
+        elif response['message_type'] == 'SERVER_FULL_RESPONSE':
+            SDKLogger.info(f"服务器响应: {response}")
+            event = response.get('event')
+            payload_msg = response.get('payload_msg', {})
+
+            if event == 450:
+                SDKLogger.info(f"清空缓存音频: {response['session_id']}")
+                while not self.audio_queue.empty():
+                    try:
+                        self.audio_queue.get_nowait()
+                    except queue.Empty:
+                        continue
+                self.is_user_querying = True
+
+            if event == 350 and self.is_sending_chat_tts_text and payload_msg.get("tts_type") == "chat_tts_text":
+                while not self.audio_queue.empty():
+                    try:
+                        self.audio_queue.get_nowait()
+                    except queue.Empty:
+                        continue
+                self.is_sending_chat_tts_text = False
+
+            if event == 459:
+                self.is_user_querying = False
+        elif response['message_type'] == 'SERVER_ERROR':
+            SDKLogger.error(f"服务器错误: {response['payload_msg']}")
+            raise Exception("服务器错误")
+
+    def _keyboard_signal(self, sig, frame):
+        SDKLogger.info(f"receive keyboard Ctrl+C")
+        self.is_recording = False
+        self.is_playing = False
+        self.is_running = False
+
+    async def receive_loop(self):
+        try:
+            while True:
+                response = await self.client.receive_server_response()
+                self.handle_server_response(response)
+                if 'event' in response and (response['event'] == 152 or response['event'] == 153):
+                    SDKLogger.info(f"receive session finished event: {response['event']}")
+                    self.is_session_finished = True
+                    break
+        except asyncio.CancelledError:
+            SDKLogger.info("接收任务已取消")
+        except Exception as e:
+            SDKLogger.error(f"接收消息错误: {e}")
+
+    async def process_microphone_input(self) -> None:
+        await self.client.say_hello()
+        """处理麦克风输入"""
+        stream = self.audio_device.open_input_stream()
+        SDKLogger.info("已打开麦克风，请讲话...")
+
+        while self.is_recording:
+            try:
+                # 添加exception_on_overflow=False参数来忽略溢出错误
+                audio_data = stream.read(config.input_audio_config["chunk"], exception_on_overflow=False)
+                save_pcm_to_wav(audio_data, "input.pcm")
+                await self.client.task_request(audio_data)
+                await asyncio.sleep(0.01)  # 避免CPU过度使用
+            except Exception as e:
+                SDKLogger.error(f"读取麦克风数据出错: {e}")
+                await asyncio.sleep(0.1)  # 给系统一些恢复时间
+
+    async def start(self) -> None:
+        """启动对话会话"""
+        try:
+            await self.client.connect()
+            asyncio.create_task(self.process_microphone_input())
+            asyncio.create_task(self.receive_loop())
+
+            while self.is_running:
+                await asyncio.sleep(0.1)
+
+            await self.client.finish_session()
+            while not self.is_session_finished:
+                await asyncio.sleep(0.1)
+            await self.client.finish_connection()
+            await asyncio.sleep(0.1)
+            await self.client.close()
+            SDKLogger.info(f"dialog request logid: {self.client.logid}")
+            # 将下发的 PCM 数据按输出音频配置保存为 WAV 文件
+            save_audio_to_wav_file(self.audio_buffer, "output.wav")
+        except Exception as e:
+            SDKLogger.error(f"会话错误: {e}")
+        finally:
+            self.audio_device.cleanup()
+
+
+def save_pcm_to_wav(pcm_data: bytes, filename: str) -> None:
+    """保存PCM数据为WAV文件"""
+    with wave.open(filename, 'wb') as wf:
+        wf.setnchannels(config.input_audio_config["channels"])
+        wf.setsampwidth(2)  # paInt16 = 2 bytes
+        wf.setframerate(config.input_audio_config["sample_rate"])
+        wf.writeframes(pcm_data)
+
+
+def save_audio_to_pcm_file(audio_data: bytes, filename: str) -> None:
+    """保存原始PCM音频数据到文件"""
+    if not audio_data:
+        SDKLogger.info("No audio data to save.")
+        return
+    try:
+        with open(filename, 'wb') as f:
+            f.write(audio_data)
+    except IOError as e:
+        SDKLogger.error(f"Failed to save pcm file: {e}")
+
+
+def save_audio_to_wav_file(audio_data: bytes, filename: str) -> None:
+    """保存PCM格式音频数据为WAV文件（使用输出音频配置）"""
+    if not audio_data:
+        SDKLogger.info("No audio data to save.")
+        return
+    try:
+        with wave.open(filename, 'wb') as wf:
+            wf.setnchannels(config.output_audio_config["channels"])
+            # paFloat32 = 4 bytes per sample
+            wf.setsampwidth(4)
+            wf.setframerate(config.output_audio_config["sample_rate"])
+            wf.writeframes(audio_data)
+    except IOError as e:
+        SDKLogger.error(f"Failed to save wav file: {e}")

kuavo_humanoid_sdk-1.3.1b98/kuavo_humanoid_sdk/kuavo/core/llm_doubao_lib/config.py

@@ -0,0 +1,48 @@
+import uuid
+import pyaudio
+
+# 配置信息
+ws_connect_config = {
+    "base_url": "wss://openspeech.bytedance.com/api/v3/realtime/dialogue",
+    "headers": {
+        "X-Api-App-ID": "",
+        "X-Api-Access-Key": "",
+        "X-Api-Resource-Id": "volc.speech.dialog",  # 固定值
+        "X-Api-App-Key": "PlgvMymc7f3tQnJ6",  # 固定值
+        "X-Api-Connect-Id": str(uuid.uuid4()),
+    }
+}
+
+start_session_req = {
+    "tts": {
+        "audio_config": {
+            "channel": 1,
+            "format": "pcm",
+            "sample_rate": 24000
+        },
+    },
+    "dialog": {
+        "bot_name": "鲁班",
+        "system_role": "你是一个充满活力的男生，性格开朗，热爱生活。",
+        "speaking_style": "你的说话风格简洁明了，语速适中，语调自然。",
+        "extra": {
+            "strict_audit": False
+        }
+    }
+}
+
+input_audio_config = {
+    "chunk": 3200,
+    "format": "pcm",
+    "channels": 1,
+    "sample_rate": 16000,
+    "bit_size": pyaudio.paInt16
+}
+
+output_audio_config = {
+    "chunk": 3200,
+    "format": "pcm",
+    "channels": 1,
+    "sample_rate": 24000,
+    "bit_size": pyaudio.paFloat32
+}

kuavo_humanoid_sdk-1.3.1b98/kuavo_humanoid_sdk/kuavo/core/llm_doubao_lib/protocol.py

@@ -0,0 +1,135 @@
+import gzip
+import json
+
+PROTOCOL_VERSION = 0b0001
+DEFAULT_HEADER_SIZE = 0b0001
+
+PROTOCOL_VERSION_BITS = 4
+HEADER_BITS = 4
+MESSAGE_TYPE_BITS = 4
+MESSAGE_TYPE_SPECIFIC_FLAGS_BITS = 4
+MESSAGE_SERIALIZATION_BITS = 4
+MESSAGE_COMPRESSION_BITS = 4
+RESERVED_BITS = 8
+
+# Message Type:
+CLIENT_FULL_REQUEST = 0b0001
+CLIENT_AUDIO_ONLY_REQUEST = 0b0010
+
+SERVER_FULL_RESPONSE = 0b1001
+SERVER_ACK = 0b1011
+SERVER_ERROR_RESPONSE = 0b1111
+
+# Message Type Specific Flags
+NO_SEQUENCE = 0b0000  # no check sequence
+POS_SEQUENCE = 0b0001
+NEG_SEQUENCE = 0b0010
+NEG_SEQUENCE_1 = 0b0011
+
+MSG_WITH_EVENT = 0b0100
+
+# Message Serialization
+NO_SERIALIZATION = 0b0000
+JSON = 0b0001
+THRIFT = 0b0011
+CUSTOM_TYPE = 0b1111
+
+# Message Compression
+NO_COMPRESSION = 0b0000
+GZIP = 0b0001
+CUSTOM_COMPRESSION = 0b1111
+
+
+def generate_header(
+        version=PROTOCOL_VERSION,
+        message_type=CLIENT_FULL_REQUEST,
+        message_type_specific_flags=MSG_WITH_EVENT,
+        serial_method=JSON,
+        compression_type=GZIP,
+        reserved_data=0x00,
+        extension_header=bytes()
+):
+    """
+    protocol_version(4 bits), header_size(4 bits),
+    message_type(4 bits), message_type_specific_flags(4 bits)
+    serialization_method(4 bits) message_compression(4 bits)
+    reserved(8bits) 保留字段
+    header_extensions 扩展头(大小等于 8 * 4 * (header_size - 1) )
+    """
+    header = bytearray()
+    header_size = int(len(extension_header) / 4) + 1
+    header.append((version << 4) | header_size)
+    header.append((message_type << 4) | message_type_specific_flags)
+    header.append((serial_method << 4) | compression_type)
+    header.append(reserved_data)
+    header.extend(extension_header)
+    return header
+
+
+def parse_response(res):
+    """
+    - header
+        - (4bytes)header
+        - (4bits)version(v1) + (4bits)header_size
+        - (4bits)messageType + (4bits)messageTypeFlags
+            -- 0001	CompleteClient | -- 0001 hasSequence
+            -- 0010	audioonly      | -- 0010 isTailPacket
+                                           | -- 0100 hasEvent
+        - (4bits)payloadFormat + (4bits)compression
+        - (8bits) reserve
+    - payload
+        - [optional 4 bytes] event
+        - [optional] session ID
+          -- (4 bytes)session ID len
+          -- session ID data
+        - (4 bytes)data len
+        - data
+    """
+    if isinstance(res, str):
+        return {}
+    protocol_version = res[0] >> 4
+    header_size = res[0] & 0x0f
+    message_type = res[1] >> 4
+    message_type_specific_flags = res[1] & 0x0f
+    serialization_method = res[2] >> 4
+    message_compression = res[2] & 0x0f
+    reserved = res[3]
+    header_extensions = res[4:header_size * 4]
+    payload = res[header_size * 4:]
+    result = {}
+    payload_msg = None
+    payload_size = 0
+    start = 0
+    if message_type == SERVER_FULL_RESPONSE or message_type == SERVER_ACK:
+        result['message_type'] = 'SERVER_FULL_RESPONSE'
+        if message_type == SERVER_ACK:
+            result['message_type'] = 'SERVER_ACK'
+        if message_type_specific_flags & NEG_SEQUENCE > 0:
+            result['seq'] = int.from_bytes(payload[:4], "big", signed=False)
+            start += 4
+        if message_type_specific_flags & MSG_WITH_EVENT > 0:
+            result['event'] = int.from_bytes(payload[:4], "big", signed=False)
+            start += 4
+        payload = payload[start:]
+        session_id_size = int.from_bytes(payload[:4], "big", signed=True)
+        session_id = payload[4:session_id_size+4]
+        result['session_id'] = str(session_id)
+        payload = payload[4 + session_id_size:]
+        payload_size = int.from_bytes(payload[:4], "big", signed=False)
+        payload_msg = payload[4:]
+    elif message_type == SERVER_ERROR_RESPONSE:
+        code = int.from_bytes(payload[:4], "big", signed=False)
+        result['code'] = code
+        payload_size = int.from_bytes(payload[4:8], "big", signed=False)
+        payload_msg = payload[8:]
+    if payload_msg is None:
+        return result
+    if message_compression == GZIP:
+        payload_msg = gzip.decompress(payload_msg)
+    if serialization_method == JSON:
+        payload_msg = json.loads(str(payload_msg, "utf-8"))
+    elif serialization_method != NO_SERIALIZATION:
+        payload_msg = str(payload_msg, "utf-8")
+    result['payload_msg'] = payload_msg
+    result['payload_size'] = payload_size
+    return result

kuavo_humanoid_sdk-1.3.1b98/kuavo_humanoid_sdk/kuavo/core/llm_doubao_lib/realtime_dialog_client.py

@@ -0,0 +1,164 @@
+import websockets
+import gzip
+import json
+
+from typing import Dict, Any
+
+from kuavo_humanoid_sdk.common.logger import SDKLogger
+from kuavo_humanoid_sdk.kuavo.core.llm_doubao_lib import protocol, config
+
+
+class RealtimeDialogClient:
+    def __init__(self, config: Dict[str, Any], session_id: str):
+        self.config = config
+        self.logid = ""
+        self.session_id = session_id
+        self.ws = None
+
+    async def start_connection(self) -> bool:
+        """建立WebSocket连接"""
+        try:
+            self.ws = await websockets.connect(
+                self.config['base_url'],
+                extra_headers=self.config['headers'],
+                ping_interval=None
+            )
+            self.logid = self.ws.response_headers.get("X-Tt-Logid")
+            SDKLogger.info(f"dialog server response logid: {self.logid}")
+
+            # StartConnection request
+            start_connection_request = bytearray(protocol.generate_header())
+            start_connection_request.extend(int(1).to_bytes(4, 'big'))
+            payload_bytes = str.encode("{}")
+            payload_bytes = gzip.compress(payload_bytes)
+            start_connection_request.extend((len(payload_bytes)).to_bytes(4, 'big'))
+            start_connection_request.extend(payload_bytes)
+            await self.ws.send(start_connection_request)
+            response = await self.ws.recv()
+            parsed_response = protocol.parse_response(response)
+            SDKLogger.info(f"StartConnection response: {parsed_response}")
+
+            if parsed_response.get('event') == 50:
+                return True
+            else:
+                return False
+                
+        except websockets.exceptions.InvalidStatusCode as e:
+            if e.status_code == 401:
+                SDKLogger.error(f"WebSocket authentication failed: HTTP {e.status_code} - App ID or Access Key error")
+            else:
+                SDKLogger.error(f"WebSocket connection failed: HTTP {e.status_code} - {e}")
+            return False
+            
+        except websockets.exceptions.ConnectionClosedError as e:
+            SDKLogger.error(f"WebSocket connection closed: {e}")
+            return False
+            
+        except websockets.exceptions.WebSocketException as e:
+            SDKLogger.error(f"WebSocket connection error: {e}")
+            return False
+            
+        except Exception as e:
+            SDKLogger.error(f"Failed to establish WebSocket connection: {e}")
+            return False
+
+    async def start_session(self) -> None:
+        """开启对话任务"""
+        # StartSession request
+        request_params = config.start_session_req
+        payload_bytes = str.encode(json.dumps(request_params))
+        payload_bytes = gzip.compress(payload_bytes)
+        start_session_request = bytearray(protocol.generate_header())
+        start_session_request.extend(int(100).to_bytes(4, 'big'))
+        start_session_request.extend((len(self.session_id)).to_bytes(4, 'big'))
+        start_session_request.extend(str.encode(self.session_id))
+        start_session_request.extend((len(payload_bytes)).to_bytes(4, 'big'))
+        start_session_request.extend(payload_bytes)
+        await self.ws.send(start_session_request)
+        response = await self.ws.recv()
+        SDKLogger.info(f"StartSession response: {protocol.parse_response(response)}")
+
+    async def say_hello(self) -> None:
+        """发送Hello消息"""
+        payload = {
+            "content": "你好，我是鲁班，有什么可以帮助你的？",
+        }
+        hello_request = bytearray(protocol.generate_header())
+        hello_request.extend(int(300).to_bytes(4, 'big'))
+        payload_bytes = str.encode(json.dumps(payload))
+        payload_bytes = gzip.compress(payload_bytes)
+        hello_request.extend((len(self.session_id)).to_bytes(4, 'big'))
+        hello_request.extend(str.encode(self.session_id))
+        hello_request.extend((len(payload_bytes)).to_bytes(4, 'big'))
+        hello_request.extend(payload_bytes)
+        await self.ws.send(hello_request)
+
+    async def chat_tts_text(self, is_user_querying: bool, start: bool, end: bool, content: str) -> None:
+        if is_user_querying:
+            return
+        """发送Chat TTS Text消息"""
+        payload = {
+            "start": start,
+            "end": end,
+            "content": content,
+        }
+        SDKLogger.info(f"ChatTTSTextRequest payload: {payload}")
+        payload_bytes = str.encode(json.dumps(payload))
+        payload_bytes = gzip.compress(payload_bytes)
+
+        chat_tts_text_request = bytearray(protocol.generate_header())
+        chat_tts_text_request.extend(int(500).to_bytes(4, 'big'))
+        chat_tts_text_request.extend((len(self.session_id)).to_bytes(4, 'big'))
+        chat_tts_text_request.extend(str.encode(self.session_id))
+        chat_tts_text_request.extend((len(payload_bytes)).to_bytes(4, 'big'))
+        chat_tts_text_request.extend(payload_bytes)
+        await self.ws.send(chat_tts_text_request)
+
+    async def task_request(self, audio: bytes) -> None:
+        task_request = bytearray(
+            protocol.generate_header(message_type=protocol.CLIENT_AUDIO_ONLY_REQUEST,
+                                     serial_method=protocol.NO_SERIALIZATION))
+        task_request.extend(int(200).to_bytes(4, 'big'))
+        task_request.extend((len(self.session_id)).to_bytes(4, 'big'))
+        task_request.extend(str.encode(self.session_id))
+        payload_bytes = gzip.compress(audio)
+        task_request.extend((len(payload_bytes)).to_bytes(4, 'big'))  # payload size(4 bytes)
+        task_request.extend(payload_bytes)
+        await self.ws.send(task_request)
+
+    async def receive_server_response(self) -> Dict[str, Any]:
+        try:
+            response = await self.ws.recv()
+            data = protocol.parse_response(response)
+            return data
+        except Exception as e:
+            SDKLogger.error(f"Failed to receive message: {e}")
+            return None
+
+    async def finish_session(self):
+        finish_session_request = bytearray(protocol.generate_header())
+        finish_session_request.extend(int(102).to_bytes(4, 'big'))
+        payload_bytes = str.encode("{}")
+        payload_bytes = gzip.compress(payload_bytes)
+        finish_session_request.extend((len(self.session_id)).to_bytes(4, 'big'))
+        finish_session_request.extend(str.encode(self.session_id))
+        finish_session_request.extend((len(payload_bytes)).to_bytes(4, 'big'))
+        finish_session_request.extend(payload_bytes)
+        await self.ws.send(finish_session_request)
+
+    async def finish_connection(self):
+        finish_connection_request = bytearray(protocol.generate_header())
+        finish_connection_request.extend(int(2).to_bytes(4, 'big'))
+        payload_bytes = str.encode("{}")
+        payload_bytes = gzip.compress(payload_bytes)
+        finish_connection_request.extend((len(payload_bytes)).to_bytes(4, 'big'))
+        finish_connection_request.extend(payload_bytes)
+        await self.ws.send(finish_connection_request)
+        response = await self.ws.recv()
+        SDKLogger.info(f"FinishConnection response: {protocol.parse_response(response)}")
+
+    async def close(self) -> None:
+        """关闭WebSocket连接"""
+        if self.ws:
+            SDKLogger.info(f"Closing WebSocket connection...")
+            await self.ws.close()