xiaozhi-sdk 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- file/greet.wav +0 -0
- file/leijun.jpg +0 -0
- file/say_hello.wav +0 -0
- file/take_photo.wav +0 -0
- libs/linux/arm64/libopus.so +0 -0
- libs/linux/x64/libopus.so +0 -0
- libs/macos/arm64/libopus.dylib +0 -0
- libs/macos/x64/libopus.dylib +0 -0
- libs/win/x86_64/opus.dll +0 -0
- xiaozhi_sdk/__init__.py +158 -0
- xiaozhi_sdk/__main__.py +94 -0
- xiaozhi_sdk/config.py +5 -0
- xiaozhi_sdk/data.py +60 -0
- xiaozhi_sdk/iot.py +52 -0
- xiaozhi_sdk/mcp.py +77 -0
- xiaozhi_sdk/opus.py +55 -0
- xiaozhi_sdk/utils.py +54 -0
- xiaozhi_sdk-0.0.1.dist-info/METADATA +100 -0
- xiaozhi_sdk-0.0.1.dist-info/RECORD +22 -0
- xiaozhi_sdk-0.0.1.dist-info/WHEEL +5 -0
- xiaozhi_sdk-0.0.1.dist-info/licenses/LICENSE +21 -0
- xiaozhi_sdk-0.0.1.dist-info/top_level.txt +1 -0
file/greet.wav
ADDED
|
Binary file
|
file/leijun.jpg
ADDED
|
Binary file
|
file/say_hello.wav
ADDED
|
Binary file
|
file/take_photo.wav
ADDED
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
libs/win/x86_64/opus.dll
ADDED
|
Binary file
|
xiaozhi_sdk/__init__.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import uuid
|
|
5
|
+
from collections import deque
|
|
6
|
+
from typing import Any, Callable, Dict
|
|
7
|
+
|
|
8
|
+
import websockets
|
|
9
|
+
|
|
10
|
+
from xiaozhi_sdk.config import INPUT_SERVER_AUDIO_SAMPLE_RATE, WSS_URL
|
|
11
|
+
from xiaozhi_sdk.iot import OtaDevice
|
|
12
|
+
from xiaozhi_sdk.mcp import McpTool
|
|
13
|
+
from xiaozhi_sdk.utils import get_wav_info, read_audio_file, setup_opus
|
|
14
|
+
|
|
15
|
+
setup_opus()
|
|
16
|
+
from xiaozhi_sdk.opus import AudioOpus
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class XiaoZhiWebsocket(McpTool):
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self, message_handler_callback=None, url=None, ota_url=None, audio_sample_rate=16000, audio_channels=1
|
|
23
|
+
):
|
|
24
|
+
super().__init__()
|
|
25
|
+
self.url = url or WSS_URL
|
|
26
|
+
self.ota_url = ota_url
|
|
27
|
+
self.audio_sample_rate = audio_sample_rate
|
|
28
|
+
self.audio_channels = audio_channels
|
|
29
|
+
self.audio_opus = AudioOpus(audio_sample_rate, audio_channels)
|
|
30
|
+
self.client_id = str(uuid.uuid4())
|
|
31
|
+
self.mac_addr = None
|
|
32
|
+
self.message_handler_callback = message_handler_callback
|
|
33
|
+
|
|
34
|
+
self.hello_received = asyncio.Event()
|
|
35
|
+
self.session_id = ""
|
|
36
|
+
self.audio_queue = deque()
|
|
37
|
+
self.websocket = None
|
|
38
|
+
self.message_handler_task = None
|
|
39
|
+
self.ota = None
|
|
40
|
+
|
|
41
|
+
async def send_hello(self, aec: bool):
|
|
42
|
+
hello_message = {
|
|
43
|
+
"type": "hello",
|
|
44
|
+
"version": 1,
|
|
45
|
+
"features": {"aec": aec, "mcp": True},
|
|
46
|
+
"transport": "websocket",
|
|
47
|
+
"audio_params": {
|
|
48
|
+
"format": "opus",
|
|
49
|
+
"sample_rate": INPUT_SERVER_AUDIO_SAMPLE_RATE,
|
|
50
|
+
"channels": 1,
|
|
51
|
+
"frame_duration": 60,
|
|
52
|
+
},
|
|
53
|
+
}
|
|
54
|
+
await self.websocket.send(json.dumps(hello_message))
|
|
55
|
+
await asyncio.wait_for(self.hello_received.wait(), timeout=10.0)
|
|
56
|
+
|
|
57
|
+
async def start_listen(self):
|
|
58
|
+
listen_message = {"session_id": self.session_id, "type": "listen", "state": "start", "mode": "realtime"}
|
|
59
|
+
await self.websocket.send(json.dumps(listen_message))
|
|
60
|
+
|
|
61
|
+
async def set_mcp_tool_callback(self, tool_func: Dict[str, Callable[..., Any]]):
|
|
62
|
+
self.tool_func = tool_func
|
|
63
|
+
|
|
64
|
+
async def activate_iot_device(self, ota_info):
|
|
65
|
+
if ota_info.get("activation"):
|
|
66
|
+
await self.send_demo_audio()
|
|
67
|
+
challenge = ota_info["activation"]["challenge"]
|
|
68
|
+
await asyncio.sleep(3)
|
|
69
|
+
for _ in range(10):
|
|
70
|
+
if await self.ota.check_activate(challenge):
|
|
71
|
+
break
|
|
72
|
+
await asyncio.sleep(3)
|
|
73
|
+
|
|
74
|
+
async def init_connection(self, mac_addr: str, aec: bool = False):
|
|
75
|
+
self.mac_addr = mac_addr
|
|
76
|
+
self.ota = OtaDevice(self.mac_addr, self.client_id, self.ota_url)
|
|
77
|
+
ota_info = await self.ota.activate_device()
|
|
78
|
+
|
|
79
|
+
headers = {
|
|
80
|
+
"Authorization": "Bearer test-token",
|
|
81
|
+
"Protocol-Version": "1",
|
|
82
|
+
"Device-Id": mac_addr,
|
|
83
|
+
"Client-Id": self.client_id,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
self.websocket = await websockets.connect(uri=self.url, additional_headers=headers)
|
|
87
|
+
self.message_handler_task = asyncio.create_task(self.message_handler())
|
|
88
|
+
await self.send_hello(aec)
|
|
89
|
+
await self.start_listen()
|
|
90
|
+
asyncio.create_task(self.activate_iot_device(ota_info))
|
|
91
|
+
|
|
92
|
+
async def send_demo_audio(self):
|
|
93
|
+
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
94
|
+
wav_path = os.path.join(current_dir, "../file/greet.wav")
|
|
95
|
+
framerate, nchannels = get_wav_info(wav_path)
|
|
96
|
+
audio_opus = AudioOpus(framerate, nchannels)
|
|
97
|
+
|
|
98
|
+
for pcm_data in read_audio_file(wav_path):
|
|
99
|
+
opus_data = await audio_opus.pcm_to_opus(pcm_data)
|
|
100
|
+
await self.websocket.send(opus_data)
|
|
101
|
+
await self.send_silence_audio()
|
|
102
|
+
|
|
103
|
+
async def send_silence_audio(self, duration_seconds: float = 1.2):
|
|
104
|
+
# 发送 静音数据
|
|
105
|
+
frames_count = int(duration_seconds * 1000 / 60)
|
|
106
|
+
pcm_frame = b"\x00\x00" * int(INPUT_SERVER_AUDIO_SAMPLE_RATE / 1000 * 60)
|
|
107
|
+
|
|
108
|
+
for _ in range(frames_count):
|
|
109
|
+
await self.send_audio(pcm_frame)
|
|
110
|
+
|
|
111
|
+
async def send_audio(self, pcm: bytes):
|
|
112
|
+
if not self.websocket:
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
state = self.websocket.state
|
|
116
|
+
if state == websockets.protocol.State.OPEN:
|
|
117
|
+
opus_data = await self.audio_opus.pcm_to_opus(pcm)
|
|
118
|
+
await self.websocket.send(opus_data)
|
|
119
|
+
elif state in [websockets.protocol.State.CLOSED, websockets.protocol.State.CLOSING]:
|
|
120
|
+
if self.message_handler_callback:
|
|
121
|
+
await self.message_handler_callback({"type": "websocket", "state": "close", "source": "sdk.send_audio"})
|
|
122
|
+
await asyncio.sleep(0.5)
|
|
123
|
+
else:
|
|
124
|
+
await asyncio.sleep(0.1)
|
|
125
|
+
|
|
126
|
+
async def message_handler(self):
|
|
127
|
+
try:
|
|
128
|
+
async for message in self.websocket:
|
|
129
|
+
if isinstance(message, bytes):
|
|
130
|
+
pcm_array = await self.audio_opus.opus_to_pcm(message)
|
|
131
|
+
self.audio_queue.extend(pcm_array)
|
|
132
|
+
else:
|
|
133
|
+
data = json.loads(message)
|
|
134
|
+
message_type = data["type"]
|
|
135
|
+
|
|
136
|
+
if message_type == "hello":
|
|
137
|
+
self.hello_received.set()
|
|
138
|
+
self.session_id = data["session_id"]
|
|
139
|
+
elif message_type == "mcp":
|
|
140
|
+
await self.mcp(data)
|
|
141
|
+
elif self.message_handler_callback:
|
|
142
|
+
await self.message_handler_callback(data)
|
|
143
|
+
except websockets.ConnectionClosed:
|
|
144
|
+
if self.message_handler_callback:
|
|
145
|
+
await self.message_handler_callback(
|
|
146
|
+
{"type": "websocket", "state": "close", "source": "sdk.message_handler"}
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
async def close(self):
|
|
150
|
+
if self.message_handler_task and not self.message_handler_task.done():
|
|
151
|
+
self.message_handler_task.cancel()
|
|
152
|
+
try:
|
|
153
|
+
await self.message_handler_task
|
|
154
|
+
except asyncio.CancelledError:
|
|
155
|
+
pass
|
|
156
|
+
|
|
157
|
+
if self.websocket:
|
|
158
|
+
await self.websocket.close()
|
xiaozhi_sdk/__main__.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import asyncio
|
|
3
|
+
import re
|
|
4
|
+
import time
|
|
5
|
+
from collections import deque
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import sounddevice as sd
|
|
9
|
+
|
|
10
|
+
from xiaozhi_sdk import XiaoZhiWebsocket
|
|
11
|
+
from xiaozhi_sdk.config import INPUT_SERVER_AUDIO_SAMPLE_RATE
|
|
12
|
+
|
|
13
|
+
input_audio: deque[bytes] = deque()
|
|
14
|
+
|
|
15
|
+
is_play_audio = False
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
async def message_handler_callback(message):
|
|
19
|
+
print("message received:", message)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def assistant_audio_play(audio_queue):
|
|
23
|
+
global is_play_audio
|
|
24
|
+
# 创建一个持续播放的流
|
|
25
|
+
stream = sd.OutputStream(samplerate=INPUT_SERVER_AUDIO_SAMPLE_RATE, channels=1, dtype=np.int16)
|
|
26
|
+
stream.start()
|
|
27
|
+
last_time = None
|
|
28
|
+
|
|
29
|
+
while True:
|
|
30
|
+
|
|
31
|
+
if not audio_queue:
|
|
32
|
+
await asyncio.sleep(0.01)
|
|
33
|
+
if last_time and time.time() - last_time > 1:
|
|
34
|
+
is_play_audio = False
|
|
35
|
+
continue
|
|
36
|
+
|
|
37
|
+
is_play_audio = True
|
|
38
|
+
pcm_data = audio_queue.popleft()
|
|
39
|
+
stream.write(pcm_data)
|
|
40
|
+
last_time = time.time()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Client:
|
|
44
|
+
def __init__(self, mac_address, url=None, ota_url=None):
|
|
45
|
+
self.mac_address = mac_address
|
|
46
|
+
self.xiaozhi = None
|
|
47
|
+
self.url = url
|
|
48
|
+
self.ota_url = ota_url
|
|
49
|
+
|
|
50
|
+
async def start(self):
|
|
51
|
+
self.xiaozhi = XiaoZhiWebsocket(message_handler_callback, url=self.url, ota_url=self.ota_url)
|
|
52
|
+
await self.xiaozhi.init_connection(self.mac_address, aec=False)
|
|
53
|
+
asyncio.create_task(assistant_audio_play(self.xiaozhi.audio_queue))
|
|
54
|
+
|
|
55
|
+
def callback_func(self, indata, frames, time, status):
|
|
56
|
+
pcm = (indata.flatten() * 32767).astype(np.int16).tobytes()
|
|
57
|
+
input_audio.append(pcm)
|
|
58
|
+
|
|
59
|
+
async def process_audio(self):
|
|
60
|
+
while True:
|
|
61
|
+
if not input_audio:
|
|
62
|
+
await asyncio.sleep(0.02)
|
|
63
|
+
continue
|
|
64
|
+
pcm = input_audio.popleft()
|
|
65
|
+
if not is_play_audio:
|
|
66
|
+
await self.xiaozhi.send_audio(pcm)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def mac_address(string):
|
|
70
|
+
"""验证是否为有效的MAC地址"""
|
|
71
|
+
if re.fullmatch(r"([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}", string):
|
|
72
|
+
return string
|
|
73
|
+
else:
|
|
74
|
+
raise argparse.ArgumentTypeError(f"无效的MAC地址格式: '{string}'")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def main():
|
|
78
|
+
parser = argparse.ArgumentParser(description="这是一个小智SDK。")
|
|
79
|
+
parser.add_argument("device", type=mac_address, help="你的小智设备的MAC地址 (格式: XX:XX:XX:XX:XX:XX)")
|
|
80
|
+
parser.add_argument("--url", help="小智服务 websocket 地址")
|
|
81
|
+
parser.add_argument("--ota_url", help="小智 OTA 地址")
|
|
82
|
+
|
|
83
|
+
args = parser.parse_args()
|
|
84
|
+
client = Client(args.device, args.url, args.ota_url)
|
|
85
|
+
await client.start()
|
|
86
|
+
await asyncio.sleep(2)
|
|
87
|
+
|
|
88
|
+
with sd.InputStream(callback=client.callback_func, channels=1, samplerate=16000, blocksize=960):
|
|
89
|
+
print("Recording... Press Ctrl+C to stop.")
|
|
90
|
+
await client.process_audio() # 持续处理音频
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
if __name__ == "__main__":
|
|
94
|
+
asyncio.run(main())
|
xiaozhi_sdk/config.py
ADDED
xiaozhi_sdk/data.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from typing import Any, Dict, List
|
|
2
|
+
|
|
3
|
+
mcp_initialize_payload: Dict[str, Any] = {
|
|
4
|
+
"jsonrpc": "2.0",
|
|
5
|
+
"id": 1,
|
|
6
|
+
"result": {
|
|
7
|
+
"protocolVersion": "2024-11-05",
|
|
8
|
+
"capabilities": {"tools": {}},
|
|
9
|
+
"serverInfo": {"name": "", "version": "0.0.1"},
|
|
10
|
+
},
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
mcp_tool_conf: Dict[str, Dict[str, Any]] = {
|
|
14
|
+
"get_device_status": {
|
|
15
|
+
"description": "Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\nUse this tool for: \n1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
|
|
16
|
+
"inputSchema": {"type": "object", "properties": {}},
|
|
17
|
+
},
|
|
18
|
+
"set_volume": {
|
|
19
|
+
"description": "Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
|
|
20
|
+
"inputSchema": {
|
|
21
|
+
"type": "object",
|
|
22
|
+
"properties": {"volume": {"type": "integer", "minimum": 0, "maximum": 100}},
|
|
23
|
+
"required": ["volume"],
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
"set_brightness": {
|
|
27
|
+
"description": "Set the brightness of the screen.",
|
|
28
|
+
"inputSchema": {
|
|
29
|
+
"type": "object",
|
|
30
|
+
"properties": {"brightness": {"type": "integer", "minimum": 0, "maximum": 100}},
|
|
31
|
+
"required": ["brightness"],
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
"set_theme": {
|
|
35
|
+
"description": "Set the theme of the screen. The theme can be `light` or `dark`.",
|
|
36
|
+
"inputSchema": {"type": "object", "properties": {"theme": {"type": "string"}}, "required": ["theme"]},
|
|
37
|
+
},
|
|
38
|
+
"take_photo": {
|
|
39
|
+
"description": "Take a photo and explain it. Use this tool after the user asks you to see something.\nArgs:\n `question`: The question that you want to ask about the photo.\nReturn:\n A JSON object that provides the photo information.",
|
|
40
|
+
"inputSchema": {
|
|
41
|
+
"type": "object",
|
|
42
|
+
"properties": {"question": {"type": "string"}},
|
|
43
|
+
"required": ["question"],
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
"open_tab": {
|
|
47
|
+
"description": "Open a web page in the browser. 小智后台:https://xiaozhi.me",
|
|
48
|
+
"inputSchema": {
|
|
49
|
+
"type": "object",
|
|
50
|
+
"properties": {"url": {"type": "string"}},
|
|
51
|
+
"required": ["url"],
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
mcp_tools_payload: Dict[str, Any] = {
|
|
57
|
+
"jsonrpc": "2.0",
|
|
58
|
+
"id": 2,
|
|
59
|
+
"result": {"tools": []},
|
|
60
|
+
}
|
xiaozhi_sdk/iot.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import aiohttp
|
|
4
|
+
|
|
5
|
+
from xiaozhi_sdk.config import OTA_URL
|
|
6
|
+
|
|
7
|
+
USER_AGENT = "XiaoXhi-SDK/1.0"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class OtaDevice(object):
|
|
11
|
+
|
|
12
|
+
def __init__(self, mac_addr: str, client_id: str, ota_url: str, serial_number: str = ""):
|
|
13
|
+
self.ota_url = ota_url or OTA_URL
|
|
14
|
+
self.mac_addr = mac_addr
|
|
15
|
+
self.client_id = client_id
|
|
16
|
+
self.serial_number = serial_number
|
|
17
|
+
|
|
18
|
+
async def activate_device(self):
|
|
19
|
+
header = {
|
|
20
|
+
"user-agent": USER_AGENT,
|
|
21
|
+
"Device-Id": self.mac_addr,
|
|
22
|
+
"Client-Id": self.client_id,
|
|
23
|
+
"Content-Type": "application/json",
|
|
24
|
+
"serial-number": self.serial_number,
|
|
25
|
+
}
|
|
26
|
+
payload = {
|
|
27
|
+
"application": {"version": "1.0.0"},
|
|
28
|
+
"board": {
|
|
29
|
+
"type": "xiaozhi-sdk-box",
|
|
30
|
+
"name": "xiaozhi-sdk-main",
|
|
31
|
+
},
|
|
32
|
+
}
|
|
33
|
+
async with aiohttp.ClientSession() as session:
|
|
34
|
+
async with session.post(self.ota_url, headers=header, data=json.dumps(payload)) as response:
|
|
35
|
+
data = await response.json()
|
|
36
|
+
return data
|
|
37
|
+
|
|
38
|
+
async def check_activate(self, challenge: str):
|
|
39
|
+
url = self.ota_url + "/activate"
|
|
40
|
+
header = {
|
|
41
|
+
"user-agent": USER_AGENT,
|
|
42
|
+
"Device-Id": self.mac_addr,
|
|
43
|
+
"Client-Id": self.client_id,
|
|
44
|
+
"Content-Type": "application/json",
|
|
45
|
+
}
|
|
46
|
+
payload = {
|
|
47
|
+
"serial_number": self.serial_number,
|
|
48
|
+
"challenge": challenge,
|
|
49
|
+
}
|
|
50
|
+
async with aiohttp.ClientSession() as session:
|
|
51
|
+
async with session.post(url, headers=header, data=json.dumps(payload)) as response:
|
|
52
|
+
return response.status == 200
|
xiaozhi_sdk/mcp.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
|
|
5
|
+
from xiaozhi_sdk.config import VL_URL
|
|
6
|
+
from xiaozhi_sdk.data import mcp_initialize_payload, mcp_tool_conf, mcp_tools_payload
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class McpTool(object):
|
|
10
|
+
|
|
11
|
+
def __init__(self):
|
|
12
|
+
self.session_id = ""
|
|
13
|
+
self.vl_token = ""
|
|
14
|
+
self.websocket = None
|
|
15
|
+
self.tool_func = {}
|
|
16
|
+
|
|
17
|
+
def get_mcp_json(self, payload: dict):
|
|
18
|
+
return json.dumps({"session_id": self.session_id, "type": "mcp", "payload": payload})
|
|
19
|
+
|
|
20
|
+
def _build_response(self, request_id: str, content: str, is_error: bool = False):
|
|
21
|
+
return self.get_mcp_json(
|
|
22
|
+
{
|
|
23
|
+
"jsonrpc": "2.0",
|
|
24
|
+
"id": request_id,
|
|
25
|
+
"result": {
|
|
26
|
+
"content": [{"type": "text", "text": content}],
|
|
27
|
+
"isError": is_error,
|
|
28
|
+
},
|
|
29
|
+
}
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
async def analyze_image(self, img_byte: bytes, question: str = "这张图片里有什么?"):
|
|
33
|
+
headers = {"Authorization": f"Bearer {self.vl_token}"}
|
|
34
|
+
files = {"file": ("camera.jpg", img_byte, "image/jpeg")}
|
|
35
|
+
payload = {"question": question}
|
|
36
|
+
|
|
37
|
+
response = requests.post(VL_URL, files=files, data=payload, headers=headers)
|
|
38
|
+
return response.json()
|
|
39
|
+
|
|
40
|
+
async def mcp_tool_call(self, mcp_json: dict):
|
|
41
|
+
tool_name = mcp_json["params"]["name"]
|
|
42
|
+
tool_func = self.tool_func[tool_name]
|
|
43
|
+
|
|
44
|
+
if tool_name == "take_photo":
|
|
45
|
+
res = await self.analyze_image(tool_func(None), mcp_json["params"]["arguments"]["question"])
|
|
46
|
+
else:
|
|
47
|
+
res = tool_func(mcp_json["params"]["arguments"])
|
|
48
|
+
|
|
49
|
+
content = json.dumps(res, ensure_ascii=False)
|
|
50
|
+
return self._build_response(mcp_json["id"], content)
|
|
51
|
+
|
|
52
|
+
async def mcp(self, data: dict):
|
|
53
|
+
payload = data["payload"]
|
|
54
|
+
method = payload["method"]
|
|
55
|
+
|
|
56
|
+
if method == "initialize":
|
|
57
|
+
self.vl_token = payload["params"]["capabilities"]["vision"]["token"]
|
|
58
|
+
mcp_initialize_payload["id"] = payload["id"]
|
|
59
|
+
await self.websocket.send(self.get_mcp_json(mcp_initialize_payload))
|
|
60
|
+
|
|
61
|
+
elif method == "tools/list":
|
|
62
|
+
mcp_tools_payload["id"] = payload["id"]
|
|
63
|
+
for name, func in self.tool_func.items():
|
|
64
|
+
if func:
|
|
65
|
+
mcp_tool_conf[name]["name"] = name
|
|
66
|
+
mcp_tools_payload["result"]["tools"].append(mcp_tool_conf[name])
|
|
67
|
+
|
|
68
|
+
await self.websocket.send(self.get_mcp_json(mcp_tools_payload))
|
|
69
|
+
|
|
70
|
+
elif method == "tools/call":
|
|
71
|
+
print("tools/call", payload)
|
|
72
|
+
tool_name = payload["params"]["name"]
|
|
73
|
+
if not self.tool_func.get(tool_name):
|
|
74
|
+
raise Exception("Tool not found")
|
|
75
|
+
|
|
76
|
+
mcp_res = await self.mcp_tool_call(payload)
|
|
77
|
+
await self.websocket.send(mcp_res)
|
xiaozhi_sdk/opus.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import av
|
|
2
|
+
import numpy as np
|
|
3
|
+
import opuslib
|
|
4
|
+
|
|
5
|
+
from xiaozhi_sdk import INPUT_SERVER_AUDIO_SAMPLE_RATE
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AudioOpus:
|
|
9
|
+
|
|
10
|
+
def __init__(self, sample_rate, channels):
|
|
11
|
+
self.sample_rate = sample_rate
|
|
12
|
+
self.channels = channels
|
|
13
|
+
|
|
14
|
+
# 创建 Opus 编码器
|
|
15
|
+
self.opus_encoder = opuslib.Encoder(
|
|
16
|
+
fs=sample_rate, channels=channels, application=opuslib.APPLICATION_VOIP # 采样率 # 单声道 # 语音应用
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
# 创建 Opus 解码器
|
|
20
|
+
self.opus_decoder = opuslib.Decoder(
|
|
21
|
+
fs=INPUT_SERVER_AUDIO_SAMPLE_RATE, # 采样率
|
|
22
|
+
channels=1, # 单声道
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
self.resampler = av.AudioResampler(format="s16", layout="mono", rate=sample_rate)
|
|
26
|
+
|
|
27
|
+
async def pcm_to_opus(self, pcm):
|
|
28
|
+
pcm_array = np.frombuffer(pcm, dtype=np.int16)
|
|
29
|
+
pcm_bytes = pcm_array.tobytes()
|
|
30
|
+
return self.opus_encoder.encode(pcm_bytes, 960)
|
|
31
|
+
|
|
32
|
+
async def change_sample_rate(self, pcm_array):
|
|
33
|
+
if self.sample_rate == INPUT_SERVER_AUDIO_SAMPLE_RATE:
|
|
34
|
+
return pcm_array.reshape(1, 960)
|
|
35
|
+
|
|
36
|
+
c = int(self.sample_rate / INPUT_SERVER_AUDIO_SAMPLE_RATE)
|
|
37
|
+
frame = av.AudioFrame.from_ndarray(np.array(pcm_array).reshape(1, -1), format="s16", layout="mono")
|
|
38
|
+
frame.sample_rate = INPUT_SERVER_AUDIO_SAMPLE_RATE # Assuming input is 16kHz
|
|
39
|
+
resampled_frames = self.resampler.resample(frame)
|
|
40
|
+
samples = resampled_frames[0].to_ndarray().flatten()
|
|
41
|
+
new_frame = av.AudioFrame.from_ndarray(
|
|
42
|
+
samples.reshape(1, -1),
|
|
43
|
+
format="s16",
|
|
44
|
+
layout="mono",
|
|
45
|
+
)
|
|
46
|
+
new_frame.sample_rate = self.sample_rate
|
|
47
|
+
new_samples = new_frame.to_ndarray().flatten()
|
|
48
|
+
arr_padded = np.pad(new_samples, (0, 960 * c - new_samples.shape[0]), mode="constant", constant_values=0)
|
|
49
|
+
return arr_padded.reshape(c, 960)
|
|
50
|
+
|
|
51
|
+
async def opus_to_pcm(self, opus):
|
|
52
|
+
pcm_data = self.opus_decoder.decode(opus, 960)
|
|
53
|
+
pcm_array = np.frombuffer(pcm_data, dtype=np.int16)
|
|
54
|
+
samples = await self.change_sample_rate(pcm_array)
|
|
55
|
+
return samples
|
xiaozhi_sdk/utils.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import ctypes.util
|
|
2
|
+
import wave
|
|
3
|
+
import platform
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_wav_info(file_path):
|
|
7
|
+
with wave.open(file_path, "rb") as wav_file:
|
|
8
|
+
return wav_file.getframerate(), wav_file.getnchannels()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def read_audio_file(file_path):
|
|
12
|
+
"""
|
|
13
|
+
读取音频文件并通过yield返回PCM流
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
file_path (str): 音频文件路径
|
|
17
|
+
|
|
18
|
+
Yields:
|
|
19
|
+
bytes: PCM音频数据块
|
|
20
|
+
"""
|
|
21
|
+
with wave.open(file_path, "rb") as wav_file:
|
|
22
|
+
while True:
|
|
23
|
+
pcm = wav_file.readframes(960) # 每次读取960帧(60ms的音频数据)
|
|
24
|
+
if not pcm:
|
|
25
|
+
break
|
|
26
|
+
yield pcm
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def setup_opus():
|
|
30
|
+
def fake_find_library(name):
|
|
31
|
+
if name == "opus":
|
|
32
|
+
system = platform.system().lower()
|
|
33
|
+
machine = platform.machine().lower()
|
|
34
|
+
|
|
35
|
+
# 检测架构
|
|
36
|
+
if machine in ['x86_64', 'amd64', 'x64']:
|
|
37
|
+
arch = 'x64'
|
|
38
|
+
elif machine in ['arm64', 'aarch64']:
|
|
39
|
+
arch = 'arm64'
|
|
40
|
+
else:
|
|
41
|
+
# 默认使用x64作为回退
|
|
42
|
+
arch = 'x64'
|
|
43
|
+
|
|
44
|
+
if system == "darwin": # macOS
|
|
45
|
+
return f"./libs/macos/{arch}/libopus.dylib"
|
|
46
|
+
elif system == "windows": # Windows
|
|
47
|
+
return f"./libs/windows/{arch}/opus.dll"
|
|
48
|
+
elif system == "linux": # Linux
|
|
49
|
+
return f"./libs/linux/{arch}/libopus.so"
|
|
50
|
+
else:
|
|
51
|
+
# 默认情况,尝试系统查找
|
|
52
|
+
return ctypes.util.find_library(name)
|
|
53
|
+
|
|
54
|
+
ctypes.util.find_library = fake_find_library
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xiaozhi-sdk
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: 一个用于连接和控制小智智能设备的Python SDK,支持实时音频通信、MCP工具集成和设备管理功能。
|
|
5
|
+
Home-page: https://github.com/dairoot/xiaozhi-sdk
|
|
6
|
+
Author: dairoot
|
|
7
|
+
Author-email: 623815825@qq.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy
|
|
15
|
+
Requires-Dist: websockets
|
|
16
|
+
Requires-Dist: aiohttp
|
|
17
|
+
Requires-Dist: av
|
|
18
|
+
Requires-Dist: opuslib
|
|
19
|
+
Requires-Dist: requests
|
|
20
|
+
Requires-Dist: sounddevice
|
|
21
|
+
Requires-Dist: python-socks
|
|
22
|
+
Dynamic: author
|
|
23
|
+
Dynamic: author-email
|
|
24
|
+
Dynamic: classifier
|
|
25
|
+
Dynamic: description
|
|
26
|
+
Dynamic: description-content-type
|
|
27
|
+
Dynamic: home-page
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
Dynamic: requires-dist
|
|
30
|
+
Dynamic: requires-python
|
|
31
|
+
Dynamic: summary
|
|
32
|
+
|
|
33
|
+
# 小智SDK (XiaoZhi SDK)
|
|
34
|
+
|
|
35
|
+
[](https://www.python.org/downloads/)
|
|
36
|
+
[](LICENSE)
|
|
37
|
+
[](https://pypi.org/project/xiaozhi-sdk/)
|
|
38
|
+
|
|
39
|
+
基于虾哥的 [小智esp32 websocket 通讯协议](https://github.com/78/xiaozhi-esp32/blob/main/docs/websocket.md) 实现的 Python SDK。
|
|
40
|
+
|
|
41
|
+
一个用于连接和控制小智设备的 Python SDK。支持以下功能:
|
|
42
|
+
- 实时音频通信
|
|
43
|
+
- MCP 工具集成
|
|
44
|
+
- 设备管理与控制
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## 📦 安装
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install xiaozhi-sdk
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## 🚀 快速开始
|
|
57
|
+
|
|
58
|
+
### 1. 终端使用
|
|
59
|
+
|
|
60
|
+
最简单的方式是通过终端直接连接设备:
|
|
61
|
+
|
|
62
|
+
#### 查看帮助信息
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
python -m xiaozhi_sdk -h
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
输出示例:
|
|
69
|
+
```text
|
|
70
|
+
positional arguments:
|
|
71
|
+
device 你的小智设备的MAC地址 (格式: XX:XX:XX:XX:XX:XX)
|
|
72
|
+
|
|
73
|
+
options:
|
|
74
|
+
-h, --help 显示帮助信息并退出
|
|
75
|
+
--url URL 小智服务 websocket 地址
|
|
76
|
+
--ota_url OTA_URL 小智 OTA 地址
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
#### 连接设备(需要提供 MAC 地址)
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
python -m xiaozhi_sdk 00:11:22:33:44:55
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### 2. 编程使用
|
|
86
|
+
参考 [examples](examples/) 文件中的示例代码,可以快速开始使用 SDK。
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
### 运行测试
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pytest tests/
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## 🫡 致敬
|
|
99
|
+
|
|
100
|
+
- 🫡 虾哥的 [xiaozhi-esp32](https://github.com/78/xiaozhi-esp32) 项目
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
file/greet.wav,sha256=F60kKKFVQZyYh67_-9AJHMviuquSWHHqwGQewUSOAFg,32720
|
|
2
|
+
file/leijun.jpg,sha256=plhBvnB4O21RjLwH-HjNq0jH4Msy5ppA_IDWe5ieNg4,70814
|
|
3
|
+
file/say_hello.wav,sha256=RGo2MDUF7npGmjFPT4III0ibf7dIZ1c47jijrF0Yjaw,34146
|
|
4
|
+
file/take_photo.wav,sha256=_DNWg31Q8NIxN3eUS4wBC7mn4MZCWLCNPuKfKPv1ojQ,51412
|
|
5
|
+
libs/linux/arm64/libopus.so,sha256=D2H5VDUomaYuLetejCvLwCgf-iAVP0isg1yGwfsuvEE,493032
|
|
6
|
+
libs/linux/x64/libopus.so,sha256=FmXJqkxLpDzNFOHYkmOzmsp1hP0eIS5b6x_XfOs-IQA,623008
|
|
7
|
+
libs/macos/arm64/libopus.dylib,sha256=H7wXwkrGwb-hesMMZGFxWb0Ri1Y4m5GWiKsd8CfOhE8,357584
|
|
8
|
+
libs/macos/x64/libopus.dylib,sha256=MqyL_OjwSACF4Xs_-KrGbcScy4IEprr5Rlkk3ddZye8,550856
|
|
9
|
+
libs/win/x86_64/opus.dll,sha256=kLfhioMvbJhOgNMAldpWk3DCZqC5Xd70LRbHnACvAnw,463360
|
|
10
|
+
xiaozhi_sdk/__init__.py,sha256=OxmYqKsXg0vcHrr5HzbsG3jJOGjhqeGMxfONkAkTD1I,6023
|
|
11
|
+
xiaozhi_sdk/__main__.py,sha256=LyEt1-9Nk4MGMLSOyjgxiRau_-WyMxC-syE5PWMYAcA,2889
|
|
12
|
+
xiaozhi_sdk/config.py,sha256=q4e_xmYzUB4_E5h-YftsyAhfeBSapwYD-ogx9ps1fIQ,189
|
|
13
|
+
xiaozhi_sdk/data.py,sha256=ST9ks_B23iUToacccDqa49LjdWRkvxtrxbplhVKlpqw,2527
|
|
14
|
+
xiaozhi_sdk/iot.py,sha256=hw2UJAMdY41AARSh7l3XTkHzV1NUiQC3YQBWTR3YSqk,1697
|
|
15
|
+
xiaozhi_sdk/mcp.py,sha256=jvXICyZ4BAdpyCIBzw9q40JjQrzi562NQdU9-vwWQJw,2786
|
|
16
|
+
xiaozhi_sdk/opus.py,sha256=4O-kz-PcUVmpa27Vju6jv-sbwywuAXFvVL23R1-vv5o,2104
|
|
17
|
+
xiaozhi_sdk/utils.py,sha256=3o2wHRCG3dHcnn9_jbRzl1patgU1I2JTJJaTNb9EUys,1591
|
|
18
|
+
xiaozhi_sdk-0.0.1.dist-info/licenses/LICENSE,sha256=Vwgps1iODKl43cAtME_0dawTjAzNW-O2BWiN5BHggww,1085
|
|
19
|
+
xiaozhi_sdk-0.0.1.dist-info/METADATA,sha256=8g3Q-kcBWqymax4PYY0ypgVTcf_9Vl03JYlzkeWtWgs,2390
|
|
20
|
+
xiaozhi_sdk-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
21
|
+
xiaozhi_sdk-0.0.1.dist-info/top_level.txt,sha256=nBpue4hU5Ykm5CtYPsAdxSa_yqbtZsIT_gF_EkBaJPM,12
|
|
22
|
+
xiaozhi_sdk-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 dairoot
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
xiaozhi_sdk
|