voxchain 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- voxchain/__init__.py +40 -0
- voxchain/audio.py +62 -0
- voxchain/core/__init__.py +18 -0
- voxchain/core/agent.py +20 -0
- voxchain/core/pipeline.py +184 -0
- voxchain/core/replies.py +96 -0
- voxchain/core/text.py +72 -0
- voxchain/core/turn_gate.py +111 -0
- voxchain/events.py +18 -0
- voxchain/exceptions.py +0 -0
- voxchain/output.py +110 -0
- voxchain/protocols/__init__.py +6 -0
- voxchain/protocols/openai_realtime.py +91 -0
- voxchain/protocols/sarvam.py +67 -0
- voxchain/realtime/__init__.py +48 -0
- voxchain/realtime/base.py +65 -0
- voxchain/realtime/events.py +68 -0
- voxchain/realtime/history.py +114 -0
- voxchain/realtime/openai.py +361 -0
- voxchain/realtime/pipeline.py +160 -0
- voxchain/realtime/playback.py +118 -0
- voxchain/realtime/response_gate.py +85 -0
- voxchain/realtime/tools.py +62 -0
- voxchain/session.py +261 -0
- voxchain/stt/__init__.py +19 -0
- voxchain/stt/base.py +33 -0
- voxchain/stt/events.py +22 -0
- voxchain/stt/openai.py +187 -0
- voxchain/stt/sarvam.py +116 -0
- voxchain/tasks.py +24 -0
- voxchain/tts/__init__.py +18 -0
- voxchain/tts/base.py +43 -0
- voxchain/tts/events.py +20 -0
- voxchain/tts/openai.py +240 -0
- voxchain/tts/sarvam.py +133 -0
- voxchain/turn/__init__.py +6 -0
- voxchain/turn/base.py +55 -0
- voxchain/turn/smart_turn.py +133 -0
- voxchain/vad/__init__.py +6 -0
- voxchain/vad/base.py +0 -0
- voxchain/vad/gate.py +104 -0
- voxchain/vad/silero.py +87 -0
- voxchain-0.1.0.dist-info/METADATA +233 -0
- voxchain-0.1.0.dist-info/RECORD +47 -0
- voxchain-0.1.0.dist-info/WHEEL +5 -0
- voxchain-0.1.0.dist-info/licenses/LICENSE +21 -0
- voxchain-0.1.0.dist-info/top_level.txt +1 -0
voxchain/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""voxchain: a thin, event-driven library for building real-time voice agents on LangChain/LangGraph.
|
|
2
|
+
|
|
3
|
+
Two ways to build an agent:
|
|
4
|
+
|
|
5
|
+
- :class:`~voxchain.core.pipeline.VoxchainPipeline` chains streaming speech-to-text, any
|
|
6
|
+
LangGraph agent and streaming text-to-speech.
|
|
7
|
+
- :class:`~voxchain.realtime.pipeline.RealtimePipeline` runs one speech-to-speech model
|
|
8
|
+
that hears, reasons, calls LangChain tools and speaks.
|
|
9
|
+
|
|
10
|
+
Both send the client the same :mod:`voxchain.output` events.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from voxchain.core import VoxchainPipeline
|
|
14
|
+
from voxchain.output import Output, PlayAudio, ReplyFinished, StopPlayback, StreamClosed
|
|
15
|
+
from voxchain.realtime import OpenAIRealtimeProvider, RealtimePipeline, RealtimeProvider
|
|
16
|
+
from voxchain.stt import OpenAISTTProvider, SarvamSTTProvider, STTProvider
|
|
17
|
+
from voxchain.tts import OpenAITTSProvider, SarvamTTSProvider, TTSProvider
|
|
18
|
+
from voxchain.turn import EndOfTurnDetector, PipecatSmartTurnDetector
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"EndOfTurnDetector",
|
|
24
|
+
"OpenAIRealtimeProvider",
|
|
25
|
+
"OpenAISTTProvider",
|
|
26
|
+
"OpenAITTSProvider",
|
|
27
|
+
"Output",
|
|
28
|
+
"PipecatSmartTurnDetector",
|
|
29
|
+
"PlayAudio",
|
|
30
|
+
"RealtimePipeline",
|
|
31
|
+
"RealtimeProvider",
|
|
32
|
+
"ReplyFinished",
|
|
33
|
+
"STTProvider",
|
|
34
|
+
"SarvamSTTProvider",
|
|
35
|
+
"SarvamTTSProvider",
|
|
36
|
+
"StopPlayback",
|
|
37
|
+
"StreamClosed",
|
|
38
|
+
"TTSProvider",
|
|
39
|
+
"VoxchainPipeline",
|
|
40
|
+
]
|
voxchain/audio.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Small audio helpers shared by providers."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from array import array
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PCM16Resampler:
|
|
8
|
+
"""Streaming linear-interpolation resampler for 16-bit little-endian mono PCM.
|
|
9
|
+
|
|
10
|
+
Keeps the last sample and the fractional read position between calls, so
|
|
11
|
+
chunk boundaries don't click, and the output is identical whether audio
|
|
12
|
+
arrives in one piece or many. Linear interpolation is plenty for speech
|
|
13
|
+
sent to a recognizer when upsampling (e.g. 16 kHz to 24 kHz); it does not
|
|
14
|
+
low-pass, so heavy downsampling can alias.
|
|
15
|
+
|
|
16
|
+
Example:
|
|
17
|
+
>>> resampler = PCM16Resampler(16000, 24000)
|
|
18
|
+
>>> out = resampler.process(chunk) # Call once per incoming chunk
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, source_rate: int, target_rate: int) -> None:
|
|
22
|
+
"""Create a resampler."""
|
|
23
|
+
if source_rate <= 0 or target_rate <= 0:
|
|
24
|
+
raise ValueError(f"Sample rates must be positive, got {source_rate} and {target_rate}")
|
|
25
|
+
# Positions are kept in units of 1/target_rate input samples, so they stay exact.
|
|
26
|
+
self._step = source_rate
|
|
27
|
+
self._scale = target_rate
|
|
28
|
+
self._pos = 0
|
|
29
|
+
self._prev: int | None = None
|
|
30
|
+
self._odd = b""
|
|
31
|
+
|
|
32
|
+
def process(self, chunk: bytes) -> bytes:
|
|
33
|
+
"""Resample one chunk."""
|
|
34
|
+
data = self._odd + chunk
|
|
35
|
+
cut = len(data) - len(data) % 2
|
|
36
|
+
data, self._odd = data[:cut], data[cut:]
|
|
37
|
+
if self._step == self._scale:
|
|
38
|
+
return data
|
|
39
|
+
|
|
40
|
+
samples = array("h")
|
|
41
|
+
samples.frombytes(data)
|
|
42
|
+
if sys.byteorder == "big":
|
|
43
|
+
samples.byteswap()
|
|
44
|
+
if self._prev is not None:
|
|
45
|
+
samples.insert(0, self._prev)
|
|
46
|
+
if not samples:
|
|
47
|
+
return b""
|
|
48
|
+
|
|
49
|
+
out = array("h")
|
|
50
|
+
pos, step, scale = self._pos, self._step, self._scale
|
|
51
|
+
end = (len(samples) - 1) * scale
|
|
52
|
+
while pos < end:
|
|
53
|
+
i, frac = divmod(pos, scale)
|
|
54
|
+
a = samples[i]
|
|
55
|
+
out.append(a + (samples[i + 1] - a) * frac // scale)
|
|
56
|
+
pos += step
|
|
57
|
+
self._pos = pos - end
|
|
58
|
+
self._prev = samples[-1]
|
|
59
|
+
|
|
60
|
+
if sys.byteorder == "big":
|
|
61
|
+
out.byteswap()
|
|
62
|
+
return out.tobytes()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""The cascaded pipeline (STT, a LangGraph agent, TTS) and the pieces it is built from."""
|
|
2
|
+
|
|
3
|
+
from voxchain.core.agent import stream_agent_text
|
|
4
|
+
from voxchain.core.pipeline import VoxchainPipeline
|
|
5
|
+
from voxchain.core.replies import Interruption, Reply, ReplyTracker
|
|
6
|
+
from voxchain.core.text import SentenceSegmenter, to_speakable
|
|
7
|
+
from voxchain.core.turn_gate import EndOfTurnGate
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"EndOfTurnGate",
|
|
11
|
+
"Interruption",
|
|
12
|
+
"Reply",
|
|
13
|
+
"ReplyTracker",
|
|
14
|
+
"SentenceSegmenter",
|
|
15
|
+
"VoxchainPipeline",
|
|
16
|
+
"stream_agent_text",
|
|
17
|
+
"to_speakable",
|
|
18
|
+
]
|
voxchain/core/agent.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Streaming a LangGraph agent's spoken reply."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncIterator
|
|
4
|
+
|
|
5
|
+
from langchain_core.messages import AIMessage
|
|
6
|
+
from langgraph.graph.state import CompiledStateGraph
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
async def stream_agent_text(agent: CompiledStateGraph, text: str, thread_id: str) -> AsyncIterator[str]:
|
|
10
|
+
"""Run one agent turn and yield its reply text as it streams. Tool output is never yielded.
|
|
11
|
+
|
|
12
|
+
``thread_id`` keys the agent's checkpointed memory across turns.
|
|
13
|
+
"""
|
|
14
|
+
async for message, _metadata in agent.astream(
|
|
15
|
+
{"messages": [("user", text)]},
|
|
16
|
+
config={"configurable": {"thread_id": thread_id}},
|
|
17
|
+
stream_mode="messages",
|
|
18
|
+
):
|
|
19
|
+
if isinstance(message, AIMessage) and (chunk := message.text):
|
|
20
|
+
yield chunk
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Runs a cascaded voice agent: speech-to-text, a LangGraph agent, then text-to-speech."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
7
|
+
|
|
8
|
+
from langgraph.graph.state import CompiledStateGraph
|
|
9
|
+
|
|
10
|
+
from voxchain.core.agent import stream_agent_text
|
|
11
|
+
from voxchain.core.replies import Reply, ReplyTracker
|
|
12
|
+
from voxchain.core.text import SentenceSegmenter
|
|
13
|
+
from voxchain.core.turn_gate import EndOfTurnGate
|
|
14
|
+
from voxchain.output import Output, OutputChannel
|
|
15
|
+
from voxchain.session import SessionClosed
|
|
16
|
+
from voxchain.stt import FinalTranscript, SpeechStarted, SpeechStopped, STTProvider
|
|
17
|
+
from voxchain.tasks import run_until_first_exit
|
|
18
|
+
from voxchain.tts import AudioChunk, Flushed, TTSProvider
|
|
19
|
+
from voxchain.turn import EndOfTurnDetector
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class VoxchainPipeline:
|
|
25
|
+
"""Turns user speech into agent replies spoken back, with turn-taking and barge-in.
|
|
26
|
+
|
|
27
|
+
Each committed user turn runs the agent; its reply streams to TTS sentence
|
|
28
|
+
by sentence, and replies run one at a time, in order. ``callback`` receives
|
|
29
|
+
:mod:`voxchain.output` events from its own task, so a callback that blocks on
|
|
30
|
+
playback never delays barge-in.
|
|
31
|
+
|
|
32
|
+
With ``interrupt`` on, the user talking over a reply cancels it and stops
|
|
33
|
+
playback. If they barge in within ``resume_window`` seconds of first hearing
|
|
34
|
+
it, their previous turn is merged with what they say next, because the
|
|
35
|
+
end-of-turn call was most likely premature (``None`` never merges). An
|
|
36
|
+
``end_of_turn`` detector holds transcripts that sound unfinished for up to
|
|
37
|
+
``end_of_turn_timeout`` seconds. ``thread_id`` keys the agent's checkpointed
|
|
38
|
+
memory. A pipeline runs once.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
stt: STTProvider,
|
|
44
|
+
tts: TTSProvider,
|
|
45
|
+
agent: CompiledStateGraph,
|
|
46
|
+
callback: Callable[[Output], Awaitable[None]],
|
|
47
|
+
*,
|
|
48
|
+
thread_id: str = "default",
|
|
49
|
+
interrupt: bool = True,
|
|
50
|
+
resume_window: float | None = 2.0,
|
|
51
|
+
end_of_turn: EndOfTurnDetector | None = None,
|
|
52
|
+
end_of_turn_timeout: float = 2.0,
|
|
53
|
+
clock: Callable[[], float] = time.monotonic,
|
|
54
|
+
) -> None:
|
|
55
|
+
if end_of_turn_timeout < 0:
|
|
56
|
+
raise ValueError(f"End-of-turn timeout must be >= 0, got {end_of_turn_timeout}")
|
|
57
|
+
if resume_window is not None and resume_window < 0:
|
|
58
|
+
raise ValueError(f"Resume window must be >= 0 or None, got {resume_window}")
|
|
59
|
+
self.stt = stt
|
|
60
|
+
self.tts = tts
|
|
61
|
+
self.agent = agent
|
|
62
|
+
self.thread_id = thread_id
|
|
63
|
+
self.interrupt = interrupt
|
|
64
|
+
self.clock = clock
|
|
65
|
+
|
|
66
|
+
self.replies = ReplyTracker(resume_window)
|
|
67
|
+
self.output = OutputChannel(
|
|
68
|
+
callback,
|
|
69
|
+
sample_rate=tts.sample_rate,
|
|
70
|
+
clock=clock,
|
|
71
|
+
on_played=lambda _tag, start, _end: self.replies.audio_played(start),
|
|
72
|
+
)
|
|
73
|
+
self._gate = EndOfTurnGate(end_of_turn, end_of_turn_timeout, self._start_reply)
|
|
74
|
+
self._started = False
|
|
75
|
+
|
|
76
|
+
async def run(self, audio_stream: AsyncIterator[bytes]) -> None:
|
|
77
|
+
"""Run until a provider's session is lost, the audio stream ends, or a task fails; then shut down."""
|
|
78
|
+
if self._started:
|
|
79
|
+
raise RuntimeError("A VoxchainPipeline runs once; create a new one")
|
|
80
|
+
self._started = True
|
|
81
|
+
|
|
82
|
+
await self.stt.connect()
|
|
83
|
+
await self.tts.connect()
|
|
84
|
+
loops = {
|
|
85
|
+
"send_audio": self._send_audio(audio_stream),
|
|
86
|
+
"handle_stt": self._handle_stt(),
|
|
87
|
+
"handle_tts": self._handle_tts(),
|
|
88
|
+
"play": self.output.run(),
|
|
89
|
+
}
|
|
90
|
+
try:
|
|
91
|
+
await run_until_first_exit("VoxchainPipeline", loops)
|
|
92
|
+
finally:
|
|
93
|
+
self._gate.close()
|
|
94
|
+
reply_tasks = [reply.task for reply in self.replies.interrupt(self.clock()).cancelled if reply.task]
|
|
95
|
+
for task in reply_tasks:
|
|
96
|
+
task.cancel()
|
|
97
|
+
await asyncio.gather(*reply_tasks, return_exceptions=True)
|
|
98
|
+
await self.stt.disconnect()
|
|
99
|
+
await self.tts.disconnect()
|
|
100
|
+
|
|
101
|
+
async def _send_audio(self, audio_stream: AsyncIterator[bytes]) -> None:
|
|
102
|
+
async for chunk in audio_stream:
|
|
103
|
+
self._gate.push_audio(chunk)
|
|
104
|
+
await self.stt.send_audio(chunk)
|
|
105
|
+
logger.info("VoxchainPipeline: Audio stream ended")
|
|
106
|
+
|
|
107
|
+
async def _handle_stt(self) -> None:
|
|
108
|
+
async for event in self.stt.receive_events():
|
|
109
|
+
match event:
|
|
110
|
+
case SpeechStarted():
|
|
111
|
+
await self._on_speech_started()
|
|
112
|
+
case SpeechStopped():
|
|
113
|
+
self._gate.on_speech_end()
|
|
114
|
+
case FinalTranscript(text=text) if text.strip():
|
|
115
|
+
self._gate.on_transcript(text.strip())
|
|
116
|
+
case SessionClosed(reason=reason):
|
|
117
|
+
await self._session_lost("STT", reason)
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
async def _handle_tts(self) -> None:
|
|
121
|
+
async for event in self.tts.receive_events():
|
|
122
|
+
match event:
|
|
123
|
+
case AudioChunk(data=data):
|
|
124
|
+
self.output.play(data)
|
|
125
|
+
case Flushed():
|
|
126
|
+
self.replies.synthesized()
|
|
127
|
+
self.output.reply_finished()
|
|
128
|
+
case SessionClosed(reason=reason):
|
|
129
|
+
await self._session_lost("TTS", reason)
|
|
130
|
+
return
|
|
131
|
+
|
|
132
|
+
async def _on_speech_started(self) -> None:
|
|
133
|
+
now = self.clock()
|
|
134
|
+
if self.interrupt and (self.replies.active or self.output.playing(now)):
|
|
135
|
+
interruption = self.replies.interrupt(now)
|
|
136
|
+
logger.debug(f"VoxchainPipeline: Barge-in (resuming {interruption.resume_text!r})")
|
|
137
|
+
for reply in interruption.cancelled:
|
|
138
|
+
if reply.task:
|
|
139
|
+
reply.task.cancel()
|
|
140
|
+
# Stop playback before any network call, so nothing stale reaches the client.
|
|
141
|
+
self.output.stop(now)
|
|
142
|
+
if interruption.cancel_synthesis:
|
|
143
|
+
await self.tts.cancel()
|
|
144
|
+
if interruption.resume_text:
|
|
145
|
+
self._gate.restore(interruption.resume_text)
|
|
146
|
+
self._gate.on_speech_start()
|
|
147
|
+
|
|
148
|
+
def _start_reply(self, text: str) -> None:
|
|
149
|
+
logger.debug(f"VoxchainPipeline: User turn {text!r}")
|
|
150
|
+
previous = self.replies.latest
|
|
151
|
+
reply = self.replies.start(text)
|
|
152
|
+
reply.task = asyncio.create_task(self._run_reply(reply, previous.task if previous else None))
|
|
153
|
+
reply.task.add_done_callback(lambda _: self.replies.finished_generating(reply))
|
|
154
|
+
|
|
155
|
+
async def _run_reply(self, reply: Reply, previous: asyncio.Task | None) -> None:
|
|
156
|
+
# Cancellation (barge-in) skips the closing flush, so a cancelled reply never ends in a late Flushed.
|
|
157
|
+
if previous is not None:
|
|
158
|
+
await asyncio.wait({previous})
|
|
159
|
+
segmenter = SentenceSegmenter()
|
|
160
|
+
spoken: list[str] = []
|
|
161
|
+
try:
|
|
162
|
+
async for chunk in stream_agent_text(self.agent, reply.text, self.thread_id):
|
|
163
|
+
for sentence in segmenter.push(chunk):
|
|
164
|
+
await self._speak(reply, sentence, spoken)
|
|
165
|
+
for sentence in segmenter.flush():
|
|
166
|
+
await self._speak(reply, sentence, spoken)
|
|
167
|
+
except asyncio.CancelledError:
|
|
168
|
+
logger.debug(f"VoxchainPipeline: Reply cancelled after {' '.join(spoken)!r}")
|
|
169
|
+
raise
|
|
170
|
+
except Exception:
|
|
171
|
+
logger.exception("VoxchainPipeline: Agent turn failed")
|
|
172
|
+
logger.debug(f"VoxchainPipeline: Agent replied {' '.join(spoken)!r}")
|
|
173
|
+
if spoken:
|
|
174
|
+
await self.tts.flush()
|
|
175
|
+
|
|
176
|
+
async def _speak(self, reply: Reply, sentence: str, spoken: list[str]) -> None:
|
|
177
|
+
if not spoken:
|
|
178
|
+
self.replies.spoke(reply)
|
|
179
|
+
await self.tts.speak(sentence)
|
|
180
|
+
spoken.append(sentence)
|
|
181
|
+
|
|
182
|
+
async def _session_lost(self, provider: str, reason: str) -> None:
|
|
183
|
+
logger.error(f"VoxchainPipeline: {provider} session closed ({reason}), stopping")
|
|
184
|
+
await self.output.close(f"{provider}: {reason}")
|
voxchain/core/replies.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Tracking the agent's replies from the user's turn to the last of their audio, and what barge-in should undo."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from collections import deque
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(eq=False)
|
|
9
|
+
class Reply:
|
|
10
|
+
"""The agent's reply to one committed user turn."""
|
|
11
|
+
|
|
12
|
+
text: str
|
|
13
|
+
task: asyncio.Task | None = None
|
|
14
|
+
first_heard_at: float | None = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class Interruption:
|
|
19
|
+
"""What a barge-in undoes: replies to cancel, whether TTS must drop work, and user text to merge back."""
|
|
20
|
+
|
|
21
|
+
cancelled: list[Reply]
|
|
22
|
+
cancel_synthesis: bool
|
|
23
|
+
resume_text: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ReplyTracker:
|
|
27
|
+
"""Follows each reply through generation, synthesis and playback.
|
|
28
|
+
|
|
29
|
+
A reply is *active* from its user turn until TTS has synthesized all of its
|
|
30
|
+
audio. On barge-in, a reply the user heard for no longer than
|
|
31
|
+
``resume_window`` seconds means the end-of-turn call was premature: its user
|
|
32
|
+
text is handed back to be merged with what they say next. ``None`` never merges.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, resume_window: float | None) -> None:
|
|
36
|
+
self.resume_window = resume_window
|
|
37
|
+
self._replies: list[Reply] = []
|
|
38
|
+
self._speaking: deque[Reply] = deque()
|
|
39
|
+
self._last_heard: Reply | None = None
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def active(self) -> bool:
|
|
43
|
+
"""Whether any reply is still being generated or synthesized."""
|
|
44
|
+
return bool(self._replies)
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def latest(self) -> Reply | None:
|
|
48
|
+
"""The most recent active reply, which the next one must wait for."""
|
|
49
|
+
return self._replies[-1] if self._replies else None
|
|
50
|
+
|
|
51
|
+
def start(self, text: str) -> Reply:
|
|
52
|
+
"""Track a reply to a newly committed user turn."""
|
|
53
|
+
reply = Reply(text)
|
|
54
|
+
self._replies.append(reply)
|
|
55
|
+
return reply
|
|
56
|
+
|
|
57
|
+
def spoke(self, reply: Reply) -> None:
|
|
58
|
+
"""A reply sent its first sentence to TTS. Replies are synthesized in this order."""
|
|
59
|
+
self._speaking.append(reply)
|
|
60
|
+
|
|
61
|
+
def finished_generating(self, reply: Reply) -> None:
|
|
62
|
+
"""A reply's agent turn ended. One that never spoke is done; one that did waits for TTS."""
|
|
63
|
+
if reply not in self._speaking and reply in self._replies:
|
|
64
|
+
self._replies.remove(reply)
|
|
65
|
+
|
|
66
|
+
def audio_played(self, now: float) -> None:
|
|
67
|
+
"""Audio of the reply now being synthesized reached the client."""
|
|
68
|
+
if self._speaking and self._speaking[0].first_heard_at is None:
|
|
69
|
+
self._speaking[0].first_heard_at = now
|
|
70
|
+
self._last_heard = self._speaking[0]
|
|
71
|
+
|
|
72
|
+
def synthesized(self) -> None:
|
|
73
|
+
"""TTS finished the oldest speaking reply's audio."""
|
|
74
|
+
if self._speaking:
|
|
75
|
+
done = self._speaking.popleft()
|
|
76
|
+
if done in self._replies:
|
|
77
|
+
self._replies.remove(done)
|
|
78
|
+
|
|
79
|
+
def interrupt(self, now: float) -> Interruption:
|
|
80
|
+
"""The user barged in: forget every reply and say what to undo."""
|
|
81
|
+
candidates = self._replies if self._last_heard in self._replies else [self._last_heard, *self._replies]
|
|
82
|
+
window = self.resume_window
|
|
83
|
+
resumed = [
|
|
84
|
+
reply.text
|
|
85
|
+
for reply in candidates
|
|
86
|
+
if reply is not None
|
|
87
|
+
and window is not None
|
|
88
|
+
and (reply.first_heard_at is None or now - reply.first_heard_at <= window)
|
|
89
|
+
]
|
|
90
|
+
interruption = Interruption(
|
|
91
|
+
cancelled=list(self._replies), cancel_synthesis=self.active, resume_text=" ".join(resumed)
|
|
92
|
+
)
|
|
93
|
+
self._replies.clear()
|
|
94
|
+
self._speaking.clear()
|
|
95
|
+
self._last_heard = None
|
|
96
|
+
return interruption
|
voxchain/core/text.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Turning streamed agent tokens into speakable sentences, so TTS can start before the agent finishes."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
BOUNDARY = re.compile(r"(?<=[.!?;:,])\s+|\n+")
|
|
6
|
+
HTML_TAG = re.compile(r"<[^>]+>")
|
|
7
|
+
MD_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)")
|
|
8
|
+
LIST_MARKER = re.compile(r"^\s*[-+*]\s+", re.MULTILINE)
|
|
9
|
+
MD_SYMBOLS = re.compile(r"[*_`#|~>•·▪]+|-{2,}")
|
|
10
|
+
WHITESPACE = re.compile(r"\s+")
|
|
11
|
+
SPACE_BEFORE_PUNCT = re.compile(r"\s+([,.;:!?])")
|
|
12
|
+
WORD_CHAR = re.compile(r"\w")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def to_speakable(text: str) -> str:
|
|
16
|
+
"""Strip markdown and HTML so ``text`` reads naturally aloud.
|
|
17
|
+
|
|
18
|
+
Returns ``""`` when nothing speakable is left (e.g. a table separator ``|---|``),
|
|
19
|
+
which TTS engines reject, so callers skip empty results.
|
|
20
|
+
"""
|
|
21
|
+
text = HTML_TAG.sub(" ", text)
|
|
22
|
+
text = MD_LINK.sub(r"\1", text)
|
|
23
|
+
text = LIST_MARKER.sub("", text)
|
|
24
|
+
text = MD_SYMBOLS.sub(" ", text)
|
|
25
|
+
text = WHITESPACE.sub(" ", text).strip()
|
|
26
|
+
text = SPACE_BEFORE_PUNCT.sub(r"\1", text)
|
|
27
|
+
return text if WORD_CHAR.search(text) else ""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class SentenceSegmenter:
|
|
31
|
+
"""Accumulates streamed tokens and emits complete, speakable sentences.
|
|
32
|
+
|
|
33
|
+
A piece ends at sentence or clause punctuation (``. ! ? ; : ,``) followed
|
|
34
|
+
by whitespace, or at a newline. Pieces shorter than ``min_chars`` are
|
|
35
|
+
merged with the next one, so abbreviations ("Dr. Smith") and short list
|
|
36
|
+
items don't become choppy standalone TTS requests.
|
|
37
|
+
|
|
38
|
+
Example:
|
|
39
|
+
>>> segmenter = SentenceSegmenter()
|
|
40
|
+
>>> segmenter.push("Hello there, how are")
|
|
41
|
+
[]
|
|
42
|
+
>>> segmenter.push(" you today? I'm fine")
|
|
43
|
+
['Hello there, how are you today?']
|
|
44
|
+
>>> segmenter.flush()
|
|
45
|
+
["I'm fine"]
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(self, min_chars: int = 20) -> None:
|
|
49
|
+
self.min_chars = min_chars
|
|
50
|
+
self._buffer = ""
|
|
51
|
+
self._carry = ""
|
|
52
|
+
|
|
53
|
+
def push(self, token: str) -> list[str]:
|
|
54
|
+
"""Add a token and return the sentences it completed (often none)."""
|
|
55
|
+
self._buffer += token
|
|
56
|
+
*complete, self._buffer = BOUNDARY.split(self._buffer)
|
|
57
|
+
sentences = []
|
|
58
|
+
for piece in complete:
|
|
59
|
+
self._carry = f"{self._carry} {piece}".strip()
|
|
60
|
+
if len(to_speakable(self._carry)) >= self.min_chars:
|
|
61
|
+
sentences.extend(self._take_carry())
|
|
62
|
+
return sentences
|
|
63
|
+
|
|
64
|
+
def flush(self) -> list[str]:
|
|
65
|
+
"""Return whatever speakable text is left once the stream has ended."""
|
|
66
|
+
self._carry = f"{self._carry} {self._buffer}".strip()
|
|
67
|
+
self._buffer = ""
|
|
68
|
+
return self._take_carry()
|
|
69
|
+
|
|
70
|
+
def _take_carry(self) -> list[str]:
|
|
71
|
+
speakable, self._carry = to_speakable(self._carry), ""
|
|
72
|
+
return [speakable] if speakable else []
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""Deciding when a user's turn is over and should go to the agent."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
|
|
7
|
+
from voxchain.turn import EndOfTurnDetector
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class EndOfTurnGate:
|
|
13
|
+
"""Holds finalized transcripts until the user's turn is judged complete.
|
|
14
|
+
|
|
15
|
+
Without a detector, every transcript is committed immediately (merged with
|
|
16
|
+
any restored text). With one,
|
|
17
|
+
the gate runs this state machine:
|
|
18
|
+
|
|
19
|
+
- **Transcript arrives** -> append it to the held text and ask the
|
|
20
|
+
detector. Complete -> commit. Incomplete -> wait ``timeout`` seconds.
|
|
21
|
+
- **Speech starts** -> pause the wait (the user is continuing). If
|
|
22
|
+
nothing is held, tell the detector a fresh turn has started.
|
|
23
|
+
- **Speech ends without a transcript** (e.g. noise) -> resume the wait.
|
|
24
|
+
- **Wait expires** -> commit whatever is held.
|
|
25
|
+
|
|
26
|
+
If a committed turn's reply is cancelled before it was spoken, the
|
|
27
|
+
pipeline hands its text back via :meth:`restore`, so it merges with the
|
|
28
|
+
user's continuation instead of being lost.
|
|
29
|
+
|
|
30
|
+
A detector error counts as "complete", so the user is never left
|
|
31
|
+
unanswered.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
detector: EndOfTurnDetector | None,
|
|
37
|
+
timeout: float,
|
|
38
|
+
commit: Callable[[str], None],
|
|
39
|
+
) -> None:
|
|
40
|
+
"""Create the gate."""
|
|
41
|
+
self.detector = detector
|
|
42
|
+
self.timeout = timeout
|
|
43
|
+
self._commit = commit
|
|
44
|
+
self._held = ""
|
|
45
|
+
self._task: asyncio.Task | None = None
|
|
46
|
+
|
|
47
|
+
def push_audio(self, chunk: bytes) -> None:
|
|
48
|
+
"""Forward an input audio chunk to the detector, if any."""
|
|
49
|
+
if self.detector is not None:
|
|
50
|
+
self.detector.push_audio(chunk)
|
|
51
|
+
|
|
52
|
+
def on_speech_start(self) -> None:
|
|
53
|
+
"""Handle the user starting to speak."""
|
|
54
|
+
if self.detector is not None and not self._held:
|
|
55
|
+
self.detector.start_turn()
|
|
56
|
+
self._cancel()
|
|
57
|
+
|
|
58
|
+
def on_speech_end(self) -> None:
|
|
59
|
+
"""Handle the user going quiet."""
|
|
60
|
+
if self._held and (self._task is None or self._task.done()):
|
|
61
|
+
self._start(check=False)
|
|
62
|
+
|
|
63
|
+
def on_transcript(self, text: str) -> None:
|
|
64
|
+
"""Handle a finalized transcript."""
|
|
65
|
+
self._held = f"{self._held} {text}".strip()
|
|
66
|
+
if self.detector is None:
|
|
67
|
+
self._cancel()
|
|
68
|
+
text, self._held = self._held, ""
|
|
69
|
+
self._commit(text)
|
|
70
|
+
return
|
|
71
|
+
self._start(check=True)
|
|
72
|
+
|
|
73
|
+
def restore(self, text: str) -> None:
|
|
74
|
+
"""Put back the text of a committed turn whose reply was cancelled before it was spoken.
|
|
75
|
+
|
|
76
|
+
It is placed ahead of anything already held and is committed together
|
|
77
|
+
with the user's next words (or after the timeout if none come).
|
|
78
|
+
"""
|
|
79
|
+
self._held = f"{text} {self._held}".strip()
|
|
80
|
+
|
|
81
|
+
def close(self) -> None:
|
|
82
|
+
"""Abandon any pending decision. Held text is discarded."""
|
|
83
|
+
self._cancel()
|
|
84
|
+
self._held = ""
|
|
85
|
+
|
|
86
|
+
def _cancel(self) -> None:
|
|
87
|
+
if self._task is not None and not self._task.done():
|
|
88
|
+
self._task.cancel()
|
|
89
|
+
|
|
90
|
+
def _start(self, check: bool) -> None:
|
|
91
|
+
self._cancel()
|
|
92
|
+
self._task = asyncio.create_task(self._resolve(check))
|
|
93
|
+
|
|
94
|
+
async def _resolve(self, check: bool) -> None:
|
|
95
|
+
complete = False
|
|
96
|
+
if check and self.detector is not None:
|
|
97
|
+
try:
|
|
98
|
+
complete = await self.detector.is_end_of_turn(self._held)
|
|
99
|
+
except Exception:
|
|
100
|
+
logger.exception("EndOfTurnGate: Detector failed, treating turn as complete")
|
|
101
|
+
complete = True
|
|
102
|
+
if not complete:
|
|
103
|
+
if check:
|
|
104
|
+
logger.debug("EndOfTurnGate: Holding %r, sounds incomplete", self._held)
|
|
105
|
+
logger.debug("EndOfTurnGate: Waiting %.1fs for the user to continue", self.timeout)
|
|
106
|
+
await asyncio.sleep(self.timeout)
|
|
107
|
+
|
|
108
|
+
# No await between taking the text and committing it, so cancellation can't split them.
|
|
109
|
+
text, self._held = self._held, ""
|
|
110
|
+
if text:
|
|
111
|
+
self._commit(text)
|
voxchain/events.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Events shared by every kind of provider."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
from voxchain.session import SessionClosed
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class SpeechStarted:
|
|
10
|
+
"""The user started talking. Used for barge-in."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class SpeechStopped:
|
|
15
|
+
"""The user stopped talking."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
__all__ = ["SessionClosed", "SpeechStarted", "SpeechStopped"]
|
voxchain/exceptions.py
ADDED
|
File without changes
|