voxchain 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. voxchain-0.1.0/LICENSE +21 -0
  2. voxchain-0.1.0/PKG-INFO +233 -0
  3. voxchain-0.1.0/README.md +202 -0
  4. voxchain-0.1.0/pyproject.toml +101 -0
  5. voxchain-0.1.0/setup.cfg +4 -0
  6. voxchain-0.1.0/tests/test_conversation_history.py +54 -0
  7. voxchain-0.1.0/tests/test_openai_realtime_events.py +168 -0
  8. voxchain-0.1.0/tests/test_openai_realtime_session.py +81 -0
  9. voxchain-0.1.0/tests/test_openai_realtime_session_lifecycle.py +225 -0
  10. voxchain-0.1.0/tests/test_openai_stt.py +111 -0
  11. voxchain-0.1.0/tests/test_openai_tts.py +115 -0
  12. voxchain-0.1.0/tests/test_output_channel.py +81 -0
  13. voxchain-0.1.0/tests/test_pcm16_resampler.py +31 -0
  14. voxchain-0.1.0/tests/test_playback_tracker.py +68 -0
  15. voxchain-0.1.0/tests/test_provider_session.py +122 -0
  16. voxchain-0.1.0/tests/test_realtime_pipeline.py +310 -0
  17. voxchain-0.1.0/tests/test_reconnecting_session.py +59 -0
  18. voxchain-0.1.0/tests/test_reply_tracker.py +64 -0
  19. voxchain-0.1.0/tests/test_response_gate.py +86 -0
  20. voxchain-0.1.0/tests/test_sarvam_stt.py +89 -0
  21. voxchain-0.1.0/tests/test_sarvam_tts.py +94 -0
  22. voxchain-0.1.0/tests/test_tasks.py +35 -0
  23. voxchain-0.1.0/tests/test_tool_runner.py +52 -0
  24. voxchain-0.1.0/tests/test_voice_gate.py +60 -0
  25. voxchain-0.1.0/tests/test_voxkit_pipeline.py +225 -0
  26. voxchain-0.1.0/voxchain/__init__.py +40 -0
  27. voxchain-0.1.0/voxchain/audio.py +62 -0
  28. voxchain-0.1.0/voxchain/core/__init__.py +18 -0
  29. voxchain-0.1.0/voxchain/core/agent.py +20 -0
  30. voxchain-0.1.0/voxchain/core/pipeline.py +184 -0
  31. voxchain-0.1.0/voxchain/core/replies.py +96 -0
  32. voxchain-0.1.0/voxchain/core/text.py +72 -0
  33. voxchain-0.1.0/voxchain/core/turn_gate.py +111 -0
  34. voxchain-0.1.0/voxchain/events.py +18 -0
  35. voxchain-0.1.0/voxchain/exceptions.py +0 -0
  36. voxchain-0.1.0/voxchain/output.py +110 -0
  37. voxchain-0.1.0/voxchain/protocols/__init__.py +6 -0
  38. voxchain-0.1.0/voxchain/protocols/openai_realtime.py +91 -0
  39. voxchain-0.1.0/voxchain/protocols/sarvam.py +67 -0
  40. voxchain-0.1.0/voxchain/realtime/__init__.py +48 -0
  41. voxchain-0.1.0/voxchain/realtime/base.py +65 -0
  42. voxchain-0.1.0/voxchain/realtime/events.py +68 -0
  43. voxchain-0.1.0/voxchain/realtime/history.py +114 -0
  44. voxchain-0.1.0/voxchain/realtime/openai.py +361 -0
  45. voxchain-0.1.0/voxchain/realtime/pipeline.py +160 -0
  46. voxchain-0.1.0/voxchain/realtime/playback.py +118 -0
  47. voxchain-0.1.0/voxchain/realtime/response_gate.py +85 -0
  48. voxchain-0.1.0/voxchain/realtime/tools.py +62 -0
  49. voxchain-0.1.0/voxchain/session.py +261 -0
  50. voxchain-0.1.0/voxchain/stt/__init__.py +19 -0
  51. voxchain-0.1.0/voxchain/stt/base.py +33 -0
  52. voxchain-0.1.0/voxchain/stt/events.py +22 -0
  53. voxchain-0.1.0/voxchain/stt/openai.py +187 -0
  54. voxchain-0.1.0/voxchain/stt/sarvam.py +116 -0
  55. voxchain-0.1.0/voxchain/tasks.py +24 -0
  56. voxchain-0.1.0/voxchain/tts/__init__.py +18 -0
  57. voxchain-0.1.0/voxchain/tts/base.py +43 -0
  58. voxchain-0.1.0/voxchain/tts/events.py +20 -0
  59. voxchain-0.1.0/voxchain/tts/openai.py +240 -0
  60. voxchain-0.1.0/voxchain/tts/sarvam.py +133 -0
  61. voxchain-0.1.0/voxchain/turn/__init__.py +6 -0
  62. voxchain-0.1.0/voxchain/turn/base.py +55 -0
  63. voxchain-0.1.0/voxchain/turn/smart_turn.py +133 -0
  64. voxchain-0.1.0/voxchain/vad/__init__.py +6 -0
  65. voxchain-0.1.0/voxchain/vad/base.py +0 -0
  66. voxchain-0.1.0/voxchain/vad/gate.py +104 -0
  67. voxchain-0.1.0/voxchain/vad/silero.py +87 -0
  68. voxchain-0.1.0/voxchain.egg-info/PKG-INFO +233 -0
  69. voxchain-0.1.0/voxchain.egg-info/SOURCES.txt +70 -0
  70. voxchain-0.1.0/voxchain.egg-info/dependency_links.txt +1 -0
  71. voxchain-0.1.0/voxchain.egg-info/requires.txt +21 -0
  72. voxchain-0.1.0/voxchain.egg-info/top_level.txt +1 -0
voxchain-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ayush Sachan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,233 @@
1
+ Metadata-Version: 2.4
2
+ Name: voxchain
3
+ Version: 0.1.0
4
+ Summary: A python library for building voice agents
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/sachanayush47/voxchain
7
+ Project-URL: Documentation, https://sachanayush47.github.io/voxchain/
8
+ Project-URL: Repository, https://github.com/sachanayush47/voxchain
9
+ Project-URL: Issues, https://github.com/sachanayush47/voxchain/issues
10
+ Requires-Python: >=3.13
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Requires-Dist: langchain>=1.3.11
14
+ Requires-Dist: websockets>=15.0.1
15
+ Provides-Extra: docs
16
+ Requires-Dist: mkdocs>=1.6.1; extra == "docs"
17
+ Requires-Dist: mkdocs-material>=9.6.0; extra == "docs"
18
+ Requires-Dist: mkdocstrings[python]>=0.27.0; extra == "docs"
19
+ Provides-Extra: vad
20
+ Requires-Dist: huggingface-hub>=1.33.0; extra == "vad"
21
+ Requires-Dist: numpy>=2.5.3; extra == "vad"
22
+ Requires-Dist: onnxruntime>=1.30.0; extra == "vad"
23
+ Provides-Extra: smart-turn
24
+ Requires-Dist: huggingface-hub>=1.33.0; extra == "smart-turn"
25
+ Requires-Dist: numpy>=2.5.3; extra == "smart-turn"
26
+ Requires-Dist: onnxruntime>=1.30.0; extra == "smart-turn"
27
+ Requires-Dist: transformers>=5.18.0; extra == "smart-turn"
28
+ Provides-Extra: all
29
+ Requires-Dist: voxchain[smart-turn,vad]; extra == "all"
30
+ Dynamic: license-file
31
+
32
+ # voxchain
33
+
34
+ A thin, event-driven Python library for building real-time voice agents on **LangChain / LangGraph**.
35
+
36
+ voxchain handles the hard parts of a voice conversation (streaming audio in and out, turn-taking, barge-in, reconnects) and leaves the "brain" to you. There are two ways to build an agent:
37
+
38
+ - **`VoxchainPipeline`** chains streaming speech-to-text, any LangGraph agent, and streaming text-to-speech.
39
+ - **`RealtimePipeline`** runs one speech-to-speech model that hears, reasons, calls your LangChain tools and speaks.
40
+
41
+ Both send your client the same small set of events, so playback code works with either.
42
+
43
+ **Design goals:**
44
+ - **Thin.** No transport bus, no call/room management. Providers report voice activity themselves.
45
+ - **LangChain/LangGraph-first.** Bring any compiled graph (e.g. from `langchain.agents.create_agent`) or LangChain tools.
46
+ - **Explicit events.** Providers and pipelines talk in small typed events, so turns, interrupts and lost sessions are easy to follow.
47
+
48
+ > **Status:** pre-alpha (`0.1.0`); the API will change. Providers: Sarvam AI (STT, TTS) and OpenAI (STT, TTS, speech-to-speech).
49
+
50
+ ## Installation
51
+
52
+ ```bash
53
+ pip install voxchain
54
+ ```
55
+
56
+ | Extra | Adds | Needed for |
57
+ |---|---|---|
58
+ | `vad` | `onnxruntime`, `huggingface-hub`, `numpy` | `OpenAISTTProvider`, which runs Silero voice detection locally |
59
+ | `smart-turn` | `onnxruntime`, `transformers`, `huggingface-hub`, `numpy` | [End-of-turn check](#end-of-turn-check) (`PipecatSmartTurnDetector`) |
60
+ | `all` | both | everything |
61
+
62
+ Requires Python 3.13+.
63
+
64
+ ## Quick start
65
+
66
+ ```python
67
+ import asyncio
68
+ import os
69
+
70
+ from langchain.agents import create_agent
71
+ from langchain_groq import ChatGroq
72
+
73
+ from voxchain import Output, PlayAudio, SarvamSTTProvider, SarvamTTSProvider, StopPlayback, VoxchainPipeline
74
+
75
+ stt = SarvamSTTProvider(api_key=os.environ["SARVAM_API_KEY"], language_code="en-IN", sample_rate=16000)
76
+ tts = SarvamTTSProvider(api_key=os.environ["SARVAM_API_KEY"], speaker="priya", language_code="en-IN")
77
+ agent = create_agent(model=ChatGroq(model="llama-3.3-70b-versatile"), tools=[])
78
+
79
+
80
+ async def play(event: Output) -> None:
81
+ match event:
82
+ case PlayAudio(data=data):
83
+ speaker.write(data) # raw PCM16 at tts.sample_rate
84
+ case StopPlayback():
85
+ speaker.clear() # the user barged in
86
+
87
+
88
+ async def main() -> None:
89
+ pipeline = VoxchainPipeline(stt, tts, agent, play)
90
+ await pipeline.run(microphone_audio_stream()) # PCM16 at stt.sample_rate
91
+
92
+
93
+ asyncio.run(main())
94
+ ```
95
+
96
+ [`main.py`](main.py) is a complete, runnable version using `sounddevice` for the microphone and speakers. `VOXCHAIN_PROVIDER=openai` switches it to OpenAI STT/TTS, and `VOXCHAIN_PROVIDER=openai-realtime` to speech-to-speech.
97
+
98
+ ### What your callback receives
99
+
100
+ | Event | Meaning |
101
+ |---|---|
102
+ | `PlayAudio(data)` | Play this raw PCM16 audio after anything already queued. |
103
+ | `StopPlayback()` | The user barged in: stop now and drop anything queued. |
104
+ | `ReplyFinished()` | The current reply's audio has all been sent. |
105
+ | `StreamClosed(reason)` | A provider's session was lost for good; the pipeline is stopping. |
106
+
107
+ The callback runs in its own task, so it can block on playback without delaying barge-in.
108
+
109
+ ## Providers
110
+
111
+ Every provider takes keyword arguments; the signature is the full reference. The settings people usually change:
112
+
113
+ **`SarvamSTTProvider`** uses Sarvam's streaming `saaras` / `saarika` models, with Sarvam's voice activity signals driving barge-in.
114
+ - `model` (`"saaras:v3"`), `mode` (`"transcribe"`, `"translate"`, …), `language_code` (`"unknown"` auto-detects).
115
+ - `sample_rate`: 8000 or 16000.
116
+ - `extra_params` passes any other Sarvam parameter through as-is, e.g. `{"interrupt_min_speech_frames": 4}` to make barge-in less twitchy.
117
+ - `await stt.flush()` finalizes buffered audio without waiting for silence (needs `flush_signal=True`).
118
+
119
+ **`SarvamTTSProvider`** uses Sarvam's `bulbul` voices, streamed as raw PCM16.
120
+ - `speaker` (required; must belong to the model), `language_code`, `model` (`"bulbul:v3"`).
121
+ - `sample_rate`: 8000, 16000, 22050 or 24000.
122
+ - `pace`, plus model-specific options: `temperature` and `dict_id` on v3, `pitch` and `loudness` on v2. Options left unset use Sarvam's defaults.
123
+ - Sarvam can't cancel synthesis on a socket, so on barge-in the provider replaces the socket and drops whatever the old one was still sending.
124
+
125
+ **`OpenAISTTProvider`** uses OpenAI's realtime transcription (`gpt-live-transcribe`). Needs the `vad` extra.
126
+ - OpenAI's current transcription models have no server-side turn detection, so a local Silero VAD decides when the user starts and stops talking. Only speech is streamed, so silence isn't billed.
127
+ - `languages`, `keywords` (terms to recognize), `prompt`.
128
+ - `vad_threshold`, `vad_start_ms`, `vad_stop_ms` tune turn detection; raise `vad_start_ms` if coughs trigger barge-in.
129
+ - Transcripts always arrive in speech order.
130
+
131
+ **`OpenAITTSProvider`** reads text aloud with `gpt-realtime-2.1-mini`, OpenAI's named replacement for its retired TTS models (`tts-1`, `gpt-4o-mini-tts` shut down 2027-01-06).
132
+ - `voice` (`"marin"` or `"cedar"` recommended), `instructions` (tone and pacing), `speed`.
133
+ - A realtime model has no "speak this text" call. It is told to read the text verbatim, and with `verify_transcript=True` (the default) a paraphrase is logged as a warning.
134
+
135
+ All providers reconnect on their own when their connection drops (5 attempts). `StreamClosed` reaches your callback only when that fails.
136
+
137
+ ## How the cascaded pipeline works
138
+
139
+ ```
140
+ audio in ──▶ STTProvider ──▶ VoxchainPipeline ──▶ LangGraph agent
141
+ │ │ streamed tokens
142
+ └── sentences ──▶ TTSProvider ──▶ your callback
143
+ ```
144
+
145
+ 1. Audio goes to the STT provider (and the optional end-of-turn detector).
146
+ 2. A final transcript starts an agent turn. The reply streams to TTS **sentence by sentence**, so speech starts before the agent finishes. Only the assistant's own text is spoken (never tool output), with markdown stripped. Replies run one at a time, in order.
147
+ 3. **Barge-in:** when the user starts talking over a reply, the reply is cancelled, TTS drops its pending work, and your callback gets `StopPlayback`. If they interrupt within `resume_window` seconds (default 2) of first hearing the reply, the end-of-turn call was probably premature. Their previous turn is then merged with what they say next ("I want a ticket to" + "Bangkok"). Later interruptions ("stop", "ok thanks") drop it. `interrupt=False` lets replies finish.
148
+
149
+ `VoxchainPipeline(stt, tts, agent, callback, *, thread_id="default", interrupt=True, resume_window=2.0, end_of_turn=None, end_of_turn_timeout=2.0)`. `thread_id` keys the agent's checkpointed memory; use one per conversation.
150
+
151
+ ### End-of-turn check
152
+
153
+ Voice detection ends a turn on silence alone, so a pause mid-sentence ("book it for... um...") gets answered too early. [Smart Turn v3](https://github.com/pipecat-ai/smart-turn) is a small local audio model that judges from intonation whether the user has finished:
154
+
155
+ ```python
156
+ from voxchain import PipecatSmartTurnDetector
157
+
158
+ pipeline = VoxchainPipeline(stt, tts, agent, play, end_of_turn=PipecatSmartTurnDetector(sample_rate=16000))
159
+ ```
160
+
161
+ A transcript that sounds unfinished is held for up to `end_of_turn_timeout` seconds of silence while more speech is appended and re-scored. It needs `pip install "voxchain[smart-turn]"`. Use one detector per pipeline.
162
+
163
+ ## Speech-to-speech (OpenAI Realtime)
164
+
165
+ `RealtimePipeline` runs a single `gpt-realtime-2.1` session that hears the user, reasons, calls your tools and speaks. This gives the lowest latency and the most natural turn-taking. The model is the brain: you give it instructions and LangChain tools rather than a graph.
166
+
167
+ ```python
168
+ from langchain_core.tools import tool
169
+
170
+ from voxchain import OpenAIRealtimeProvider, RealtimePipeline
171
+
172
+
173
+ @tool
174
+ def book_appointment(pet_name: str, date: str) -> str:
175
+ """Book a vet appointment for a pet on a date (YYYY-MM-DD)."""
176
+ return f"Booked {pet_name} on {date}."
177
+
178
+
179
+ provider = OpenAIRealtimeProvider(
180
+ api_key=os.environ["OPENAI_API_KEY"],
181
+ instructions="# Role and Objective\nYou are a friendly vet clinic receptionist. Keep answers short.",
182
+ )
183
+ pipeline = RealtimePipeline(provider, play, tools=[book_appointment], input_sample_rate=16000)
184
+ await pipeline.run(microphone_audio_stream())
185
+ ```
186
+
187
+ What it handles:
188
+ - **Turn-taking:** OpenAI's `semantic_vad` decides when the user has finished.
189
+ - **Barge-in:** when the user starts talking, playback stops and the conversation is cut to what they actually heard. Partly heard replies are truncated, unheard ones deleted, so the model never believes it said something the user missed.
190
+ - **Tools:** calls run concurrently and their results go back to the model. If a tool raises, the model gets the error. A call from a reply the user interrupted is not run, which matters for bookings.
191
+ - **Reconnects:** after a drop, or before OpenAI's 60-minute session limit, the conversation is replayed into a fresh session.
192
+ - **Transcripts:** pass `on_transcript=async def(role, text)` for `"user"` and `"agent"` transcripts.
193
+
194
+ Common settings: `model` (`"gpt-realtime-2.1"` or the cheaper `-mini`), `voice`, `turn_detection` (`"semantic_vad"` / `"server_vad"`), `interrupt_response`, `noise_reduction` (`"near_field"` / `"far_field"`), `reasoning_effort` (OpenAI suggests `"low"`), and `truncation` (e.g. `0.8` to keep 80% of the context when trimming). `PipecatSmartTurnDetector` doesn't apply here.
195
+
196
+ With either pipeline, use headphones or echo-cancelled audio capture. Otherwise the agent can hear itself and interrupt its own reply.
197
+
198
+ ## Logging
199
+
200
+ voxchain logs routine activity at `DEBUG` (transcripts, replies, voice activity) and only real problems at `WARNING`/`ERROR`. To watch a conversation:
201
+
202
+ ```python
203
+ logging.getLogger("voxchain").setLevel(logging.DEBUG)
204
+ ```
205
+
206
+ ## Adding a provider
207
+
208
+ Implement the interface in `voxchain/stt/base.py`, `voxchain/tts/base.py` or `voxchain/realtime/base.py`. Compose the shared pieces rather than inheriting behaviour:
209
+ - `ProviderSession` (`voxchain/session.py`) gives you the socket, reconnects and the event stream. You supply `open` (connect and configure) and `handle` (one message to events).
210
+ - `VoiceGate` (`voxchain/vad/gate.py`) if the vendor doesn't report voice activity.
211
+ - For speech-to-speech, `ResponseGate` and `ConversationHistory`.
212
+
213
+ Put the vendor's wire vocabulary (URLs, message types as `StrEnum`s, handshake) in `voxchain/protocols/<vendor>.py`. The existing providers are compact references. The pipelines only depend on the interfaces.
214
+
215
+ ## Documentation
216
+
217
+ API reference, generated from the docstrings: **[sachanayush47.github.io/voxchain](https://sachanayush47.github.io/voxchain/)**. Build it locally with `uv sync --extra docs && uv run mkdocs serve`.
218
+
219
+ ## Development
220
+
221
+ ```bash
222
+ uv sync # dependencies and dev tooling
223
+ uv run pytest # test suite: no network or credentials needed
224
+ uv run ruff check --fix . # lint
225
+ uv run ruff format . # format
226
+ uv run pre-commit install # once per clone: lint and format on commit
227
+ ```
228
+
229
+ Contributor conventions (coding standards, design principles, testing) are in [`AGENTS.md`](AGENTS.md).
230
+
231
+ ## License
232
+
233
+ MIT. See [`LICENSE`](LICENSE).
@@ -0,0 +1,202 @@
1
+ # voxchain
2
+
3
+ A thin, event-driven Python library for building real-time voice agents on **LangChain / LangGraph**.
4
+
5
+ voxchain handles the hard parts of a voice conversation (streaming audio in and out, turn-taking, barge-in, reconnects) and leaves the "brain" to you. There are two ways to build an agent:
6
+
7
+ - **`VoxchainPipeline`** chains streaming speech-to-text, any LangGraph agent, and streaming text-to-speech.
8
+ - **`RealtimePipeline`** runs one speech-to-speech model that hears, reasons, calls your LangChain tools and speaks.
9
+
10
+ Both send your client the same small set of events, so playback code works with either.
11
+
12
+ **Design goals:**
13
+ - **Thin.** No transport bus, no call/room management. Providers report voice activity themselves.
14
+ - **LangChain/LangGraph-first.** Bring any compiled graph (e.g. from `langchain.agents.create_agent`) or LangChain tools.
15
+ - **Explicit events.** Providers and pipelines talk in small typed events, so turns, interrupts and lost sessions are easy to follow.
16
+
17
+ > **Status:** pre-alpha (`0.1.0`); the API will change. Providers: Sarvam AI (STT, TTS) and OpenAI (STT, TTS, speech-to-speech).
18
+
19
+ ## Installation
20
+
21
+ ```bash
22
+ pip install voxchain
23
+ ```
24
+
25
+ | Extra | Adds | Needed for |
26
+ |---|---|---|
27
+ | `vad` | `onnxruntime`, `huggingface-hub`, `numpy` | `OpenAISTTProvider`, which runs Silero voice detection locally |
28
+ | `smart-turn` | `onnxruntime`, `transformers`, `huggingface-hub`, `numpy` | [End-of-turn check](#end-of-turn-check) (`PipecatSmartTurnDetector`) |
29
+ | `all` | both | everything |
30
+
31
+ Requires Python 3.13+.
32
+
33
+ ## Quick start
34
+
35
+ ```python
36
+ import asyncio
37
+ import os
38
+
39
+ from langchain.agents import create_agent
40
+ from langchain_groq import ChatGroq
41
+
42
+ from voxchain import Output, PlayAudio, SarvamSTTProvider, SarvamTTSProvider, StopPlayback, VoxchainPipeline
43
+
44
+ stt = SarvamSTTProvider(api_key=os.environ["SARVAM_API_KEY"], language_code="en-IN", sample_rate=16000)
45
+ tts = SarvamTTSProvider(api_key=os.environ["SARVAM_API_KEY"], speaker="priya", language_code="en-IN")
46
+ agent = create_agent(model=ChatGroq(model="llama-3.3-70b-versatile"), tools=[])
47
+
48
+
49
+ async def play(event: Output) -> None:
50
+ match event:
51
+ case PlayAudio(data=data):
52
+ speaker.write(data) # raw PCM16 at tts.sample_rate
53
+ case StopPlayback():
54
+ speaker.clear() # the user barged in
55
+
56
+
57
+ async def main() -> None:
58
+ pipeline = VoxchainPipeline(stt, tts, agent, play)
59
+ await pipeline.run(microphone_audio_stream()) # PCM16 at stt.sample_rate
60
+
61
+
62
+ asyncio.run(main())
63
+ ```
64
+
65
+ [`main.py`](main.py) is a complete, runnable version using `sounddevice` for the microphone and speakers. `VOXCHAIN_PROVIDER=openai` switches it to OpenAI STT/TTS, and `VOXCHAIN_PROVIDER=openai-realtime` to speech-to-speech.
66
+
67
+ ### What your callback receives
68
+
69
+ | Event | Meaning |
70
+ |---|---|
71
+ | `PlayAudio(data)` | Play this raw PCM16 audio after anything already queued. |
72
+ | `StopPlayback()` | The user barged in: stop now and drop anything queued. |
73
+ | `ReplyFinished()` | The current reply's audio has all been sent. |
74
+ | `StreamClosed(reason)` | A provider's session was lost for good; the pipeline is stopping. |
75
+
76
+ The callback runs in its own task, so it can block on playback without delaying barge-in.
77
+
78
+ ## Providers
79
+
80
+ Every provider takes keyword arguments; the signature is the full reference. The settings people usually change:
81
+
82
+ **`SarvamSTTProvider`** uses Sarvam's streaming `saaras` / `saarika` models, with Sarvam's voice activity signals driving barge-in.
83
+ - `model` (`"saaras:v3"`), `mode` (`"transcribe"`, `"translate"`, …), `language_code` (`"unknown"` auto-detects).
84
+ - `sample_rate`: 8000 or 16000.
85
+ - `extra_params` passes any other Sarvam parameter through as-is, e.g. `{"interrupt_min_speech_frames": 4}` to make barge-in less twitchy.
86
+ - `await stt.flush()` finalizes buffered audio without waiting for silence (needs `flush_signal=True`).
87
+
88
+ **`SarvamTTSProvider`** uses Sarvam's `bulbul` voices, streamed as raw PCM16.
89
+ - `speaker` (required; must belong to the model), `language_code`, `model` (`"bulbul:v3"`).
90
+ - `sample_rate`: 8000, 16000, 22050 or 24000.
91
+ - `pace`, plus model-specific options: `temperature` and `dict_id` on v3, `pitch` and `loudness` on v2. Options left unset use Sarvam's defaults.
92
+ - Sarvam can't cancel synthesis on a socket, so on barge-in the provider replaces the socket and drops whatever the old one was still sending.
93
+
94
+ **`OpenAISTTProvider`** uses OpenAI's realtime transcription (`gpt-live-transcribe`). Needs the `vad` extra.
95
+ - OpenAI's current transcription models have no server-side turn detection, so a local Silero VAD decides when the user starts and stops talking. Only speech is streamed, so silence isn't billed.
96
+ - `languages`, `keywords` (terms to recognize), `prompt`.
97
+ - `vad_threshold`, `vad_start_ms`, `vad_stop_ms` tune turn detection; raise `vad_start_ms` if coughs trigger barge-in.
98
+ - Transcripts always arrive in speech order.
99
+
100
+ **`OpenAITTSProvider`** reads text aloud with `gpt-realtime-2.1-mini`, OpenAI's named replacement for its retired TTS models (`tts-1`, `gpt-4o-mini-tts` shut down 2027-01-06).
101
+ - `voice` (`"marin"` or `"cedar"` recommended), `instructions` (tone and pacing), `speed`.
102
+ - A realtime model has no "speak this text" call. It is told to read the text verbatim, and with `verify_transcript=True` (the default) a paraphrase is logged as a warning.
103
+
104
+ All providers reconnect on their own when their connection drops (5 attempts). `StreamClosed` reaches your callback only when that fails.
105
+
106
+ ## How the cascaded pipeline works
107
+
108
+ ```
109
+ audio in ──▶ STTProvider ──▶ VoxchainPipeline ──▶ LangGraph agent
110
+ │ │ streamed tokens
111
+ └── sentences ──▶ TTSProvider ──▶ your callback
112
+ ```
113
+
114
+ 1. Audio goes to the STT provider (and the optional end-of-turn detector).
115
+ 2. A final transcript starts an agent turn. The reply streams to TTS **sentence by sentence**, so speech starts before the agent finishes. Only the assistant's own text is spoken (never tool output), with markdown stripped. Replies run one at a time, in order.
116
+ 3. **Barge-in:** when the user starts talking over a reply, the reply is cancelled, TTS drops its pending work, and your callback gets `StopPlayback`. If they interrupt within `resume_window` seconds (default 2) of first hearing the reply, the end-of-turn call was probably premature. Their previous turn is then merged with what they say next ("I want a ticket to" + "Bangkok"). Later interruptions ("stop", "ok thanks") drop it. `interrupt=False` lets replies finish.
117
+
118
+ `VoxchainPipeline(stt, tts, agent, callback, *, thread_id="default", interrupt=True, resume_window=2.0, end_of_turn=None, end_of_turn_timeout=2.0)`. `thread_id` keys the agent's checkpointed memory; use one per conversation.
119
+
120
+ ### End-of-turn check
121
+
122
+ Voice detection ends a turn on silence alone, so a pause mid-sentence ("book it for... um...") gets answered too early. [Smart Turn v3](https://github.com/pipecat-ai/smart-turn) is a small local audio model that judges from intonation whether the user has finished:
123
+
124
+ ```python
125
+ from voxchain import PipecatSmartTurnDetector
126
+
127
+ pipeline = VoxchainPipeline(stt, tts, agent, play, end_of_turn=PipecatSmartTurnDetector(sample_rate=16000))
128
+ ```
129
+
130
+ A transcript that sounds unfinished is held for up to `end_of_turn_timeout` seconds of silence while more speech is appended and re-scored. It needs `pip install "voxchain[smart-turn]"`. Use one detector per pipeline.
131
+
132
+ ## Speech-to-speech (OpenAI Realtime)
133
+
134
+ `RealtimePipeline` runs a single `gpt-realtime-2.1` session that hears the user, reasons, calls your tools and speaks. This gives the lowest latency and the most natural turn-taking. The model is the brain: you give it instructions and LangChain tools rather than a graph.
135
+
136
+ ```python
137
+ from langchain_core.tools import tool
138
+
139
+ from voxchain import OpenAIRealtimeProvider, RealtimePipeline
140
+
141
+
142
+ @tool
143
+ def book_appointment(pet_name: str, date: str) -> str:
144
+ """Book a vet appointment for a pet on a date (YYYY-MM-DD)."""
145
+ return f"Booked {pet_name} on {date}."
146
+
147
+
148
+ provider = OpenAIRealtimeProvider(
149
+ api_key=os.environ["OPENAI_API_KEY"],
150
+ instructions="# Role and Objective\nYou are a friendly vet clinic receptionist. Keep answers short.",
151
+ )
152
+ pipeline = RealtimePipeline(provider, play, tools=[book_appointment], input_sample_rate=16000)
153
+ await pipeline.run(microphone_audio_stream())
154
+ ```
155
+
156
+ What it handles:
157
+ - **Turn-taking:** OpenAI's `semantic_vad` decides when the user has finished.
158
+ - **Barge-in:** when the user starts talking, playback stops and the conversation is cut to what they actually heard. Partly heard replies are truncated, unheard ones deleted, so the model never believes it said something the user missed.
159
+ - **Tools:** calls run concurrently and their results go back to the model. If a tool raises, the model gets the error. A call from a reply the user interrupted is not run, which matters for bookings.
160
+ - **Reconnects:** after a drop, or before OpenAI's 60-minute session limit, the conversation is replayed into a fresh session.
161
+ - **Transcripts:** pass `on_transcript=async def(role, text)` for `"user"` and `"agent"` transcripts.
162
+
163
+ Common settings: `model` (`"gpt-realtime-2.1"` or the cheaper `-mini`), `voice`, `turn_detection` (`"semantic_vad"` / `"server_vad"`), `interrupt_response`, `noise_reduction` (`"near_field"` / `"far_field"`), `reasoning_effort` (OpenAI suggests `"low"`), and `truncation` (e.g. `0.8` to keep 80% of the context when trimming). `PipecatSmartTurnDetector` doesn't apply here.
164
+
165
+ With either pipeline, use headphones or echo-cancelled audio capture. Otherwise the agent can hear itself and interrupt its own reply.
166
+
167
+ ## Logging
168
+
169
+ voxchain logs routine activity at `DEBUG` (transcripts, replies, voice activity) and only real problems at `WARNING`/`ERROR`. To watch a conversation:
170
+
171
+ ```python
172
+ logging.getLogger("voxchain").setLevel(logging.DEBUG)
173
+ ```
174
+
175
+ ## Adding a provider
176
+
177
+ Implement the interface in `voxchain/stt/base.py`, `voxchain/tts/base.py` or `voxchain/realtime/base.py`. Compose the shared pieces rather than inheriting behaviour:
178
+ - `ProviderSession` (`voxchain/session.py`) gives you the socket, reconnects and the event stream. You supply `open` (connect and configure) and `handle` (one message to events).
179
+ - `VoiceGate` (`voxchain/vad/gate.py`) if the vendor doesn't report voice activity.
180
+ - For speech-to-speech, `ResponseGate` and `ConversationHistory`.
181
+
182
+ Put the vendor's wire vocabulary (URLs, message types as `StrEnum`s, handshake) in `voxchain/protocols/<vendor>.py`. The existing providers are compact references. The pipelines only depend on the interfaces.
183
+
184
+ ## Documentation
185
+
186
+ API reference, generated from the docstrings: **[sachanayush47.github.io/voxchain](https://sachanayush47.github.io/voxchain/)**. Build it locally with `uv sync --extra docs && uv run mkdocs serve`.
187
+
188
+ ## Development
189
+
190
+ ```bash
191
+ uv sync # dependencies and dev tooling
192
+ uv run pytest # test suite: no network or credentials needed
193
+ uv run ruff check --fix . # lint
194
+ uv run ruff format . # format
195
+ uv run pre-commit install # once per clone: lint and format on commit
196
+ ```
197
+
198
+ Contributor conventions (coding standards, design principles, testing) are in [`AGENTS.md`](AGENTS.md).
199
+
200
+ ## License
201
+
202
+ MIT. See [`LICENSE`](LICENSE).
@@ -0,0 +1,101 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "voxchain"
7
+ version = "0.1.0"
8
+ description = "A python library for building voice agents"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.13"
13
+ dependencies = [
14
+ "langchain>=1.3.11",
15
+ "websockets>=15.0.1",
16
+ ]
17
+
18
+ [project.urls]
19
+ Homepage = "https://github.com/sachanayush47/voxchain"
20
+ Documentation = "https://sachanayush47.github.io/voxchain/"
21
+ Repository = "https://github.com/sachanayush47/voxchain"
22
+ Issues = "https://github.com/sachanayush47/voxchain/issues"
23
+
24
+ [project.optional-dependencies]
25
+ docs = [
26
+ "mkdocs>=1.6.1",
27
+ "mkdocs-material>=9.6.0",
28
+ "mkdocstrings[python]>=0.27.0",
29
+ ]
30
+ vad = [
31
+ "huggingface-hub>=1.33.0",
32
+ "numpy>=2.5.3",
33
+ "onnxruntime>=1.30.0",
34
+ ]
35
+ smart-turn = [
36
+ "huggingface-hub>=1.33.0",
37
+ "numpy>=2.5.3",
38
+ "onnxruntime>=1.30.0",
39
+ "transformers>=5.18.0",
40
+ ]
41
+ all = [
42
+ "voxchain[vad,smart-turn]",
43
+ ]
44
+
45
+ [tool.setuptools.packages.find]
46
+ include = ["voxchain*"] # Only the library; docs output, tests and scripts stay out of the wheel
47
+
48
+ [dependency-groups]
49
+ dev = [
50
+ "langchain-groq>=1.1.3",
51
+ "pre-commit>=4.6.2",
52
+ "pytest>=8.4.0",
53
+ "pytest-asyncio>=1.2.0",
54
+ "sounddevice>=0.5.5",
55
+ "python-dotenv>=1.2.2",
56
+ "voxchain[vad,smart-turn]",
57
+ ]
58
+
59
+ [tool.pytest.ini_options]
60
+ asyncio_mode = "auto" # Coroutine tests need no decorator
61
+ asyncio_default_fixture_loop_scope = "function"
62
+ testpaths = ["tests"]
63
+ pythonpath = ["."]
64
+ addopts = "--strict-markers"
65
+ faulthandler_timeout = 30 # A hung async test dumps its stack instead of stalling the run
66
+
67
+ [tool.ruff]
68
+ line-length = 120
69
+ target-version = "py313"
70
+ extend-exclude = ["site"]
71
+
72
+ [tool.ruff.lint]
73
+ select = [
74
+ "E", # Style errors (pycodestyle)
75
+ "W", # Style warnings (pycodestyle)
76
+ "F", # Unused and undefined names (pyflakes)
77
+ "I", # Import order (isort)
78
+ "UP", # Modern syntax (pyupgrade)
79
+ "B", # Likely bugs (flake8-bugbear)
80
+ "C4", # Comprehensions (flake8-comprehensions)
81
+ "SIM", # Simplifications (flake8-simplify)
82
+ "RUF", # Ruff's own rules
83
+ "D", # Docstrings (pydocstyle): they are the published API reference
84
+ ]
85
+ ignore = [
86
+ "D107", # __init__ docstring: mkdocstrings merges init into the class, which documents itself
87
+ ]
88
+
89
+ [tool.ruff.lint.pydocstyle]
90
+ convention = "google"
91
+
92
+ [tool.ruff.lint.per-file-ignores]
93
+ # Runnable example, not part of the published API surface.
94
+ "main.py" = ["D100", "D103"]
95
+ # Intentionally-empty stubs (see AGENTS.md) -- nothing to document yet.
96
+ "voxchain/exceptions.py" = ["D100"]
97
+ "voxchain/vad/base.py" = ["D100"]
98
+ "tests/*" = ["D101", "D102", "D103", "D105", "D107"] # Test doubles need no docstrings
99
+
100
+ [tool.uv.sources]
101
+ voxchain = { workspace = true }
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,54 @@
1
+ """ConversationHistory: turns stay in conversation order, and replay only what the user heard."""
2
+
3
+ from voxchain.realtime import ConversationHistory
4
+
5
+
6
+ def test_turns_replay_in_order_even_when_the_user_transcript_arrives_late():
7
+ history = ConversationHistory()
8
+ history.user_turn("u1", awaiting_transcript=True)
9
+ history.assistant_text("a1", "Sure.")
10
+ history.user_transcript("u1", "Can you help?")
11
+ assert [(t.kind, t.text) for t in history.replay()] == [("user", "Can you help?"), ("assistant", "Sure.")]
12
+
13
+
14
+ def test_an_interrupted_reply_replays_only_the_heard_part():
15
+ history = ConversationHistory()
16
+ history.assistant_audio("a1", 2000)
17
+ history.assistant_text("a1", "one two three four five six seven eight nine ten")
18
+ history.heard("a1", 600)
19
+ assert history.replay()[0].heard_text() == "one two three"
20
+
21
+
22
+ def test_removed_and_empty_turns_are_not_replayed():
23
+ history = ConversationHistory()
24
+ history.user_turn("u1", awaiting_transcript=True)
25
+ history.user_transcript("u1", None)
26
+ history.assistant_text("a1", "Hello")
27
+ history.remove("a1")
28
+ assert history.replay() == []
29
+
30
+
31
+ def test_a_tool_result_replays_after_its_call():
32
+ history = ConversationHistory()
33
+ history.function_call("f1", "c1", "get_time", "{}")
34
+ history.function_result("c1", '"noon"')
35
+ assert [t.kind for t in history.replay()] == ["function_call", "function_call_output"]
36
+
37
+
38
+ def test_history_is_unsettled_while_a_transcript_or_tool_result_is_outstanding():
39
+ history = ConversationHistory()
40
+ assert history.settled
41
+ history.user_turn("u1", awaiting_transcript=True)
42
+ assert not history.settled
43
+ history.user_transcript("u1", "hi")
44
+ history.open_call("c1")
45
+ assert not history.settled
46
+ history.function_result("c1", "{}")
47
+ assert history.settled
48
+
49
+
50
+ def test_an_undeliverable_call_stops_being_owed():
51
+ history = ConversationHistory()
52
+ history.open_call("c1")
53
+ history.close_call("c1")
54
+ assert history.settled