livekit-plugins-duplexjev 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit_plugins_duplexjev-0.2.0/.gitignore +7 -0
- livekit_plugins_duplexjev-0.2.0/CHANGELOG.md +23 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/PKG-INFO +15 -3
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/README.md +14 -2
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/livekit/plugins/duplexjev/turn_detector.py +30 -1
- livekit_plugins_duplexjev-0.2.0/livekit/plugins/duplexjev/version.py +1 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/tests/test_turn_detector.py +33 -0
- livekit_plugins_duplexjev-0.1.0/CHANGELOG.md +0 -10
- livekit_plugins_duplexjev-0.1.0/livekit/plugins/duplexjev/version.py +0 -1
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/LICENSE +0 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/examples/agent.py +0 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/livekit/plugins/duplexjev/__init__.py +0 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/livekit/plugins/duplexjev/log.py +0 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/livekit/plugins/duplexjev/py.typed +0 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/pyproject.toml +0 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/tests/conftest.py +0 -0
- {livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/tests/test_live.py +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.2.0 (2026-10-13)
|
|
4
|
+
|
|
5
|
+
- Turn-taking context: `detector.attach(session)` (or `detector.set_assistant_said(text)`) sends the agent's previous
|
|
6
|
+
message with every request. Needs a gateway from `duplexjev>=0.6`. On LiveKit eot-bench (English,
|
|
7
|
+
DuplexJev-32B-Para v1.2) false cut-offs at a 300 ms budget drop from 23.5% to 19.7%.
|
|
8
|
+
|
|
9
|
+
## 0.1.1 (2026-10-12)
|
|
10
|
+
|
|
11
|
+
- The turn probability is now P(finished) + P(backchannel): a short "ok" / "yeah" with nobody else talking also
|
|
12
|
+
ends the user's turn ("still talking" and "asking to wait" still hold the turn). On LiveKit eot-bench
|
|
13
|
+
(DuplexJev-32B-Turn, English) false cut-offs at a 300 ms budget drop from 37% to 30%; Smart Turn test-set accuracy
|
|
14
|
+
is unchanged.
|
|
15
|
+
|
|
16
|
+
## 0.1.0 (2026-10-10)
|
|
17
|
+
|
|
18
|
+
- First release: `duplexjev.TurnDetector`, an audio (streaming) turn detector for LiveKit Agents that asks a
|
|
19
|
+
DuplexJev gateway for the turn state of the current user turn.
|
|
20
|
+
- Extra questions (emotion, non-verbal sound, barge-in, or your own) answered in the same request; see
|
|
21
|
+
`last_answers` / `on_answers`.
|
|
22
|
+
- Optional backchannel opportunity from "What should the assistant do now?" (`backchannel_threshold`).
|
|
23
|
+
- Tested with livekit-agents 1.8.6.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: livekit-plugins-duplexjev
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Audio end-of-turn detection for LiveKit Agents with DuplexJev: turn state, emotion and non-verbal sounds from one request, no transcript needed.
|
|
5
5
|
Project-URL: Homepage, https://github.com/adventists-ai/duplexjev/tree/main/integrations/livekit
|
|
6
6
|
Project-URL: Models, https://huggingface.co/adventists-ai/DuplexJev-4B-Para
|
|
@@ -66,6 +66,18 @@ session = AgentSession(
|
|
|
66
66
|
)
|
|
67
67
|
```
|
|
68
68
|
|
|
69
|
+
**Give it the agent's last message** (new in 0.2.0). Turn-taking is clearly better when the model knows what the
|
|
70
|
+
agent had just said. Keep a reference to the detector and attach it to the session once:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
detector = duplexjev.TurnDetector(api_url="http://localhost:8420")
|
|
74
|
+
session = AgentSession(..., turn_handling={"turn_detection": detector})
|
|
75
|
+
detector.attach(session) # follows conversation_item_added; or detector.set_assistant_said(text)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
It needs a gateway from `duplexjev>=0.6`. On LiveKit eot-bench (English, DuplexJev-32B-Para v1.2) this cuts false
|
|
79
|
+
cut-offs at a 300 ms budget from 23.5% to 19.7%.
|
|
80
|
+
|
|
69
81
|
`AgentSession(turn_detection=duplexjev.TurnDetector(), ...)` works too (LiveKit marks that argument as deprecated).
|
|
70
82
|
A complete agent is in [`examples/agent.py`](examples/agent.py).
|
|
71
83
|
|
|
@@ -80,10 +92,10 @@ commits the turn, as with any turn detector, because the LLM needs the text.
|
|
|
80
92
|
|
|
81
93
|
## Run the model yourself (recommended for real-time)
|
|
82
94
|
|
|
83
|
-
The default URL is our free trial API (rate limited to 30 requests per minute; the round trip from outside mainland China adds network latency). For a live agent, serve the model next to it (DuplexJev-4B-Para needs about 10 GB of GPU memory):
|
|
95
|
+
The default URL is our free trial API (rate limited to 30 requests per minute; the round trip from outside mainland China adds network latency). For a live agent, serve the model next to it (DuplexJev-4B-Para needs about 10 GB of GPU memory; for the best turn-taking use DuplexJev-32B-Para v1.2 on one 80 GB GPU):
|
|
84
96
|
|
|
85
97
|
```bash
|
|
86
|
-
pip install "vllm[audio]>=0.29" duplexjev-vllm "duplexjev[server]>=0.
|
|
98
|
+
pip install "vllm[audio]>=0.29" duplexjev-vllm "duplexjev[server]>=0.6"
|
|
87
99
|
vllm serve adventists-ai/DuplexJev-4B-Para --max-model-len 4096 --port 8000 &
|
|
88
100
|
duplexjev gateway --vllm http://localhost:8000/v1 --port 8420
|
|
89
101
|
```
|
|
@@ -40,6 +40,18 @@ session = AgentSession(
|
|
|
40
40
|
)
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
**Give it the agent's last message** (new in 0.2.0). Turn-taking is clearly better when the model knows what the
|
|
44
|
+
agent had just said. Keep a reference to the detector and attach it to the session once:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
detector = duplexjev.TurnDetector(api_url="http://localhost:8420")
|
|
48
|
+
session = AgentSession(..., turn_handling={"turn_detection": detector})
|
|
49
|
+
detector.attach(session) # follows conversation_item_added; or detector.set_assistant_said(text)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
It needs a gateway from `duplexjev>=0.6`. On LiveKit eot-bench (English, DuplexJev-32B-Para v1.2) this cuts false
|
|
53
|
+
cut-offs at a 300 ms budget from 23.5% to 19.7%.
|
|
54
|
+
|
|
43
55
|
`AgentSession(turn_detection=duplexjev.TurnDetector(), ...)` works too (LiveKit marks that argument as deprecated).
|
|
44
56
|
A complete agent is in [`examples/agent.py`](examples/agent.py).
|
|
45
57
|
|
|
@@ -54,10 +66,10 @@ commits the turn, as with any turn detector, because the LLM needs the text.
|
|
|
54
66
|
|
|
55
67
|
## Run the model yourself (recommended for real-time)
|
|
56
68
|
|
|
57
|
-
The default URL is our free trial API (rate limited to 30 requests per minute; the round trip from outside mainland China adds network latency). For a live agent, serve the model next to it (DuplexJev-4B-Para needs about 10 GB of GPU memory):
|
|
69
|
+
The default URL is our free trial API (rate limited to 30 requests per minute; the round trip from outside mainland China adds network latency). For a live agent, serve the model next to it (DuplexJev-4B-Para needs about 10 GB of GPU memory; for the best turn-taking use DuplexJev-32B-Para v1.2 on one 80 GB GPU):
|
|
58
70
|
|
|
59
71
|
```bash
|
|
60
|
-
pip install "vllm[audio]>=0.29" duplexjev-vllm "duplexjev[server]>=0.
|
|
72
|
+
pip install "vllm[audio]>=0.29" duplexjev-vllm "duplexjev[server]>=0.6"
|
|
61
73
|
vllm serve adventists-ai/DuplexJev-4B-Para --max-model-len 4096 --port 8000 &
|
|
62
74
|
duplexjev gateway --vllm http://localhost:8000/v1 --port 8420
|
|
63
75
|
```
|
|
@@ -8,6 +8,10 @@ exposed on ``last_answers`` / ``on_answers``.
|
|
|
8
8
|
This module implements LiveKit's *streaming* (audio) turn-detector protocol: ``AgentSession`` pushes every input
|
|
9
9
|
audio frame into the stream, calls ``predict()`` once the VAD has seen ~200 ms of silence, and waits up to
|
|
10
10
|
``prediction_timeout`` for the resulting ``TurnDetectionEvent``.
|
|
11
|
+
|
|
12
|
+
Turn-taking is markedly better when the model also knows what the agent had just said. Call
|
|
13
|
+
``detector.attach(session)`` once after creating the ``AgentSession`` (it follows ``conversation_item_added``), or
|
|
14
|
+
set it yourself with ``detector.set_assistant_said(text)``.
|
|
11
15
|
"""
|
|
12
16
|
|
|
13
17
|
from __future__ import annotations
|
|
@@ -173,6 +177,27 @@ class TurnDetector:
|
|
|
173
177
|
"""Answers of the latest successful request: ``{id: {"answer", "confidence", "probs"}}``."""
|
|
174
178
|
self.last_server_ms: float | None = None
|
|
175
179
|
"""Server-side model time of the latest request, in milliseconds."""
|
|
180
|
+
self.assistant_said: str = ""
|
|
181
|
+
"""The agent's previous message, sent with every request (last 300 characters)."""
|
|
182
|
+
|
|
183
|
+
def set_assistant_said(self, text: str | None) -> None:
|
|
184
|
+
"""What the agent said just before the user's current turn."""
|
|
185
|
+
self.assistant_said = " ".join((text or "").split())[-300:]
|
|
186
|
+
|
|
187
|
+
def attach(self, session: Any) -> None:
|
|
188
|
+
"""Keep ``assistant_said`` up to date from an ``AgentSession``'s ``conversation_item_added`` events."""
|
|
189
|
+
|
|
190
|
+
def _on_item(ev: Any) -> None:
|
|
191
|
+
item = getattr(ev, "item", None)
|
|
192
|
+
if getattr(item, "role", None) != "assistant":
|
|
193
|
+
return
|
|
194
|
+
text = getattr(item, "text_content", None)
|
|
195
|
+
if callable(text):
|
|
196
|
+
text = text()
|
|
197
|
+
if text:
|
|
198
|
+
self.set_assistant_said(text)
|
|
199
|
+
|
|
200
|
+
session.on("conversation_item_added", _on_item)
|
|
176
201
|
|
|
177
202
|
def _as_question(self, q: str | dict[str, Any]) -> dict[str, Any]:
|
|
178
203
|
if isinstance(q, str):
|
|
@@ -366,6 +391,8 @@ class TurnDetectorStream:
|
|
|
366
391
|
det = self._detector
|
|
367
392
|
payload = {"audio_b64": base64.b64encode(wav).decode(), "format": "wav", "lang": det._lang,
|
|
368
393
|
"questions": det._questions}
|
|
394
|
+
if det.assistant_said:
|
|
395
|
+
payload["assistant_said"] = det.assistant_said
|
|
369
396
|
try:
|
|
370
397
|
async with self._http().post(
|
|
371
398
|
det._decide_url, json=payload, headers=det._headers,
|
|
@@ -377,7 +404,9 @@ class TurnDetectorStream:
|
|
|
377
404
|
data = await resp.json()
|
|
378
405
|
answers: dict[str, Any] = data["answers"]
|
|
379
406
|
probs = answers["turn"]["probs"]
|
|
380
|
-
|
|
407
|
+
o = TURN_QUESTION[det._lang]["options"]
|
|
408
|
+
# finished + backchannel: a short "ok" / "yeah" with nobody else talking also ends the turn
|
|
409
|
+
p_finished = float(probs.get(o[0], 0.0)) + float(probs.get(o[2], 0.0))
|
|
381
410
|
except asyncio.CancelledError:
|
|
382
411
|
raise
|
|
383
412
|
except Exception as e: # network or server error: never block the conversation
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
{livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/tests/test_turn_detector.py
RENAMED
|
@@ -172,3 +172,36 @@ async def test_agent_session_accepts_it(gateway):
|
|
|
172
172
|
det = duplexjev.TurnDetector(api_url=gateway.url)
|
|
173
173
|
session = AgentSession(turn_handling={"turn_detection": det})
|
|
174
174
|
assert session.turn_detection is det
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
async def test_assistant_said_is_sent_and_attach_follows_session(gateway):
|
|
178
|
+
det = duplexjev.TurnDetector(api_url=gateway.url)
|
|
179
|
+
stream = det.stream()
|
|
180
|
+
push(stream, 1.0)
|
|
181
|
+
await asyncio.wait_for(stream.predict(), 2)
|
|
182
|
+
assert "assistant_said" not in gateway.requests[-1]
|
|
183
|
+
|
|
184
|
+
handlers = {}
|
|
185
|
+
|
|
186
|
+
class FakeSession:
|
|
187
|
+
def on(self, name, fn):
|
|
188
|
+
handlers[name] = fn
|
|
189
|
+
|
|
190
|
+
class Item:
|
|
191
|
+
def __init__(self, role, text):
|
|
192
|
+
self.role, self.text_content = role, text
|
|
193
|
+
|
|
194
|
+
class Ev:
|
|
195
|
+
def __init__(self, item):
|
|
196
|
+
self.item = item
|
|
197
|
+
|
|
198
|
+
det.attach(FakeSession())
|
|
199
|
+
handlers["conversation_item_added"](Ev(Item("user", "hello")))
|
|
200
|
+
assert det.assistant_said == ""
|
|
201
|
+
handlers["conversation_item_added"](Ev(Item("assistant", "Sure, what time\nworks for you?")))
|
|
202
|
+
assert det.assistant_said == "Sure, what time works for you?"
|
|
203
|
+
await asyncio.wait_for(stream.predict(), 2)
|
|
204
|
+
assert gateway.requests[-1]["assistant_said"] == "Sure, what time works for you?"
|
|
205
|
+
det.set_assistant_said("y" * 500)
|
|
206
|
+
assert len(det.assistant_said) == 300
|
|
207
|
+
await stream.aclose()
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## 0.1.0 (2026-10-10)
|
|
4
|
-
|
|
5
|
-
- First release: `duplexjev.TurnDetector`, an audio (streaming) turn detector for LiveKit Agents that asks a
|
|
6
|
-
DuplexJev gateway for the turn state of the current user turn.
|
|
7
|
-
- Extra questions (emotion, non-verbal sound, barge-in, or your own) answered in the same request; see
|
|
8
|
-
`last_answers` / `on_answers`.
|
|
9
|
-
- Optional backchannel opportunity from "What should the assistant do now?" (`backchannel_threshold`).
|
|
10
|
-
- Tested with livekit-agents 1.8.6.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.1.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{livekit_plugins_duplexjev-0.1.0 → livekit_plugins_duplexjev-0.2.0}/livekit/plugins/duplexjev/log.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|