pipecat-duplexjev 0.1.0__py3-none-any.whl

pipecat_duplexjev/__init__.py

@@ -0,0 +1,5 @@
+"""Pipecat integration for DuplexJev speech decision models."""
+from .turn import DEFAULT_URL, PRESETS, TURN_QUESTION, DuplexJevTurnAnalyzer
+
+__all__ = ["DuplexJevTurnAnalyzer", "DEFAULT_URL", "PRESETS", "TURN_QUESTION"]
+__version__ = "0.1.0"

pipecat_duplexjev/turn.py

@@ -0,0 +1,136 @@
+"""DuplexJev end-of-turn analyzer for Pipecat.
+
+DuplexJev reads turn state (finished / still talking / backchannel / asking to wait) straight from the audio as one
+constrained token of a speech LLM, so no transcript and no decoding are needed. Optional extra questions (emotion,
+non-verbal sounds, intent, ...) are answered in the same forward pass and exposed on ``last_answers``.
+"""
+from __future__ import annotations
+
+import base64
+import io
+import json
+import urllib.error
+import urllib.request
+import wave
+from typing import Any, Callable
+
+import numpy as np
+from loguru import logger
+from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn, SmartTurnParams, SmartTurnTimeoutException
+
+DEFAULT_URL = "https://api.adventists.cn/duplexjev"
+
+TURN_QUESTION = {
+    "en": {"id": "turn", "text": "Has the user finished speaking?",
+           "options": ["finished, the assistant can reply", "not finished, still talking",
+                       "just a backchannel, not taking the turn", "hesitating or asking to wait"], "lang": "en"},
+    "zh": {"id": "turn", "text": "用户现在处于什么话轮状态？",
+           "options": ["话说完了，可以接话", "句子没说完，还在继续", "简短附和，不是要接话", "还在组织语言，或要求先等一下"],
+           "lang": "zh"},
+}
+
+PRESETS = {
+    "emotion": {"en": ("What is the speaker's emotional state?", ["neutral", "happy", "angry", "sad"]),
+                "zh": ("说话人当时的情绪状态是？", ["中性", "高兴", "生气", "伤心"])},
+    "sound": {"en": ("Besides speech, which sound can be heard in this audio?",
+                     ["laughter", "breathing", "coughing", "a sigh", "None of these"]),
+              "zh": ("除了说话，这段音频里还有哪种声音？", ["笑声", "呼吸声", "咳嗽", "叹气", "都没有"])},
+    "barge_in": {"en": ("Is the user trying to interrupt or take over the conversation?", ["yes", "no"]),
+                 "zh": ("用户是不是想打断或者抢话？", ["是", "不是"])},
+}
+
+
+def _wav16k(audio: np.ndarray, sample_rate: int) -> bytes:
+    """Mono float32 in [-1, 1] -> 16 kHz PCM16 WAV bytes."""
+    x = np.asarray(audio, dtype=np.float32).reshape(-1)
+    if sample_rate != 16000 and len(x):
+        n = max(1, int(round(len(x) * 16000 / sample_rate)))
+        x = np.interp(np.linspace(0, len(x) - 1, n), np.arange(len(x)), x).astype(np.float32)
+    pcm = (np.clip(x, -1.0, 1.0) * 32767).astype("<i2").tobytes()
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as w:
+        w.setnchannels(1)
+        w.setsampwidth(2)
+        w.setframerate(16000)
+        w.writeframes(pcm)
+    return buf.getvalue()
+
+
+class DuplexJevTurnAnalyzer(BaseSmartTurn):
+    """Audio end-of-turn detection with DuplexJev, via the hosted API or your own ``duplexjev gateway``.
+
+    Args:
+        url: base URL of a DuplexJev gateway. Defaults to the free hosted trial API (rate limited, served from China).
+            For production run your own: ``vllm serve adventists-ai/DuplexJev-4B-Para`` + ``duplexjev gateway``.
+        api_key: bearer key, if the gateway requires one.
+        lang: ``"en"`` or ``"zh"``: language of the question wording (the model hears both languages either way).
+        threshold: probability of "finished" at or above which the turn is complete.
+        extra_questions: names from ``PRESETS`` (``"emotion"``, ``"sound"``, ``"barge_in"``) or question dicts
+            ``{"id", "text", "options"}``; answered in the same pass, see ``last_answers``.
+        on_answers: optional callback ``f(answers: dict)`` called after every prediction.
+        timeout: HTTP timeout in seconds (defaults to ``params.stop_secs``).
+    """
+
+    def __init__(self, *, url: str = DEFAULT_URL, api_key: str | None = None, lang: str = "en",
+                 threshold: float = 0.5, extra_questions: list | None = None,
+                 on_answers: Callable[[dict], Any] | None = None, timeout: float | None = None,
+                 sample_rate: int | None = None, params: SmartTurnParams | None = None):
+        super().__init__(sample_rate=sample_rate, params=params)
+        if lang not in TURN_QUESTION:
+            raise ValueError("lang must be 'en' or 'zh'")
+        self._url = url.rstrip("/") + "/v1/decide"
+        self._headers = {"Content-Type": "application/json"}
+        if api_key:
+            self._headers["Authorization"] = f"Bearer {api_key}"
+        self._lang = lang
+        self._threshold = threshold
+        self._timeout = timeout
+        self._on_answers = on_answers
+        self._questions = [TURN_QUESTION[lang]] + [self._as_question(q) for q in (extra_questions or [])]
+        self.last_answers: dict = {}
+        self.last_server_ms: float | None = None
+
+    def _as_question(self, q) -> dict:
+        if isinstance(q, str):
+            if q not in PRESETS:
+                raise ValueError(f"unknown preset {q!r}; choose from {sorted(PRESETS)} or pass a dict")
+            text, options = PRESETS[q][self._lang]
+            return {"id": q, "text": text, "options": options, "lang": self._lang}
+        d = dict(q)
+        d.setdefault("lang", self._lang)
+        return d
+
+    def _request(self, wav: bytes) -> dict:
+        payload = {"audio_b64": base64.b64encode(wav).decode(), "format": "wav", "lang": self._lang,
+                   "questions": self._questions}
+        req = urllib.request.Request(self._url, data=json.dumps(payload).encode(), headers=self._headers)
+        timeout = self._timeout if self._timeout is not None else self.params.stop_secs
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as r:
+                return json.load(r)
+        except TimeoutError as e:
+            raise SmartTurnTimeoutException(str(e)) from e
+        except urllib.error.URLError as e:
+            if isinstance(e.reason, TimeoutError):
+                raise SmartTurnTimeoutException(str(e)) from e
+            raise
+
+    def _predict_endpoint(self, audio_array: np.ndarray) -> dict[str, Any]:
+        try:
+            r = self._request(_wav16k(audio_array, self.sample_rate or 16000))
+        except SmartTurnTimeoutException:
+            raise
+        except Exception as e:  # network or server error: do not block the conversation
+            logger.error(f"DuplexJev request failed: {e}")
+            return {"prediction": 0, "probability": 0.0}
+        answers = r.get("answers", {})
+        self.last_answers = answers
+        self.last_server_ms = r.get("ms")
+        if self._on_answers:
+            try:
+                self._on_answers(answers)
+            except Exception as e:
+                logger.warning(f"on_answers callback raised: {e}")
+        turn = answers.get("turn", {})
+        p = float(turn.get("probs", {}).get(TURN_QUESTION[self._lang]["options"][0], 0.0))
+        return {"prediction": 1 if p >= self._threshold else 0, "probability": p}

pipecat_duplexjev-0.1.0.dist-info/METADATA

@@ -0,0 +1,111 @@
+Metadata-Version: 2.5
+Name: pipecat-duplexjev
+Version: 0.1.0
+Summary: Audio end-of-turn detection for Pipecat with DuplexJev: turn state, emotion and non-verbal sounds from one forward pass, no transcript needed.
+Project-URL: Homepage, https://github.com/adventists-ai/duplexjev/tree/main/integrations/pipecat
+Project-URL: Models, https://huggingface.co/adventists-ai/DuplexJev-4B-Para
+Project-URL: Paper, https://arxiv.org/abs/2610.02638
+Author-email: Jie Jin <jiejin@adventists.ai>
+License: BSD-2-Clause
+License-File: LICENSE
+Keywords: duplexjev,end-of-turn,pipecat,speech-llm,turn-detection,voice-agent
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
+Requires-Python: >=3.10
+Requires-Dist: numpy
+Requires-Dist: pipecat-ai>=1.0
+Description-Content-Type: text/markdown
+
+# pipecat-duplexjev
+
+**Audio end-of-turn detection for [Pipecat](https://github.com/pipecat-ai/pipecat), powered by
+[DuplexJev](https://github.com/adventists-ai/duplexjev).** It listens to *how* the user speaks, not only what they say,
+and tells your bot whether the user is **finished**, **still talking**, **just backchanneling** ("mm-hmm") or
+**asking you to wait**. The same request can also return **emotion**, **laughter / breathing / coughs / sighs** and
+**barge-in** at no extra cost.
+
+- **Four turn states, not a yes/no.** A backchannel or a "hold on" is not a reason to start talking.
+- **Hears beyond the transcript.** Every answer is read from the audio as one constrained token of a speech LLM.
+  Nothing is decoded and no STT is needed for the decision.
+- **More than turns, same forward pass.** Ask for emotion, non-verbal sounds or your own multiple-choice questions.
+  They come back in `last_answers` with the turn decision.
+- **~55 ms on the server** for a turn decision with DuplexJev-4B-Para on one GPU.
+- **Chinese and English.**
+- On the CoDeTT turn-taking benchmark (English, zero-shot), DuplexJev-32B-Turn scores **70.0**, against **51.4** for
+  Smart Turn v3. See the [paper](https://arxiv.org/abs/2610.02638) and the
+  [project page](https://adventists-ai.github.io/duplexjev/).
+
+Tested with `pipecat-ai` 1.12.0. Maintained by Adventists.ai (AI降临派).
+
+## Install
+
+```bash
+pip install pipecat-duplexjev
+```
+
+## Use
+
+Drop it in wherever Pipecat takes a turn analyzer (Pipecat >= 1.0):
+
+```python
+from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair, LLMUserAggregatorParams
+from pipecat.turns.user_stop import TurnAnalyzerUserTurnStopStrategy
+from pipecat.turns.user_turn_strategies import UserTurnStrategies
+from pipecat_duplexjev import DuplexJevTurnAnalyzer
+
+turn = DuplexJevTurnAnalyzer(
+    url="http://localhost:8420",          # your own gateway; omit to use the free hosted trial API
+    extra_questions=["emotion", "sound"],
+    on_answers=lambda a: print({k: v["answer"] for k, v in a.items()}),
+)
+
+aggregators = LLMContextAggregatorPair(context, user_params=LLMUserAggregatorParams(
+    vad_analyzer=SileroVADAnalyzer(),
+    user_turn_strategies=UserTurnStrategies(stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=turn)]),
+))
+```
+
+A complete snippet is in [`examples/wiring.py`](examples/wiring.py). [`examples/live_check.py`](examples/live_check.py)
+feeds real clips through the analyzer, frame by frame, the way a transport does:
+
+```text
+ex4.wav: COMPLETE   p(finished)=1.00  turn='finished, the assistant can reply'   emotion=neutral  server 59.6 ms
+ex3.wav: INCOMPLETE p(finished)=0.11  turn='not finished, still talking'          emotion=sad      server 53.2 ms
+ex5.wav: INCOMPLETE p(finished)=0.28  turn='hesitating or asking to wait'         emotion=angry    server 56.0 ms
+```
+
+## Run the model yourself (recommended for real-time)
+
+The default URL is our free trial API. It is rate limited (30 requests per minute) and served from mainland China,
+so the network round trip from elsewhere can be over a second. For a live bot, serve the model next to your agent.
+DuplexJev-4B-Para needs about 10 GB of GPU memory:
+
+```bash
+pip install "vllm[audio]>=0.29" duplexjev-vllm "duplexjev[server]>=0.4.1"
+vllm serve adventists-ai/DuplexJev-4B-Para --max-model-len 4096 --port 8000 &
+duplexjev gateway --vllm http://localhost:8000/v1 --port 8420
+```
+
+Then pass `url="http://localhost:8420"`.
+
+## Options
+
+| argument | default | |
+|---|---|---|
+| `url` | hosted trial API | base URL of a `duplexjev gateway` |
+| `api_key` | `None` | bearer key, if your gateway requires one |
+| `lang` | `"en"` | `"en"` or `"zh"`: wording of the questions (the model hears both languages either way) |
+| `threshold` | `0.5` | probability of "finished" from which the turn is complete |
+| `extra_questions` | `[]` | `"emotion"`, `"sound"`, `"barge_in"`, or dicts `{"id", "text", "options"}` |
+| `on_answers` | `None` | callback with every answer dict |
+| `timeout` | `params.stop_secs` | HTTP timeout; on timeout Pipecat treats the turn as complete |
+| `params` | `SmartTurnParams()` | Pipecat's usual `stop_secs`, `pre_speech_ms`, `max_duration_secs` |
+
+If the gateway can't be reached, the analyzer logs the error and reports "not finished", so Pipecat falls back to its
+silence timeout and the conversation never blocks.
+
+## License
+
+The plugin code is BSD-2-Clause. DuplexJev model weights are CC BY-NC 4.0 (non-commercial); see the
+[model cards](https://huggingface.co/adventists-ai). For commercial use, contact jiejin@adventists.ai.

pipecat_duplexjev-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+pipecat_duplexjev/__init__.py,sha256=mPSbS_djP21XvieyTYp6iSl53h2pGPcP-qvh3oHq4gU,243
+pipecat_duplexjev/turn.py,sha256=o5GV7-boEpcmREql7hE9BQOSfG7a6wukUYO3r3SL1to,6900
+pipecat_duplexjev-0.1.0.dist-info/METADATA,sha256=tyDLSGrYzRqHk0JBp9pnWuKLpO2YiLdYCJnGqqgP24U,5483
+pipecat_duplexjev-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+pipecat_duplexjev-0.1.0.dist-info/licenses/LICENSE,sha256=nmfjvVoTdFz0Qzyh6vHkBLTW-YpHWApu0IhvpZlJEs0,1316
+pipecat_duplexjev-0.1.0.dist-info/RECORD,,

pipecat_duplexjev-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

pipecat_duplexjev-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2026, Adventists.ai (AI降临派)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.