livekit-plugins-volcenginee 1.3.12__tar.gz → 1.3.13__tar.gz

{livekit_plugins_volcenginee-1.3.12 → livekit_plugins_volcenginee-1.3.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: livekit-plugins-volcenginee
-Version: 1.3.12
+Version: 1.3.13
 Summary: LiveKit Agent Plugins for Volcengine
 Author-email: linsanzhu <2287225730@qq.com>
 Keywords: audio,livekit,realtime,video,webrtc
@@ -13,7 +13,7 @@ Classifier: Topic :: Multimedia :: Sound/Audio
 Classifier: Topic :: Multimedia :: Video
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
-Requires-Dist: livekit-agents~=1.5.13
+Requires-Dist: livekit-agents>=1.5.13
 Requires-Dist: numpy
 Requires-Dist: openai>=1.80.0
 Requires-Dist: osc-data~=0.3.1
@@ -434,12 +434,62 @@ volcengine.STT(
     enable_itn: bool = False,
     enable_punc: bool = True,
     enable_ddc: bool = False,
+    enable_diarization: bool = False,   # 开启说话人分离（同时启用性别识别）
     vad_segment_duration: int = 3000,
     end_window_size: int = 500,
     force_to_speech_time: int = 1000,
 )
 ```
 
+#### 说话人分离（Diarization）
+
+把 `enable_diarization=True` 打开后，插件会：
+
+1. 在 `STTCapabilities.diarization` 上声明能力，符合
+   [LiveKit STT 文档](https://docs.livekit.io/agents/models/stt/#speaker-diarization)
+   要求的三件事之一。
+2. 在每条 `SpeechData` 上填上来自服务端 `additions.speaker` 的 `speaker_id`（形如
+   `"speaker_1"`）。
+3. 在 `SpeechData.metadata["gender"]` 里带上 `additions.gender`（按官方文档，性别
+   只在 `definite=true` 的分句上返回，所以中间结果不会有）。
+4. 按分句拆分 `utterances` 数组，每段独立事件（START → INTERIM/FINAL → END），
+   而不是把多段合并到一个事件里——这样下游的 `MultiSpeakerAdapter` 能在段粒度上
+   决定谁是主说话人、要不要抑制背景。
+
+> 注意：插件只做"原始说话人分离"的暴露，**不**做主说话人判断或背景抑制。这两个
+> 策略属于 `MultiSpeakerAdapter`。典型用法：
+
+```python
+from livekit.agents import Agent, AgentSession, JobContext, cli, WorkerOptions
+from livekit.agents.stt import MultiSpeakerAdapter
+from livekit.plugins import volcengine
+
+
+async def entry_point(ctx: JobContext):
+    raw_stt = volcengine.STT(
+        api_key="your_api_key",
+        enable_diarization=True,        # 暴露每段 speaker_id + gender
+    )
+    # 用官方适配器做主说话人检测 / 背景抑制
+    stt = MultiSpeakerAdapter(
+        raw_stt,
+        detect_primary_speaker=True,        # 默认 True：识别谁是主说话人
+        suppress_background_speaker=False,  # True 则只把主说话人文本交给 agent
+    )
+
+    agent = Agent(instructions="你是一个友好的助手。")
+    session = AgentSession(stt=stt, llm=..., tts=...)
+    await session.start(agent=agent, room=ctx.room)
+    await ctx.connect()
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entry_point))
+```
+
+当 `MultiSpeakerAdapter.suppress_background_speaker=True` 时，下游 agent 只会
+看到主说话人的分句，其它说话人（如旁人的声音、背景里的电视）会直接被吞掉。
+
 ### LLM (大语言模型)
 
 ```python
@@ -582,6 +632,15 @@ A: 根据您的应用需求选择合适的版本：
 
 ## 📝 更新日志
 
+### 最新变更
+- `volcengine.STT` 新增 `enable_diarization` 参数（默认关闭）。开启后：
+  - `STTCapabilities.diarization = True`，可被 `MultiSpeakerAdapter` 包装做主说话人
+    检测 / 背景抑制。
+  - 每段 `SpeechData` 携带 `speaker_id`（来自 `additions.speaker`），性别写入
+    `SpeechData.metadata["gender"]`（按官方文档，gender 仅在 `definite=true` 的分句
+    上返回）。
+  - `utterances[]` 数组按分句拆分事件，方便下游在段粒度上做判断。
+
 ### v1.5.4 适配
 - 升级 `livekit-agents` 到 `1.5.4`
 - Python 要求调整为 `>=3.10`

{livekit_plugins_volcenginee-1.3.12 → livekit_plugins_volcenginee-1.3.13}/README.md

@@ -407,12 +407,62 @@ volcengine.STT(
     enable_itn: bool = False,
     enable_punc: bool = True,
     enable_ddc: bool = False,
+    enable_diarization: bool = False,   # 开启说话人分离（同时启用性别识别）
     vad_segment_duration: int = 3000,
     end_window_size: int = 500,
     force_to_speech_time: int = 1000,
 )
 ```
 
+#### 说话人分离（Diarization）
+
+把 `enable_diarization=True` 打开后，插件会：
+
+1. 在 `STTCapabilities.diarization` 上声明能力，符合
+   [LiveKit STT 文档](https://docs.livekit.io/agents/models/stt/#speaker-diarization)
+   要求的三件事之一。
+2. 在每条 `SpeechData` 上填上来自服务端 `additions.speaker` 的 `speaker_id`（形如
+   `"speaker_1"`）。
+3. 在 `SpeechData.metadata["gender"]` 里带上 `additions.gender`（按官方文档，性别
+   只在 `definite=true` 的分句上返回，所以中间结果不会有）。
+4. 按分句拆分 `utterances` 数组，每段独立事件（START → INTERIM/FINAL → END），
+   而不是把多段合并到一个事件里——这样下游的 `MultiSpeakerAdapter` 能在段粒度上
+   决定谁是主说话人、要不要抑制背景。
+
+> 注意：插件只做"原始说话人分离"的暴露，**不**做主说话人判断或背景抑制。这两个
+> 策略属于 `MultiSpeakerAdapter`。典型用法：
+
+```python
+from livekit.agents import Agent, AgentSession, JobContext, cli, WorkerOptions
+from livekit.agents.stt import MultiSpeakerAdapter
+from livekit.plugins import volcengine
+
+
+async def entry_point(ctx: JobContext):
+    raw_stt = volcengine.STT(
+        api_key="your_api_key",
+        enable_diarization=True,        # 暴露每段 speaker_id + gender
+    )
+    # 用官方适配器做主说话人检测 / 背景抑制
+    stt = MultiSpeakerAdapter(
+        raw_stt,
+        detect_primary_speaker=True,        # 默认 True：识别谁是主说话人
+        suppress_background_speaker=False,  # True 则只把主说话人文本交给 agent
+    )
+
+    agent = Agent(instructions="你是一个友好的助手。")
+    session = AgentSession(stt=stt, llm=..., tts=...)
+    await session.start(agent=agent, room=ctx.room)
+    await ctx.connect()
+
+
+if __name__ == "__main__":
+    cli.run_app(WorkerOptions(entrypoint_fnc=entry_point))
+```
+
+当 `MultiSpeakerAdapter.suppress_background_speaker=True` 时，下游 agent 只会
+看到主说话人的分句，其它说话人（如旁人的声音、背景里的电视）会直接被吞掉。
+
 ### LLM (大语言模型)
 
 ```python
@@ -555,6 +605,15 @@ A: 根据您的应用需求选择合适的版本：
 
 ## 📝 更新日志
 
+### 最新变更
+- `volcengine.STT` 新增 `enable_diarization` 参数（默认关闭）。开启后：
+  - `STTCapabilities.diarization = True`，可被 `MultiSpeakerAdapter` 包装做主说话人
+    检测 / 背景抑制。
+  - 每段 `SpeechData` 携带 `speaker_id`（来自 `additions.speaker`），性别写入
+    `SpeechData.metadata["gender"]`（按官方文档，gender 仅在 `definite=true` 的分句
+    上返回）。
+  - `utterances[]` 数组按分句拆分事件，方便下游在段粒度上做判断。
+
 ### v1.5.4 适配
 - 升级 `livekit-agents` 到 `1.5.4`
 - Python 要求调整为 `>=3.10`

{livekit_plugins_volcenginee-1.3.12 → livekit_plugins_volcenginee-1.3.13}/livekit/plugins/volcengine/stt.py

@@ -98,6 +98,16 @@ class STTOptions:
     end_window_size: int = 500
     force_to_speech_time: int = 1000
 
+    # Speaker diarization. When ``enable_diarization`` is true, the
+    # plugin asks the Volcengine ASR for per-utterance ``speaker`` ids
+    # and exposes them on each emitted ``SpeechData.speaker_id`` so
+    # that LiveKit's ``MultiSpeakerAdapter`` can do main-speaker
+    # detection or background-speaker suppression on top. Gender
+    # detection is bundled in — the Volcengine docs tie it to
+    # ``enable_speaker_info``, so once diarization is on we always
+    # request gender and surface it via ``SpeechData.metadata``.
+    enable_diarization: bool = False
+
     def get_ws_url(self):
         return self.base_url
 
@@ -119,9 +129,9 @@ class STTOptions:
                 "enable_punc": self.enable_punc,
                 "enable_ddc": self.enable_ddc,
                 "enable_nonstream": True,
-                "enable_speaker_info": True,
+                "enable_speaker_info": self.enable_diarization,
                 "ssd_version": "200",
-                "enable_gender_detection": True,
+                "enable_gender_detection": self.enable_diarization,
                 "show_utterance": self.show_utterance,
                 "result_type": self.result_type,
                 "vad_segment_duration": self.vad_segment_duration,
@@ -191,17 +201,25 @@ class STT(stt.STT):
         enable_itn: bool = False,
         enable_punc: bool = True,
         enable_ddc: bool = False,
+        enable_diarization: bool = False,
         vad_segment_duration: int = 3000,
         end_window_size: int = 500,
         force_to_speech_time: int = 1000,
         http_session: aiohttp.ClientSession | None = None,
         interim_results: bool = True,
     ) -> None:
-        super().__init__(
-            capabilities=stt.STTCapabilities(
-                streaming=True, interim_results=interim_results
-            )
-        )
+        # Per the LiveKit STT API contract, a plugin that exposes
+        # per-segment speaker ids must set ``diarization=True`` on its
+        # capabilities so the framework knows it can wrap us in
+        # ``MultiSpeakerAdapter`` for primary-speaker detection /
+        # background-speaker suppression.
+        capabilities_kwargs: dict = {
+            "streaming": True,
+            "interim_results": interim_results,
+            "diarization": enable_diarization,
+        }
+
+        super().__init__(capabilities=stt.STTCapabilities(**capabilities_kwargs))
 
         api_key = api_key if is_given(api_key) else os.getenv("VOLCENGINE_STT_API_KEY")
         if api_key is None:
@@ -218,6 +236,7 @@ class STT(stt.STT):
             enable_itn=enable_itn,
             enable_punc=enable_punc,
             enable_ddc=enable_ddc,
+            enable_diarization=enable_diarization,
             vad_segment_duration=vad_segment_duration,
             end_window_size=end_window_size,
             force_to_speech_time=force_to_speech_time,
@@ -436,69 +455,127 @@ class SpeechStream(stt.SpeechStream):
         text = result.get("text", "")
         if text == "":
             return
+
         utterances = result.get("utterances", [])
         if len(utterances) == 0:
             return
-        language = self._opts.language
-        definite = utterances[0].get("definite", False)
-        start_time = utterances[0].get("start_time", 0.0)
-        end_time = utterances[0].get("end_time", 0.0)
         confidence = result.get("confidence", 0.0)
+
+        if not self._opts.enable_diarization:
+            # Fast path — no diarization, so we don't need to walk the
+            # utterance array. Use the original behavior: top-level
+            # ``result.text`` (cumulative across the current utterance)
+            # and ``utterances[0]`` for the active utterance's definite
+            # flag and timing. ``speaker_id`` stays None so this path
+            # is indistinguishable from the pre-diarization plugin.
+            self._emit_segment(
+                text=text,
+                definite=utterances[0].get("definite", False),
+                start_time=utterances[0].get("start_time", 0.0),
+                end_time=utterances[0].get("end_time", 0.0),
+                confidence=confidence,
+                speaker_id=None,
+                metadata=None,
+            )
+            return
+
+        # Diarization path — ``utterances`` is a cumulative list of
+        # completed sentence segments (see the Volcengine ASR docs).
+        # In streaming mode each response usually carries one active
+        # utterance, but end-of-stream flushes can return several at
+        # once. Emit each segment as its own event so
+        # ``MultiSpeakerAdapter`` (or any other downstream consumer) can
+        # apply per-speaker logic on a per-segment basis.
+        for utterance in utterances:
+            utt_text = utterance.get("text") or ""
+            if not utt_text:
+                continue
+            speaker_id, metadata = self._extract_diarization(utterance)
+            self._emit_segment(
+                text=utt_text,
+                definite=utterance.get("definite", False),
+                start_time=utterance.get("start_time", 0.0),
+                end_time=utterance.get("end_time", 0.0),
+                confidence=confidence,
+                speaker_id=speaker_id,
+                metadata=metadata,
+            )
+
+    def _extract_diarization(
+        self, utterance: dict
+    ) -> tuple[str | None, dict | None]:
+        """Pull ``speaker_id`` from ``additions.speaker`` and, when the
+        segment is definite, the ``gender`` tag for ``metadata``. Per
+        the Volcengine docs, ``additions.gender`` is only present on
+        definite utterances — interim updates legitimately omit it."""
+        additions = utterance.get("additions") or {}
+        raw_speaker = additions.get("speaker")
+        speaker_id = f"speaker_{raw_speaker}" if raw_speaker is not None else None
+
+        metadata: dict | None = None
+        if utterance.get("definite", False):
+            raw_gender = additions.get("gender")
+            if raw_gender is not None:
+                metadata = {"gender": str(raw_gender)}
+
+        return speaker_id, metadata
+
+    def _emit_segment(
+        self,
+        *,
+        text: str,
+        definite: bool,
+        start_time: float,
+        end_time: float,
+        confidence: float,
+        speaker_id: str | None,
+        metadata: dict | None,
+    ) -> None:
+        """Run the START/INTERIM/FINAL/END state machine for a single
+        segment. The plugin itself does not do main-speaker detection
+        or background suppression — that's the framework's
+        ``MultiSpeakerAdapter``'s job. We just emit per-segment events
+        with ``speaker_id`` set so the adapter can apply its policy."""
+        language = self._opts.language
+
+        def _make_alternative() -> stt.SpeechData:
+            return stt.SpeechData(
+                language=language,
+                text=text,
+                start_time=start_time,
+                end_time=end_time,
+                confidence=confidence,
+                speaker_id=speaker_id,
+                metadata=metadata,
+            )
+
         if not definite and not self._speaking:
             self._speaking = True
             self._event_ch.send_nowait(
                 stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
             )
             logger.info("transcription start")
-            if text:
-                alternatives = [
-                    stt.SpeechData(
-                        language=language,
-                        text=text,
-                        start_time=start_time,
-                        end_time=end_time,
-                        confidence=confidence,
-                    )
-                ]
-                self._event_ch.send_nowait(
-                    stt.SpeechEvent(
-                        type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
-                        request_id=self._request_id,
-                        alternatives=alternatives,
-                    )
+            self._event_ch.send_nowait(
+                stt.SpeechEvent(
+                    type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
+                    request_id=self._request_id,
+                    alternatives=[_make_alternative()],
                 )
+            )
         elif not definite and self._speaking:
-            alternatives = [
-                stt.SpeechData(
-                    text=text,
-                    start_time=start_time,
-                    end_time=end_time,
-                    confidence=confidence,
-                    language=language,
-                )
-            ]
             self._event_ch.send_nowait(
                 stt.SpeechEvent(
                     type=stt.SpeechEventType.INTERIM_TRANSCRIPT,
                     request_id=self._request_id,
-                    alternatives=alternatives,
+                    alternatives=[_make_alternative()],
                 )
             )
         elif definite and self._speaking:
-            alternatives = [
-                stt.SpeechData(
-                    text=text,
-                    start_time=start_time,
-                    end_time=end_time,
-                    confidence=confidence,
-                    language=language,
-                )
-            ]
             self._event_ch.send_nowait(
                 stt.SpeechEvent(
                     type=stt.SpeechEventType.FINAL_TRANSCRIPT,
                     request_id=self._request_id,
-                    alternatives=alternatives,
+                    alternatives=[_make_alternative()],
                 )
             )
             self._event_ch.send_nowait(
@@ -509,6 +586,29 @@ class SpeechStream(stt.SpeechStream):
             )
             self._speaking = False
             logger.info("transcription end", extra={"text": text})
+        elif definite and not self._speaking:
+            # The ASR flushed a finished segment without any preceding
+            # interim update in this session — typical for short
+            # snippets or for segments that arrived in the same batch
+            # as a previous FINAL. Emit a complete cycle so the agent
+            # still sees the text instead of dropping it.
+            self._event_ch.send_nowait(
+                stt.SpeechEvent(type=stt.SpeechEventType.START_OF_SPEECH)
+            )
+            self._event_ch.send_nowait(
+                stt.SpeechEvent(
+                    type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                    request_id=self._request_id,
+                    alternatives=[_make_alternative()],
+                )
+            )
+            self._event_ch.send_nowait(
+                stt.SpeechEvent(
+                    type=stt.SpeechEventType.END_OF_SPEECH,
+                    request_id=self._request_id,
+                )
+            )
+            logger.info("transcription end (flushed)", extra={"text": text})
 
 
 def parse_response(res):

livekit_plugins_volcenginee-1.3.13/livekit/plugins/volcengine/version.py

@@ -0,0 +1 @@
+__version__ = "1.3.13"

{livekit_plugins_volcenginee-1.3.12 → livekit_plugins_volcenginee-1.3.13}/pyproject.toml

@@ -3,9 +3,7 @@ name = "livekit-plugins-volcenginee"
 dynamic = ["version"]
 description = "LiveKit Agent Plugins for Volcengine"
 readme = "README.md"
-authors = [
-    { name = "linsanzhu", email = "2287225730@qq.com" }
-]
+authors = [{ name = "linsanzhu", email = "2287225730@qq.com" }]
 keywords = ["webrtc", "realtime", "audio", "video", "livekit"]
 requires-python = ">=3.10"
 dependencies = [
@@ -13,7 +11,7 @@ dependencies = [
     "osc-data~=0.3.1",
     "pydantic",
     "numpy",
-    "livekit-agents~=1.5.13",
+    "livekit-agents>=1.5.13",
 ]
 
 classifiers = [

livekit_plugins_volcenginee-1.3.12/livekit/plugins/volcengine/version.py

@@ -1 +0,0 @@
-__version__ = "1.3.12"