deeptutor-audio-overview 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deeptutor/capabilities/audio_overview/capability.py +143 -0
- deeptutor/capabilities/audio_overview/pipeline.py +513 -0
- deeptutor/capabilities/audio_overview/request_config.py +18 -0
- deeptutor_audio_overview-1.7.0.dist-info/METADATA +7 -0
- deeptutor_audio_overview-1.7.0.dist-info/RECORD +7 -0
- deeptutor_audio_overview-1.7.0.dist-info/WHEEL +5 -0
- deeptutor_audio_overview-1.7.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""KB Audio Overview capability."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, cast
|
|
6
|
+
import uuid
|
|
7
|
+
|
|
8
|
+
from deeptutor.agents._shared.capability_result import emit_capability_result
|
|
9
|
+
from deeptutor.capabilities.audio_overview.pipeline import (
|
|
10
|
+
AudioOverviewError,
|
|
11
|
+
AudioOverviewPipeline,
|
|
12
|
+
)
|
|
13
|
+
from deeptutor.capabilities.audio_overview.request_config import AudioOverviewRequestConfig
|
|
14
|
+
from deeptutor.core.capability_protocol import StreamBusProtocol, TurnCapability
|
|
15
|
+
from deeptutor.core.context import UnifiedContext
|
|
16
|
+
from deeptutor.i18n import StatusI18n
|
|
17
|
+
from deeptutor.runtime.bootstrap.builtin_capabilities import builtin_capability_manifest
|
|
18
|
+
from deeptutor.runtime.stream_bus import StreamBus
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AudioOverviewCapability(TurnCapability):
|
|
22
|
+
manifest = builtin_capability_manifest("audio_overview")
|
|
23
|
+
|
|
24
|
+
async def run(self, context: UnifiedContext, protocol_stream: StreamBusProtocol) -> None:
|
|
25
|
+
stream = cast(StreamBus, protocol_stream)
|
|
26
|
+
i18n = StatusI18n(self.name, context.language, module="capabilities")
|
|
27
|
+
kb_name = str(context.knowledge_bases[0]) if context.knowledge_bases else ""
|
|
28
|
+
note_id = str(context.metadata.get("note_document_id") or "")
|
|
29
|
+
if not kb_name and not note_id:
|
|
30
|
+
raise AudioOverviewError(
|
|
31
|
+
i18n.t("missing_kb", "Attach a knowledge base to create an audio overview.")
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
request_config = AudioOverviewRequestConfig.model_validate(context.config_overrides or {})
|
|
35
|
+
if request_config.topic and request_config.topic != context.user_message:
|
|
36
|
+
topic = request_config.topic
|
|
37
|
+
else:
|
|
38
|
+
topic = context.user_message or "Knowledge base overview"
|
|
39
|
+
|
|
40
|
+
workspace = context.runtime.workspace
|
|
41
|
+
if workspace is None or not workspace.output_dir:
|
|
42
|
+
raise AudioOverviewError(
|
|
43
|
+
i18n.t("missing_workspace", "No writable runtime workspace is available.")
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
from deeptutor.services.prompt import get_prompt_manager
|
|
47
|
+
|
|
48
|
+
prompts = get_prompt_manager().load_prompts(
|
|
49
|
+
"capabilities",
|
|
50
|
+
self.name,
|
|
51
|
+
language=context.language,
|
|
52
|
+
)
|
|
53
|
+
pipeline = AudioOverviewPipeline(
|
|
54
|
+
language=context.language,
|
|
55
|
+
system_prompt=str(prompts.get("system") or ""),
|
|
56
|
+
instruction_prompt=str(prompts.get("instructions") or ""),
|
|
57
|
+
workspace_output_dir=workspace.output_dir,
|
|
58
|
+
workspace_logical_dir=workspace.logical_output_dir,
|
|
59
|
+
workspace_id=workspace.workspace_id,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
turn_id = str(context.metadata.get("turn_id", "") or context.session_id or uuid.uuid4().hex)
|
|
63
|
+
async with stream.stage("retrieving", source=self.name):
|
|
64
|
+
if note_id:
|
|
65
|
+
from deeptutor.capabilities.audio_overview.pipeline import (
|
|
66
|
+
RetrievedContext,
|
|
67
|
+
SourceCitation,
|
|
68
|
+
)
|
|
69
|
+
from deeptutor.modules.catalog import require_module
|
|
70
|
+
|
|
71
|
+
require_module("co-writer")
|
|
72
|
+
from deeptutor.co_writer.storage import get_co_writer_storage
|
|
73
|
+
|
|
74
|
+
note = get_co_writer_storage().load_document(note_id)
|
|
75
|
+
if note is None or not note.content.strip():
|
|
76
|
+
raise AudioOverviewError("The notebook page is missing or empty.")
|
|
77
|
+
retrieved = RetrievedContext(
|
|
78
|
+
answer=note.content[:16000],
|
|
79
|
+
citations=[
|
|
80
|
+
SourceCitation(
|
|
81
|
+
source_id="note-1",
|
|
82
|
+
title=note.title,
|
|
83
|
+
locator=f"/co-writer/{note.id}",
|
|
84
|
+
snippet=note.content[:2000],
|
|
85
|
+
)
|
|
86
|
+
],
|
|
87
|
+
)
|
|
88
|
+
else:
|
|
89
|
+
retrieved = await pipeline.retrieve(
|
|
90
|
+
topic=topic, kb_name=kb_name, request_config=request_config
|
|
91
|
+
)
|
|
92
|
+
if retrieved.citations:
|
|
93
|
+
await stream.sources(
|
|
94
|
+
[
|
|
95
|
+
{"type": "rag", "kb_name": kb_name, **citation.reference_dict()}
|
|
96
|
+
for citation in retrieved.citations
|
|
97
|
+
],
|
|
98
|
+
source=self.name,
|
|
99
|
+
stage="retrieving",
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
async with stream.stage("script_writing", source=self.name):
|
|
103
|
+
script = await pipeline.generate_script(
|
|
104
|
+
topic=topic,
|
|
105
|
+
context=retrieved,
|
|
106
|
+
request_config=request_config,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
artifacts: Any
|
|
110
|
+
async with stream.stage("voice_generation", source=self.name):
|
|
111
|
+
artifacts = await pipeline.publish(
|
|
112
|
+
turn_id=turn_id,
|
|
113
|
+
script=script,
|
|
114
|
+
context=retrieved,
|
|
115
|
+
request_config=request_config,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
async with stream.stage("publishing", source=self.name):
|
|
119
|
+
if artifacts.workspace_items:
|
|
120
|
+
await stream.sources(
|
|
121
|
+
[{"type": "workspace_item", **item} for item in artifacts.workspace_items],
|
|
122
|
+
source=self.name,
|
|
123
|
+
stage="publishing",
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
await emit_capability_result(
|
|
127
|
+
stream,
|
|
128
|
+
{
|
|
129
|
+
"response": artifacts.transcript,
|
|
130
|
+
"title": artifacts.title,
|
|
131
|
+
"kb_name": kb_name,
|
|
132
|
+
"selected_kbs": list(context.knowledge_bases),
|
|
133
|
+
"citations": artifacts.citations,
|
|
134
|
+
"segments": [segment.model_dump() for segment in artifacts.script.segments],
|
|
135
|
+
"audio_path": artifacts.audio_path.name,
|
|
136
|
+
"transcript_path": artifacts.transcript_path.name,
|
|
137
|
+
"workspace_items": artifacts.workspace_items,
|
|
138
|
+
},
|
|
139
|
+
source=self.name,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
__all__ = ["AudioOverviewCapability"]
|
|
@@ -0,0 +1,513 @@
|
|
|
1
|
+
"""Grounded two-voice audio overview generation pipeline."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
import io
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
import re
|
|
11
|
+
import subprocess
|
|
12
|
+
import tempfile
|
|
13
|
+
from typing import Any
|
|
14
|
+
import wave
|
|
15
|
+
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
|
|
17
|
+
|
|
18
|
+
from deeptutor.capabilities.audio_overview.request_config import AudioOverviewRequestConfig
|
|
19
|
+
from deeptutor.utils.json_parser import parse_json_response
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AudioOverviewError(ValueError):
|
|
23
|
+
"""A user-readable failure in one of the deterministic pipeline gates."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ScriptSegment(BaseModel):
|
|
27
|
+
model_config = ConfigDict(extra="ignore")
|
|
28
|
+
|
|
29
|
+
speaker: str
|
|
30
|
+
text: str = Field(min_length=1, max_length=4_000)
|
|
31
|
+
source_ids: list[str] = Field(default_factory=list)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AudioOverviewScript(BaseModel):
|
|
35
|
+
model_config = ConfigDict(extra="ignore")
|
|
36
|
+
|
|
37
|
+
title: str = Field(min_length=1, max_length=200)
|
|
38
|
+
segments: list[ScriptSegment] = Field(min_length=2, max_length=80)
|
|
39
|
+
|
|
40
|
+
@model_validator(mode="after")
|
|
41
|
+
def _validate_alternating_roles(self) -> AudioOverviewScript:
|
|
42
|
+
for index, segment in enumerate(self.segments):
|
|
43
|
+
expected = "host" if index % 2 == 0 else "expert"
|
|
44
|
+
if segment.speaker != expected:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
f"segment {index + 1} must be spoken by {expected}, "
|
|
47
|
+
f"not {segment.speaker or 'unknown'}"
|
|
48
|
+
)
|
|
49
|
+
segment.source_ids = list(dict.fromkeys(segment.source_ids))
|
|
50
|
+
return self
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True, slots=True)
|
|
54
|
+
class SourceCitation:
|
|
55
|
+
source_id: str
|
|
56
|
+
title: str
|
|
57
|
+
locator: str
|
|
58
|
+
snippet: str
|
|
59
|
+
|
|
60
|
+
def reference_dict(self) -> dict[str, str]:
|
|
61
|
+
return {
|
|
62
|
+
"source_id": self.source_id,
|
|
63
|
+
"title": self.title,
|
|
64
|
+
"locator": self.locator,
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True, slots=True)
|
|
69
|
+
class RetrievedContext:
|
|
70
|
+
answer: str
|
|
71
|
+
citations: list[SourceCitation]
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def citation_ids(self) -> set[str]:
|
|
75
|
+
return {citation.source_id for citation in self.citations}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True, slots=True)
|
|
79
|
+
class AudioOverviewArtifacts:
|
|
80
|
+
title: str
|
|
81
|
+
script: AudioOverviewScript
|
|
82
|
+
transcript: str
|
|
83
|
+
audio_path: Path
|
|
84
|
+
transcript_path: Path
|
|
85
|
+
citations: list[dict[str, str]]
|
|
86
|
+
workspace_items: list[dict[str, Any]]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
RagSearchFunc = Callable[..., Awaitable[dict[str, Any]]]
|
|
90
|
+
CompleteFunc = Callable[[str, str], Awaitable[str]]
|
|
91
|
+
SpeechFunc = Callable[..., Awaitable[tuple[bytes, str]]]
|
|
92
|
+
|
|
93
|
+
_MAX_QUERY_CHARS = 1_000
|
|
94
|
+
_MAX_SNIPPET_CHARS = 2_000
|
|
95
|
+
_MAX_CONTEXT_CHARS = 16_000
|
|
96
|
+
_SOURCE_FIELDS = ("sources", "citations", "search_results", "documents")
|
|
97
|
+
_TITLE_FIELDS = ("title", "source", "filename", "file_name", "document", "document_name", "path")
|
|
98
|
+
_LOCATOR_FIELDS = ("locator", "page", "page_number", "section", "url", "source_url")
|
|
99
|
+
_SNIPPET_FIELDS = ("snippet", "content", "text", "preview", "excerpt")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _wave_parameters(audio: bytes) -> tuple[int, int, int] | None:
|
|
103
|
+
try:
|
|
104
|
+
with wave.open(io.BytesIO(audio), "rb") as source:
|
|
105
|
+
return source.getnchannels(), source.getsampwidth(), source.getframerate()
|
|
106
|
+
except (EOFError, wave.Error):
|
|
107
|
+
return None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _render_audio(parts: list[tuple[bytes, str]], output_dir: Path, stem: str) -> Path:
|
|
111
|
+
"""Join decoded audio frames, never whole encoded files or mismatched MIME types."""
|
|
112
|
+
if not parts or any(not audio for audio, _ in parts):
|
|
113
|
+
raise AudioOverviewError("The speech provider returned empty audio.")
|
|
114
|
+
wave_params = [_wave_parameters(audio) for audio, _ in parts]
|
|
115
|
+
if all(params is not None and params == wave_params[0] for params in wave_params):
|
|
116
|
+
output = output_dir / f"{stem}.wav"
|
|
117
|
+
channels, sample_width, sample_rate = wave_params[0]
|
|
118
|
+
with wave.open(str(output), "wb") as target:
|
|
119
|
+
target.setnchannels(channels)
|
|
120
|
+
target.setsampwidth(sample_width)
|
|
121
|
+
target.setframerate(sample_rate)
|
|
122
|
+
for audio, _ in parts:
|
|
123
|
+
with wave.open(io.BytesIO(audio), "rb") as source:
|
|
124
|
+
target.writeframes(source.readframes(source.getnframes()))
|
|
125
|
+
return output
|
|
126
|
+
|
|
127
|
+
output = output_dir / f"{stem}.mp3"
|
|
128
|
+
with tempfile.TemporaryDirectory(prefix="audio-overview-") as directory:
|
|
129
|
+
raw_path = Path(directory) / "joined.pcm"
|
|
130
|
+
try:
|
|
131
|
+
with raw_path.open("wb") as raw:
|
|
132
|
+
for audio, content_type in parts:
|
|
133
|
+
input_args: list[str] = []
|
|
134
|
+
from deeptutor.services.voice.audio import _parse_pcm_content_type
|
|
135
|
+
|
|
136
|
+
pcm = _parse_pcm_content_type(content_type)
|
|
137
|
+
if pcm is not None:
|
|
138
|
+
sample_rate, channels = pcm
|
|
139
|
+
input_args = ["-f", "s16le", "-ar", str(sample_rate), "-ac", str(channels)]
|
|
140
|
+
decoded = subprocess.run( # nosec B607 - fixed ffmpeg argv, no shell; missing binary is caught below
|
|
141
|
+
[
|
|
142
|
+
"ffmpeg",
|
|
143
|
+
"-hide_banner",
|
|
144
|
+
"-loglevel",
|
|
145
|
+
"error",
|
|
146
|
+
"-nostdin",
|
|
147
|
+
*input_args,
|
|
148
|
+
"-i",
|
|
149
|
+
"pipe:0",
|
|
150
|
+
"-f",
|
|
151
|
+
"s16le",
|
|
152
|
+
"-ar",
|
|
153
|
+
"24000",
|
|
154
|
+
"-ac",
|
|
155
|
+
"1",
|
|
156
|
+
"pipe:1",
|
|
157
|
+
],
|
|
158
|
+
input=audio,
|
|
159
|
+
capture_output=True,
|
|
160
|
+
check=False,
|
|
161
|
+
)
|
|
162
|
+
if decoded.returncode != 0 or not decoded.stdout:
|
|
163
|
+
raise AudioOverviewError(
|
|
164
|
+
"The speech provider returned audio that could not be decoded."
|
|
165
|
+
)
|
|
166
|
+
raw.write(decoded.stdout)
|
|
167
|
+
encoded = subprocess.run( # nosec B607 - same fixed ffmpeg argv as the decode step
|
|
168
|
+
[
|
|
169
|
+
"ffmpeg",
|
|
170
|
+
"-hide_banner",
|
|
171
|
+
"-loglevel",
|
|
172
|
+
"error",
|
|
173
|
+
"-nostdin",
|
|
174
|
+
"-f",
|
|
175
|
+
"s16le",
|
|
176
|
+
"-ar",
|
|
177
|
+
"24000",
|
|
178
|
+
"-ac",
|
|
179
|
+
"1",
|
|
180
|
+
"-i",
|
|
181
|
+
str(raw_path),
|
|
182
|
+
"-codec:a",
|
|
183
|
+
"libmp3lame",
|
|
184
|
+
"-q:a",
|
|
185
|
+
"4",
|
|
186
|
+
"-y",
|
|
187
|
+
str(output),
|
|
188
|
+
],
|
|
189
|
+
capture_output=True,
|
|
190
|
+
check=False,
|
|
191
|
+
)
|
|
192
|
+
except FileNotFoundError as exc:
|
|
193
|
+
raise AudioOverviewError("ffmpeg is required to join this speech format.") from exc
|
|
194
|
+
if encoded.returncode != 0 or not output.is_file() or output.stat().st_size == 0:
|
|
195
|
+
raise AudioOverviewError("Could not encode the audio overview.")
|
|
196
|
+
return output
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def parse_script(raw_response: str, citation_ids: set[str]) -> AudioOverviewScript:
|
|
200
|
+
"""Parse and strictly validate the LLM script without repairing citations."""
|
|
201
|
+
|
|
202
|
+
payload = parse_json_response(raw_response, fallback=None)
|
|
203
|
+
if not isinstance(payload, dict):
|
|
204
|
+
raise AudioOverviewError("The audio overview model did not return a JSON object.")
|
|
205
|
+
try:
|
|
206
|
+
script = AudioOverviewScript.model_validate(payload)
|
|
207
|
+
except ValidationError as exc:
|
|
208
|
+
details = "; ".join(
|
|
209
|
+
f"{'.'.join(str(part) for part in error['loc'])}: {error['msg']}"
|
|
210
|
+
for error in exc.errors()
|
|
211
|
+
)
|
|
212
|
+
raise AudioOverviewError(f"Invalid audio overview script: {details}") from exc
|
|
213
|
+
|
|
214
|
+
for index, segment in enumerate(script.segments, start=1):
|
|
215
|
+
unknown = sorted(set(segment.source_ids) - citation_ids)
|
|
216
|
+
if unknown:
|
|
217
|
+
raise AudioOverviewError(f"Invalid citation in segment {index}: {', '.join(unknown)}.")
|
|
218
|
+
return script
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _first_text(row: Mapping[str, Any], fields: tuple[str, ...]) -> str:
|
|
222
|
+
for field in fields:
|
|
223
|
+
value = row.get(field)
|
|
224
|
+
if value is None:
|
|
225
|
+
continue
|
|
226
|
+
text = str(value).strip()
|
|
227
|
+
if text:
|
|
228
|
+
return text
|
|
229
|
+
return ""
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _citation_title(row: Mapping[str, Any], fallback: str) -> str:
|
|
233
|
+
title = _first_text(row, _TITLE_FIELDS)
|
|
234
|
+
if title and ("/" in title or "\\" in title) and not title.startswith("http"):
|
|
235
|
+
title = Path(title).name
|
|
236
|
+
return title[:200] or fallback
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _source_rows(result: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
|
240
|
+
for field in _SOURCE_FIELDS:
|
|
241
|
+
value = result.get(field)
|
|
242
|
+
if not isinstance(value, list):
|
|
243
|
+
continue
|
|
244
|
+
rows = [row for row in value if isinstance(row, Mapping)]
|
|
245
|
+
if rows:
|
|
246
|
+
return rows
|
|
247
|
+
return []
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def normalize_rag_result(result: Mapping[str, Any]) -> RetrievedContext:
|
|
251
|
+
if result.get("error_type") or result.get("needs_reindex"):
|
|
252
|
+
reason = str(
|
|
253
|
+
result.get("answer") or result.get("content") or "the knowledge base is not searchable"
|
|
254
|
+
)
|
|
255
|
+
raise AudioOverviewError(f"RAG retrieval failed: {reason[:500]}")
|
|
256
|
+
|
|
257
|
+
citations: list[SourceCitation] = []
|
|
258
|
+
for index, row in enumerate(_source_rows(result), start=1):
|
|
259
|
+
source_id = f"S{index}"
|
|
260
|
+
snippet = _first_text(row, _SNIPPET_FIELDS)[:_MAX_SNIPPET_CHARS]
|
|
261
|
+
citations.append(
|
|
262
|
+
SourceCitation(
|
|
263
|
+
source_id=source_id,
|
|
264
|
+
title=_citation_title(row, source_id),
|
|
265
|
+
locator=_first_text(row, _LOCATOR_FIELDS)[:200],
|
|
266
|
+
snippet=snippet,
|
|
267
|
+
)
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
answer = str(result.get("answer") or result.get("content") or "").strip()
|
|
271
|
+
if not answer and not citations:
|
|
272
|
+
raise AudioOverviewError("The knowledge base returned no usable content.")
|
|
273
|
+
return RetrievedContext(answer=answer[:_MAX_CONTEXT_CHARS], citations=citations)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _safe_stem(value: str, fallback: str) -> str:
|
|
277
|
+
cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-.")
|
|
278
|
+
return (cleaned[:100] or fallback).lower()
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _transcript(script: AudioOverviewScript, citations: list[SourceCitation], language: str) -> str:
|
|
282
|
+
labels = (
|
|
283
|
+
{"host": "Host", "expert": "Expert"}
|
|
284
|
+
if not language.startswith("zh")
|
|
285
|
+
else {
|
|
286
|
+
"host": "主持人",
|
|
287
|
+
"expert": "专家",
|
|
288
|
+
}
|
|
289
|
+
)
|
|
290
|
+
lines = [f"# {script.title}", ""]
|
|
291
|
+
if citations:
|
|
292
|
+
lines.extend(["## Sources", ""])
|
|
293
|
+
lines.extend(
|
|
294
|
+
f"- [{citation.source_id}] {citation.title}"
|
|
295
|
+
+ (f" — {citation.locator}" if citation.locator else "")
|
|
296
|
+
for citation in citations
|
|
297
|
+
)
|
|
298
|
+
lines.append("")
|
|
299
|
+
lines.append("## Transcript")
|
|
300
|
+
lines.append("")
|
|
301
|
+
for segment in script.segments:
|
|
302
|
+
marker = " ".join(f"[{source_id}]" for source_id in segment.source_ids)
|
|
303
|
+
suffix = f" {marker}" if marker else ""
|
|
304
|
+
lines.append(f"### {labels[segment.speaker]}")
|
|
305
|
+
lines.append("")
|
|
306
|
+
lines.append(f"{segment.text}{suffix}")
|
|
307
|
+
lines.append("")
|
|
308
|
+
return "\n".join(lines).rstrip() + "\n"
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
class AudioOverviewPipeline:
|
|
312
|
+
"""Generate and publish one bounded, KB-grounded overview artifact set."""
|
|
313
|
+
|
|
314
|
+
def __init__(
|
|
315
|
+
self,
|
|
316
|
+
*,
|
|
317
|
+
language: str = "en",
|
|
318
|
+
system_prompt: str,
|
|
319
|
+
instruction_prompt: str,
|
|
320
|
+
rag_search_func: RagSearchFunc | None = None,
|
|
321
|
+
complete_func: CompleteFunc | None = None,
|
|
322
|
+
speech_func: SpeechFunc | None = None,
|
|
323
|
+
workspace_output_dir: str | Path | None = None,
|
|
324
|
+
workspace_logical_dir: str = "",
|
|
325
|
+
workspace_id: str = "",
|
|
326
|
+
) -> None:
|
|
327
|
+
self.language = language
|
|
328
|
+
self.system_prompt = system_prompt
|
|
329
|
+
self.instruction_prompt = instruction_prompt
|
|
330
|
+
self._rag_search_func = rag_search_func
|
|
331
|
+
self._complete_func = complete_func
|
|
332
|
+
self._speech_func = speech_func
|
|
333
|
+
self.workspace_output_dir = (
|
|
334
|
+
Path(workspace_output_dir).resolve() if workspace_output_dir else None
|
|
335
|
+
)
|
|
336
|
+
self.workspace_logical_dir = workspace_logical_dir.strip("/")
|
|
337
|
+
self.workspace_id = workspace_id
|
|
338
|
+
|
|
339
|
+
async def retrieve(
|
|
340
|
+
self, *, topic: str, kb_name: str, request_config: AudioOverviewRequestConfig
|
|
341
|
+
) -> RetrievedContext:
|
|
342
|
+
query = " ".join(part.strip() for part in (topic, "audio overview") if part)
|
|
343
|
+
query = query[:_MAX_QUERY_CHARS]
|
|
344
|
+
rag_search = self._rag_search_func
|
|
345
|
+
if rag_search is None:
|
|
346
|
+
from deeptutor.tools.rag_tool import rag_search as rag_search_impl
|
|
347
|
+
|
|
348
|
+
rag_search = rag_search_impl
|
|
349
|
+
result = await rag_search(
|
|
350
|
+
query,
|
|
351
|
+
kb_name,
|
|
352
|
+
top_k=request_config.max_context_chunks,
|
|
353
|
+
)
|
|
354
|
+
return normalize_rag_result(result)
|
|
355
|
+
|
|
356
|
+
async def generate_script(
|
|
357
|
+
self,
|
|
358
|
+
*,
|
|
359
|
+
topic: str,
|
|
360
|
+
context: RetrievedContext,
|
|
361
|
+
request_config: AudioOverviewRequestConfig,
|
|
362
|
+
) -> AudioOverviewScript:
|
|
363
|
+
complete = self._complete_func
|
|
364
|
+
if complete is None:
|
|
365
|
+
from deeptutor.services.llm import factory as llm_factory
|
|
366
|
+
from deeptutor.services.llm.config import get_llm_config
|
|
367
|
+
|
|
368
|
+
llm_config = get_llm_config()
|
|
369
|
+
|
|
370
|
+
async def complete(prompt: str, system_prompt: str) -> str:
|
|
371
|
+
return await llm_factory.complete_with_config(
|
|
372
|
+
llm_config,
|
|
373
|
+
prompt,
|
|
374
|
+
system_prompt=system_prompt,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
citation_block = (
|
|
378
|
+
"\n".join(
|
|
379
|
+
f"- [{citation.source_id}] {citation.title}"
|
|
380
|
+
+ (f" ({citation.locator})" if citation.locator else "")
|
|
381
|
+
+ f"\n {citation.snippet}"
|
|
382
|
+
for citation in context.citations
|
|
383
|
+
)
|
|
384
|
+
or "- No source metadata was returned; rely only on the retrieved answer."
|
|
385
|
+
)
|
|
386
|
+
language_label = "Simplified Chinese" if self.language.startswith("zh") else "English"
|
|
387
|
+
prompt = (
|
|
388
|
+
f"{self.instruction_prompt}\n\n"
|
|
389
|
+
f"Language: {language_label}\n"
|
|
390
|
+
f"Topic: {topic}\n"
|
|
391
|
+
f"Target duration: about {request_config.target_minutes} minutes\n"
|
|
392
|
+
"Valid citation IDs:\n"
|
|
393
|
+
f"{citation_block}\n\n"
|
|
394
|
+
f"Retrieved context:\n{context.answer or '(no synthesized answer)'}"
|
|
395
|
+
)
|
|
396
|
+
raw = await complete(prompt, self.system_prompt)
|
|
397
|
+
return parse_script(raw, context.citation_ids)
|
|
398
|
+
|
|
399
|
+
async def publish(
|
|
400
|
+
self,
|
|
401
|
+
*,
|
|
402
|
+
turn_id: str,
|
|
403
|
+
script: AudioOverviewScript,
|
|
404
|
+
context: RetrievedContext,
|
|
405
|
+
request_config: AudioOverviewRequestConfig,
|
|
406
|
+
) -> AudioOverviewArtifacts:
|
|
407
|
+
if self.workspace_output_dir is None:
|
|
408
|
+
raise AudioOverviewError("No writable runtime workspace is available.")
|
|
409
|
+
|
|
410
|
+
speech = self._speech_func
|
|
411
|
+
if speech is None:
|
|
412
|
+
from deeptutor.services.voice import synthesize_speech as synthesize_speech_impl
|
|
413
|
+
|
|
414
|
+
speech = synthesize_speech_impl
|
|
415
|
+
|
|
416
|
+
host_voice = request_config.host_voice
|
|
417
|
+
expert_voice = request_config.expert_voice
|
|
418
|
+
if self._speech_func is None and (host_voice is None or expert_voice is None):
|
|
419
|
+
from deeptutor.services.config.provider_runtime import resolve_tts_runtime_config
|
|
420
|
+
from deeptutor.services.voice.options import voice_model_options
|
|
421
|
+
|
|
422
|
+
tts = resolve_tts_runtime_config()
|
|
423
|
+
choices = voice_model_options(tts.provider_name, "tts", tts.model).get("voices", [])
|
|
424
|
+
host_voice = host_voice or tts.voice
|
|
425
|
+
expert_voice = expert_voice or next(
|
|
426
|
+
(str(item["id"]) for item in choices if item.get("id") != host_voice), None
|
|
427
|
+
)
|
|
428
|
+
if not host_voice or not expert_voice:
|
|
429
|
+
raise AudioOverviewError(
|
|
430
|
+
"Configure distinct host and expert voices for the active TTS model."
|
|
431
|
+
)
|
|
432
|
+
else:
|
|
433
|
+
host_voice = host_voice or "host"
|
|
434
|
+
expert_voice = expert_voice or "expert"
|
|
435
|
+
|
|
436
|
+
audio_parts: list[tuple[bytes, str]] = []
|
|
437
|
+
for segment in script.segments:
|
|
438
|
+
voice = host_voice if segment.speaker == "host" else expert_voice
|
|
439
|
+
audio, content_type = await speech(
|
|
440
|
+
segment.text,
|
|
441
|
+
voice=voice,
|
|
442
|
+
response_format="mp3",
|
|
443
|
+
)
|
|
444
|
+
audio_parts.append((audio, content_type))
|
|
445
|
+
|
|
446
|
+
stem = _safe_stem(f"{turn_id}-audio-overview", "audio-overview")
|
|
447
|
+
transcript_path = self.workspace_output_dir / f"{stem}.md"
|
|
448
|
+
transcript = _transcript(script, context.citations, self.language)
|
|
449
|
+
|
|
450
|
+
try:
|
|
451
|
+
self.workspace_output_dir.mkdir(parents=True, exist_ok=True)
|
|
452
|
+
audio_path = await asyncio.to_thread(
|
|
453
|
+
_render_audio, audio_parts, self.workspace_output_dir, stem
|
|
454
|
+
)
|
|
455
|
+
transcript_path.write_text(transcript, encoding="utf-8")
|
|
456
|
+
except OSError as exc:
|
|
457
|
+
raise AudioOverviewError(f"Could not write audio overview artifacts: {exc}") from exc
|
|
458
|
+
|
|
459
|
+
workspace_items = self._publish_workspace_items(
|
|
460
|
+
audio_path,
|
|
461
|
+
transcript_path,
|
|
462
|
+
title=script.title,
|
|
463
|
+
)
|
|
464
|
+
return AudioOverviewArtifacts(
|
|
465
|
+
title=script.title,
|
|
466
|
+
script=script,
|
|
467
|
+
transcript=transcript,
|
|
468
|
+
audio_path=audio_path,
|
|
469
|
+
transcript_path=transcript_path,
|
|
470
|
+
citations=[citation.reference_dict() for citation in context.citations],
|
|
471
|
+
workspace_items=workspace_items,
|
|
472
|
+
)
|
|
473
|
+
|
|
474
|
+
def _publish_workspace_items(
|
|
475
|
+
self,
|
|
476
|
+
audio_path: Path,
|
|
477
|
+
transcript_path: Path,
|
|
478
|
+
*,
|
|
479
|
+
title: str,
|
|
480
|
+
) -> list[dict[str, Any]]:
|
|
481
|
+
if not self.workspace_id or not self.workspace_logical_dir:
|
|
482
|
+
return []
|
|
483
|
+
from deeptutor.services.workspace import get_content_workspace_service
|
|
484
|
+
|
|
485
|
+
service = get_content_workspace_service()
|
|
486
|
+
binding = service.binding_by_id(self.workspace_id)
|
|
487
|
+
logical = self.workspace_logical_dir
|
|
488
|
+
rows = [
|
|
489
|
+
{
|
|
490
|
+
"path": f"{logical}/{audio_path.name}",
|
|
491
|
+
"title": title,
|
|
492
|
+
"caption": "Audio overview",
|
|
493
|
+
},
|
|
494
|
+
{
|
|
495
|
+
"path": f"{logical}/{transcript_path.name}",
|
|
496
|
+
"title": title,
|
|
497
|
+
"caption": "Audio overview transcript",
|
|
498
|
+
},
|
|
499
|
+
]
|
|
500
|
+
items = service.publish(binding, rows)
|
|
501
|
+
return [item.to_dict() for item in items]
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
__all__ = [
|
|
505
|
+
"AudioOverviewArtifacts",
|
|
506
|
+
"AudioOverviewError",
|
|
507
|
+
"AudioOverviewPipeline",
|
|
508
|
+
"AudioOverviewScript",
|
|
509
|
+
"RetrievedContext",
|
|
510
|
+
"SourceCitation",
|
|
511
|
+
"normalize_rag_result",
|
|
512
|
+
"parse_script",
|
|
513
|
+
]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Validated request config for the KB audio overview capability."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AudioOverviewRequestConfig(BaseModel):
|
|
9
|
+
model_config = ConfigDict(extra="forbid")
|
|
10
|
+
|
|
11
|
+
topic: str = Field(default="", max_length=500)
|
|
12
|
+
target_minutes: int = Field(default=5, ge=1, le=15)
|
|
13
|
+
host_voice: str | None = Field(default=None, min_length=1, max_length=64)
|
|
14
|
+
expert_voice: str | None = Field(default=None, min_length=1, max_length=64)
|
|
15
|
+
max_context_chunks: int = Field(default=6, ge=1, le=12)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
__all__ = ["AudioOverviewRequestConfig"]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
deeptutor/capabilities/audio_overview/capability.py,sha256=2WR9B_nJtJHE3SDByqC_ddil2xOrgc_G2JqRAhueAEs,5873
|
|
2
|
+
deeptutor/capabilities/audio_overview/pipeline.py,sha256=lW69765g-evONY4FkstAsmhq0THFdTva1GyWrBYuOzU,18984
|
|
3
|
+
deeptutor/capabilities/audio_overview/request_config.py,sha256=8H7f-m5LRMCLjt32FC-3QRv_EaHs0Pa2OoLOciBzFlo,618
|
|
4
|
+
deeptutor_audio_overview-1.7.0.dist-info/METADATA,sha256=CJ4mMjkbFey51oi4ubMfLDd_vbR2RjMAHmhsbqwf3V4,213
|
|
5
|
+
deeptutor_audio_overview-1.7.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
6
|
+
deeptutor_audio_overview-1.7.0.dist-info/top_level.txt,sha256=hnSHUG_vphMnHpT-DbMqcemCKadJ24laFGAWlovG2xM,10
|
|
7
|
+
deeptutor_audio_overview-1.7.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
deeptutor
|