recording-transcriber 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- recording_transcriber/__init__.py +1 -0
- recording_transcriber/__main__.py +7 -0
- recording_transcriber/audio/__init__.py +1 -0
- recording_transcriber/audio/chunker.py +54 -0
- recording_transcriber/audio/ffmpeg.py +146 -0
- recording_transcriber/backends/__init__.py +1 -0
- recording_transcriber/backends/base.py +67 -0
- recording_transcriber/backends/chat_api.py +181 -0
- recording_transcriber/backends/gemini.py +204 -0
- recording_transcriber/backends/gemini_speech.py +166 -0
- recording_transcriber/backends/mistral_speech.py +53 -0
- recording_transcriber/backends/openai_speech.py +40 -0
- recording_transcriber/backends/registry.py +97 -0
- recording_transcriber/backends/speech_api.py +118 -0
- recording_transcriber/cli.py +155 -0
- recording_transcriber/configuration/__init__.py +1 -0
- recording_transcriber/configuration/config.py +315 -0
- recording_transcriber/configuration/job.py +236 -0
- recording_transcriber/evaluation/__init__.py +1 -0
- recording_transcriber/evaluation/alignment.py +101 -0
- recording_transcriber/evaluation/commands.py +107 -0
- recording_transcriber/evaluation/leaderboard.py +95 -0
- recording_transcriber/evaluation/legacy.py +34 -0
- recording_transcriber/evaluation/manifest.py +75 -0
- recording_transcriber/evaluation/report_data.py +314 -0
- recording_transcriber/evaluation/report_page.html +283 -0
- recording_transcriber/evaluation/report_writer.py +185 -0
- recording_transcriber/evaluation/runner.py +111 -0
- recording_transcriber/evaluation/scoring.py +253 -0
- recording_transcriber/evaluation/spans.py +189 -0
- recording_transcriber/evaluation/spot_checks.py +106 -0
- recording_transcriber/evaluation/words.py +259 -0
- recording_transcriber/models/__init__.py +1 -0
- recording_transcriber/models/chunk_result.py +26 -0
- recording_transcriber/models/chunk_span.py +16 -0
- recording_transcriber/models/document.py +12 -0
- recording_transcriber/models/document_section.py +11 -0
- recording_transcriber/models/job_context.py +11 -0
- recording_transcriber/models/model_response.py +15 -0
- recording_transcriber/models/paragraph.py +12 -0
- recording_transcriber/models/protocol_result.py +16 -0
- recording_transcriber/models/roster_entry.py +12 -0
- recording_transcriber/models/segment.py +14 -0
- recording_transcriber/models/speaker_assignment.py +17 -0
- recording_transcriber/models/speaker_mapping.py +61 -0
- recording_transcriber/models/speech_result.py +13 -0
- recording_transcriber/models/turn.py +11 -0
- recording_transcriber/models/unclear_passage.py +10 -0
- recording_transcriber/models/usage.py +26 -0
- recording_transcriber/notebook.py +126 -0
- recording_transcriber/pipeline/__init__.py +1 -0
- recording_transcriber/pipeline/chunk_transcriber.py +301 -0
- recording_transcriber/pipeline/pipeline.py +521 -0
- recording_transcriber/pipeline/protocol_writer.py +206 -0
- recording_transcriber/pipeline/speaker_reconciler.py +130 -0
- recording_transcriber/pipeline/speech_transcriber.py +156 -0
- recording_transcriber/pipeline/turn_polisher.py +89 -0
- recording_transcriber/prompts/__init__.py +51 -0
- recording_transcriber/prompts/agenda.md +32 -0
- recording_transcriber/prompts/polish.md +19 -0
- recording_transcriber/prompts/prompt_renderer.py +40 -0
- recording_transcriber/prompts/protocol.md +27 -0
- recording_transcriber/prompts/protocol_part.md +25 -0
- recording_transcriber/prompts/reconcile_speakers.md +25 -0
- recording_transcriber/prompts/schemas.py +106 -0
- recording_transcriber/prompts/transcribe.md +42 -0
- recording_transcriber/render/__init__.py +1 -0
- recording_transcriber/render/markdown.py +191 -0
- recording_transcriber/render/paragraphs.py +37 -0
- recording_transcriber/render/quality.py +65 -0
- recording_transcriber/render/records.py +160 -0
- recording_transcriber/render/review.py +83 -0
- recording_transcriber/render/word.py +61 -0
- recording_transcriber/utils/__init__.py +1 -0
- recording_transcriber/utils/checks.py +94 -0
- recording_transcriber/utils/const.py +324 -0
- recording_transcriber/utils/cost.py +20 -0
- recording_transcriber/utils/retry.py +51 -0
- recording_transcriber/utils/roster.py +15 -0
- recording_transcriber/utils/serialize.py +20 -0
- recording_transcriber/utils/timestamps.py +36 -0
- recording_transcriber-1.0.0.dist-info/METADATA +260 -0
- recording_transcriber-1.0.0.dist-info/RECORD +86 -0
- recording_transcriber-1.0.0.dist-info/WHEEL +4 -0
- recording_transcriber-1.0.0.dist-info/entry_points.txt +3 -0
- recording_transcriber-1.0.0.dist-info/licenses/LICENSE +674 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Transcribe meeting recordings into clean transcripts with named speakers."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Audio preparation and chunking."""
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Silence-aligned chunk planning."""
|
|
2
|
+
|
|
3
|
+
from ..models.chunk_span import ChunkSpan
|
|
4
|
+
from ..utils.const import CHUNK_WINDOW_S
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def best_cut(
|
|
8
|
+
silences: list[tuple[float, float]], target_s: float, low_s: float, high_s: float
|
|
9
|
+
) -> float:
|
|
10
|
+
"""Return the middle of the longest silence in [low_s, high_s], or target_s if none.
|
|
11
|
+
|
|
12
|
+
Ties go to the silence closest to the target.
|
|
13
|
+
|
|
14
|
+
silences: the silent stretches as (start, end) seconds
|
|
15
|
+
target_s: the preferred cut
|
|
16
|
+
low_s: the earliest allowed cut
|
|
17
|
+
high_s: the latest allowed cut"""
|
|
18
|
+
candidates = [
|
|
19
|
+
(end - start, -abs((start + end) / 2 - target_s), (start + end) / 2)
|
|
20
|
+
for start, end in silences
|
|
21
|
+
# the window limits are tuning
|
|
22
|
+
if low_s <= (start + end) / 2 <= high_s # pragma: no mutate
|
|
23
|
+
]
|
|
24
|
+
if not candidates:
|
|
25
|
+
return target_s
|
|
26
|
+
return max(candidates)[2]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def plan_chunks(
|
|
30
|
+
duration_s: float,
|
|
31
|
+
silences: list[tuple[float, float]],
|
|
32
|
+
target_s: float,
|
|
33
|
+
window_s: float = CHUNK_WINDOW_S,
|
|
34
|
+
) -> list[ChunkSpan]:
|
|
35
|
+
"""Split a recording into chunks of about target_s, cut at silences.
|
|
36
|
+
|
|
37
|
+
A remainder up to target_s + window_s stays one chunk (25 min target: recordings up
|
|
38
|
+
to 28 min are not split). A remainder shorter than two chunks is cut near its
|
|
39
|
+
middle, so there is never a tiny last chunk.
|
|
40
|
+
|
|
41
|
+
duration_s: the recording length
|
|
42
|
+
silences: the silent stretches as (start, end) seconds
|
|
43
|
+
target_s: the preferred chunk length
|
|
44
|
+
window_s: how far a cut may move from the target (default: CHUNK_WINDOW_S)"""
|
|
45
|
+
spans = []
|
|
46
|
+
start = 0.0
|
|
47
|
+
while duration_s - start > target_s + window_s:
|
|
48
|
+
remaining = duration_s - start
|
|
49
|
+
target = start + min(target_s, remaining / 2)
|
|
50
|
+
cut = best_cut(silences, target, target - window_s, target + window_s)
|
|
51
|
+
spans.append(ChunkSpan(index=len(spans), start_s=start, end_s=cut))
|
|
52
|
+
start = cut
|
|
53
|
+
spans.append(ChunkSpan(index=len(spans), start_s=start, end_s=duration_s))
|
|
54
|
+
return spans
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""ffmpeg wrapper."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from ..utils.const import (
|
|
10
|
+
BITRATE,
|
|
11
|
+
DENOISE_FILTER,
|
|
12
|
+
SAMPLE_RATE,
|
|
13
|
+
SILENCE_MIN_S,
|
|
14
|
+
SILENCE_NOISE,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
_LOGGER = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
_DURATION = re.compile(r"Duration: (\d+):(\d+):(\d+(?:\.\d+)?)")
|
|
20
|
+
_SILENCE_START = re.compile(r"silence_start: (-?\d+(?:\.\d+)?)")
|
|
21
|
+
_SILENCE_END = re.compile(r"silence_end: (\d+(?:\.\d+)?)")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def find_binary() -> str:
|
|
25
|
+
"""Return the system ffmpeg if installed, else the imageio-ffmpeg binary."""
|
|
26
|
+
system = shutil.which("ffmpeg")
|
|
27
|
+
if system is not None:
|
|
28
|
+
return system
|
|
29
|
+
import imageio_ffmpeg
|
|
30
|
+
|
|
31
|
+
return imageio_ffmpeg.get_ffmpeg_exe()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Ffmpeg:
|
|
35
|
+
"""Run ffmpeg for probing, converting, silence detection and cutting."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, binary: str | None = None):
|
|
38
|
+
"""Initialize the wrapper.
|
|
39
|
+
|
|
40
|
+
binary: the ffmpeg executable (default: None, found automatically)"""
|
|
41
|
+
self.binary = binary or find_binary()
|
|
42
|
+
|
|
43
|
+
def duration(self, path: Path) -> float | None:
|
|
44
|
+
"""Return the duration in seconds, parsed from `ffmpeg -i` (no ffprobe needed).
|
|
45
|
+
|
|
46
|
+
path: the audio file"""
|
|
47
|
+
result = self._run(["-i", str(path)], check=False)
|
|
48
|
+
match = _DURATION.search(result.stderr) if result is not None else None
|
|
49
|
+
if match is None:
|
|
50
|
+
_LOGGER.error("Could not read the duration of %s.", path)
|
|
51
|
+
return None
|
|
52
|
+
hours, minutes, seconds = match.groups()
|
|
53
|
+
return int(hours) * 3600 + int(minutes) * 60 + float(seconds)
|
|
54
|
+
|
|
55
|
+
def prepare(self, sources: list[Path], target: Path, denoise: bool) -> bool:
|
|
56
|
+
"""Join the sources into one 16 kHz mono mp3 with loudness normalization.
|
|
57
|
+
|
|
58
|
+
sources: the recordings, in order
|
|
59
|
+
target: the mp3 to write
|
|
60
|
+
denoise: whether to apply the denoise filter"""
|
|
61
|
+
inputs = []
|
|
62
|
+
for source in sources:
|
|
63
|
+
inputs += ["-i", str(source)]
|
|
64
|
+
filters = [DENOISE_FILTER] if denoise else []
|
|
65
|
+
chain = ",".join([*filters, "loudnorm"])
|
|
66
|
+
joined = "".join(f"[{index}:a]" for index in range(len(sources)))
|
|
67
|
+
graph = f"{joined}concat=n={len(sources)}:v=0:a=1,{chain}[out]"
|
|
68
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
69
|
+
args = [
|
|
70
|
+
*inputs,
|
|
71
|
+
"-filter_complex",
|
|
72
|
+
graph,
|
|
73
|
+
"-map",
|
|
74
|
+
"[out]",
|
|
75
|
+
"-ac",
|
|
76
|
+
"1",
|
|
77
|
+
"-ar",
|
|
78
|
+
str(SAMPLE_RATE),
|
|
79
|
+
"-b:a",
|
|
80
|
+
BITRATE,
|
|
81
|
+
"-codec:a",
|
|
82
|
+
"libmp3lame",
|
|
83
|
+
str(target),
|
|
84
|
+
]
|
|
85
|
+
return self._run(args) is not None
|
|
86
|
+
|
|
87
|
+
def silences(self, path: Path) -> list[tuple[float, float]]:
|
|
88
|
+
"""Return the silent stretches as (start, end) seconds.
|
|
89
|
+
|
|
90
|
+
path: the audio file"""
|
|
91
|
+
detector = f"silencedetect=noise={SILENCE_NOISE}:d={SILENCE_MIN_S}"
|
|
92
|
+
# pragma: no mutate start (ffmpeg arguments)
|
|
93
|
+
result = self._run(["-i", str(path), "-af", detector, "-f", "null", "-"])
|
|
94
|
+
# pragma: no mutate end
|
|
95
|
+
if result is None:
|
|
96
|
+
return []
|
|
97
|
+
starts = [
|
|
98
|
+
max(0.0, float(value)) for value in _SILENCE_START.findall(result.stderr)
|
|
99
|
+
]
|
|
100
|
+
ends = [float(value) for value in _SILENCE_END.findall(result.stderr)]
|
|
101
|
+
# a start without an end (the file ends in silence) is dropped
|
|
102
|
+
return list(zip(starts, ends, strict=False))
|
|
103
|
+
|
|
104
|
+
def extract(self, path: Path, start_s: float, end_s: float, target: Path) -> bool:
|
|
105
|
+
"""Copy a time range into a new file without re-encoding.
|
|
106
|
+
|
|
107
|
+
path: the source audio
|
|
108
|
+
start_s: the range start in seconds
|
|
109
|
+
end_s: the range end in seconds
|
|
110
|
+
target: the file to write"""
|
|
111
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
args = [
|
|
113
|
+
"-ss",
|
|
114
|
+
f"{start_s:.3f}",
|
|
115
|
+
"-to",
|
|
116
|
+
f"{end_s:.3f}",
|
|
117
|
+
"-i",
|
|
118
|
+
str(path),
|
|
119
|
+
"-c",
|
|
120
|
+
"copy",
|
|
121
|
+
str(target),
|
|
122
|
+
]
|
|
123
|
+
return self._run(args) is not None
|
|
124
|
+
|
|
125
|
+
def _run(
|
|
126
|
+
self, args: list[str], check: bool = True
|
|
127
|
+
) -> subprocess.CompletedProcess | None:
|
|
128
|
+
"""Run ffmpeg; return None and log an error if it fails.
|
|
129
|
+
|
|
130
|
+
args: the arguments after the binary
|
|
131
|
+
check: whether a non-zero exit code is an error (default: True)"""
|
|
132
|
+
command = [self.binary, "-hide_banner", "-nostdin", "-y", *args]
|
|
133
|
+
try:
|
|
134
|
+
result = subprocess.run(
|
|
135
|
+
command,
|
|
136
|
+
check=False,
|
|
137
|
+
capture_output=True,
|
|
138
|
+
text=True,
|
|
139
|
+
)
|
|
140
|
+
except OSError as error:
|
|
141
|
+
_LOGGER.error("Could not run ffmpeg: %s.", error)
|
|
142
|
+
return None
|
|
143
|
+
if check and result.returncode != 0:
|
|
144
|
+
_LOGGER.error("ffmpeg failed: %s.", result.stderr.strip()[-500:])
|
|
145
|
+
return None
|
|
146
|
+
return result
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Model provider backends."""
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Backend interfaces."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Protocol
|
|
5
|
+
|
|
6
|
+
from ..models.model_response import ModelResponse
|
|
7
|
+
from ..models.speech_result import SpeechResult
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TextBackend(Protocol):
|
|
11
|
+
"""A provider for text-only steps (speaker reconciliation, protocol)."""
|
|
12
|
+
|
|
13
|
+
def complete(self, prompt: str, schema: dict, model: str) -> ModelResponse | None:
|
|
14
|
+
"""Return the JSON answer to a prompt, or None if the call failed.
|
|
15
|
+
|
|
16
|
+
prompt: the instructions
|
|
17
|
+
schema: the JSON schema of the answer
|
|
18
|
+
model: the model id"""
|
|
19
|
+
|
|
20
|
+
def describe(self) -> dict:
|
|
21
|
+
"""Return provider, endpoint and logging setting for run.json."""
|
|
22
|
+
|
|
23
|
+
def check(self, model: str) -> bool:
|
|
24
|
+
"""Return whether the key works and the model exists (cheap, before real work).
|
|
25
|
+
|
|
26
|
+
model: the model id"""
|
|
27
|
+
|
|
28
|
+
def close(self) -> None:
|
|
29
|
+
"""Delete anything left at the provider (uploads) and release the client."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class TranscriptionBackend(TextBackend, Protocol):
|
|
33
|
+
"""A provider that transcribes audio."""
|
|
34
|
+
|
|
35
|
+
def transcribe(
|
|
36
|
+
self, audio: Path, prompt: str, schema: dict, model: str
|
|
37
|
+
) -> ModelResponse | None:
|
|
38
|
+
"""Return the JSON transcription of an audio file, or None if the call failed.
|
|
39
|
+
|
|
40
|
+
audio: the audio file
|
|
41
|
+
prompt: the instructions
|
|
42
|
+
schema: the JSON schema of the answer
|
|
43
|
+
model: the model id"""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class SpeechBackend(Protocol):
|
|
47
|
+
"""A speech-to-text provider: audio in, segments out, no prompt."""
|
|
48
|
+
|
|
49
|
+
def check(self, model: str) -> bool:
|
|
50
|
+
"""Return whether the key works and the model exists (cheap, before real work).
|
|
51
|
+
|
|
52
|
+
model: the model id"""
|
|
53
|
+
|
|
54
|
+
def transcribe_speech(
|
|
55
|
+
self, audio: Path, model: str, vocabulary: list[str]
|
|
56
|
+
) -> SpeechResult | None:
|
|
57
|
+
"""Return the segments of an audio file, or None if the call failed.
|
|
58
|
+
|
|
59
|
+
audio: the audio file
|
|
60
|
+
model: the model id
|
|
61
|
+
vocabulary: terms to prefer, if the model takes them"""
|
|
62
|
+
|
|
63
|
+
def describe(self) -> dict:
|
|
64
|
+
"""Return provider, endpoint and logging setting for run.json."""
|
|
65
|
+
|
|
66
|
+
def close(self) -> None:
|
|
67
|
+
"""Delete anything left at the provider (uploads)."""
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Text backend for OpenAI-compatible /chat/completions APIs."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from ..models.model_response import ModelResponse
|
|
8
|
+
from ..models.usage import Usage
|
|
9
|
+
from ..utils.const import (
|
|
10
|
+
CHAT_ROUTE,
|
|
11
|
+
CHAT_SCHEMA_NAME,
|
|
12
|
+
MODELS_ROUTE,
|
|
13
|
+
REQUEST_TIMEOUT_MS,
|
|
14
|
+
STATUS_COMPLETED,
|
|
15
|
+
STATUS_INCOMPLETE,
|
|
16
|
+
STATUS_UNKNOWN,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
_LOGGER = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
# the finish reasons of a chat completion: a normal stop, and a cut-off at the limit
|
|
22
|
+
_FINISH_STATUS = {"stop": STATUS_COMPLETED, "length": STATUS_INCOMPLETE}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ChatApiBackend:
|
|
26
|
+
"""Complete text prompts with the JSON-schema structured output of an
|
|
27
|
+
OpenAI-compatible chat API (OpenAI, Mistral or a custom endpoint).
|
|
28
|
+
|
|
29
|
+
Nothing is stored at the provider, so there is nothing to delete."""
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
provider: str,
|
|
34
|
+
base_url: str,
|
|
35
|
+
api_key: str,
|
|
36
|
+
data_policy: str,
|
|
37
|
+
client: httpx.Client | None = None,
|
|
38
|
+
):
|
|
39
|
+
"""Initialize the backend.
|
|
40
|
+
|
|
41
|
+
provider: the provider name, for logs and run.json
|
|
42
|
+
base_url: the API root; /chat/completions is appended
|
|
43
|
+
api_key: the provider key
|
|
44
|
+
data_policy: the provider's logging and retention terms, for run.json
|
|
45
|
+
client: a preconfigured client, used by tests (default: None)"""
|
|
46
|
+
self.provider = provider
|
|
47
|
+
root = base_url.removesuffix("/")
|
|
48
|
+
self.url = root + CHAT_ROUTE
|
|
49
|
+
self.models_url = root + MODELS_ROUTE
|
|
50
|
+
self.data_policy = data_policy
|
|
51
|
+
self.__api_key = api_key
|
|
52
|
+
self.__client = client or httpx.Client(timeout=REQUEST_TIMEOUT_MS / 1000)
|
|
53
|
+
|
|
54
|
+
def describe(self) -> dict:
|
|
55
|
+
"""Return provider, endpoint and logging setting for run.json."""
|
|
56
|
+
return {
|
|
57
|
+
"provider": self.provider,
|
|
58
|
+
"endpoint": self.url,
|
|
59
|
+
"logging": self.data_policy,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
def check(self, model: str) -> bool:
|
|
63
|
+
"""Return whether the key works and the provider lists the model.
|
|
64
|
+
|
|
65
|
+
Only a rejected key or a listing without the model fail: some keys and
|
|
66
|
+
gateways cannot list models but still complete text.
|
|
67
|
+
|
|
68
|
+
model: the model id"""
|
|
69
|
+
try:
|
|
70
|
+
response = self.__client.get(
|
|
71
|
+
self.models_url,
|
|
72
|
+
headers=self._headers(),
|
|
73
|
+
)
|
|
74
|
+
except httpx.HTTPError as error:
|
|
75
|
+
_LOGGER.error(
|
|
76
|
+
"Could not access %s model %s: %s.", self.provider, model, error
|
|
77
|
+
)
|
|
78
|
+
return False
|
|
79
|
+
if response.status_code == httpx.codes.UNAUTHORIZED:
|
|
80
|
+
_LOGGER.error(
|
|
81
|
+
"%s rejected the key: %s.", self.provider, response.text[:300]
|
|
82
|
+
)
|
|
83
|
+
return False
|
|
84
|
+
try:
|
|
85
|
+
response.raise_for_status()
|
|
86
|
+
listed = _listed(response.json()["data"])
|
|
87
|
+
except (httpx.HTTPStatusError, ValueError, KeyError, TypeError) as error:
|
|
88
|
+
_LOGGER.warning(
|
|
89
|
+
"Could not list the %s models (%s); trying %s anyway.",
|
|
90
|
+
self.provider,
|
|
91
|
+
error,
|
|
92
|
+
model,
|
|
93
|
+
)
|
|
94
|
+
return True
|
|
95
|
+
if model not in listed:
|
|
96
|
+
_LOGGER.error("%s does not list the model %s.", self.provider, model)
|
|
97
|
+
return False
|
|
98
|
+
return True
|
|
99
|
+
|
|
100
|
+
def complete(self, prompt: str, schema: dict, model: str) -> ModelResponse | None:
|
|
101
|
+
"""Return the JSON answer to a text prompt, or None if the call failed.
|
|
102
|
+
|
|
103
|
+
prompt: the instructions
|
|
104
|
+
schema: the JSON schema of the answer
|
|
105
|
+
model: the model id"""
|
|
106
|
+
# pragma: no mutate start (wire format)
|
|
107
|
+
body = {
|
|
108
|
+
"model": model,
|
|
109
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
110
|
+
"response_format": {
|
|
111
|
+
"type": "json_schema",
|
|
112
|
+
"json_schema": {"name": CHAT_SCHEMA_NAME, "schema": schema},
|
|
113
|
+
},
|
|
114
|
+
}
|
|
115
|
+
# pragma: no mutate end
|
|
116
|
+
try:
|
|
117
|
+
response = self.__client.post(
|
|
118
|
+
self.url,
|
|
119
|
+
headers=self._headers(),
|
|
120
|
+
json=body,
|
|
121
|
+
)
|
|
122
|
+
except httpx.HTTPError as error:
|
|
123
|
+
_LOGGER.error(
|
|
124
|
+
"Could not call %s model %s: %s.", self.provider, model, error
|
|
125
|
+
)
|
|
126
|
+
return None
|
|
127
|
+
if response.status_code != httpx.codes.OK:
|
|
128
|
+
_LOGGER.error(
|
|
129
|
+
"%s model %s returned %s: %s.",
|
|
130
|
+
self.provider,
|
|
131
|
+
model,
|
|
132
|
+
response.status_code,
|
|
133
|
+
response.text[:300],
|
|
134
|
+
)
|
|
135
|
+
return None
|
|
136
|
+
try:
|
|
137
|
+
return _parse(response.json())
|
|
138
|
+
except (ValueError, KeyError, IndexError, TypeError, AttributeError) as error:
|
|
139
|
+
_LOGGER.error("Invalid answer from %s: %s.", self.provider, error)
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
def _headers(self) -> dict:
|
|
143
|
+
"""Return the request headers with the key."""
|
|
144
|
+
# header names are case-insensitive
|
|
145
|
+
return {"Authorization": f"Bearer {self.__api_key}"}
|
|
146
|
+
|
|
147
|
+
def close(self) -> None:
|
|
148
|
+
"""Close the HTTP client; nothing is stored at the provider."""
|
|
149
|
+
self.__client.close()
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _listed(entries: list[dict]) -> set[str]:
|
|
153
|
+
"""Return the model ids and aliases of a model list, without a "models/" prefix.
|
|
154
|
+
|
|
155
|
+
entries: the "data" entries of the answer"""
|
|
156
|
+
names = set()
|
|
157
|
+
for entry in entries:
|
|
158
|
+
names.update([entry["id"], *(entry.get("aliases") or [])])
|
|
159
|
+
return {name.removeprefix("models/") for name in names}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _parse(data: dict) -> ModelResponse:
|
|
163
|
+
"""Return the first choice and the usage of a chat completion; raise on an
|
|
164
|
+
unexpected shape.
|
|
165
|
+
|
|
166
|
+
data: the JSON answer"""
|
|
167
|
+
choice = data["choices"][0]
|
|
168
|
+
usage = data.get("usage") or {}
|
|
169
|
+
# reasoning tokens are part of completion_tokens; Usage counts them separately
|
|
170
|
+
reasoning = (usage.get("completion_tokens_details") or {}).get(
|
|
171
|
+
"reasoning_tokens"
|
|
172
|
+
) or 0
|
|
173
|
+
return ModelResponse(
|
|
174
|
+
text=choice["message"]["content"],
|
|
175
|
+
status=_FINISH_STATUS.get(choice.get("finish_reason"), STATUS_UNKNOWN),
|
|
176
|
+
usage=Usage(
|
|
177
|
+
input_text_tokens=usage.get("prompt_tokens") or 0,
|
|
178
|
+
output_tokens=(usage.get("completion_tokens") or 0) - reasoning,
|
|
179
|
+
thought_tokens=reasoning,
|
|
180
|
+
),
|
|
181
|
+
)
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Google Gemini backend (Developer API, Interactions API)."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import time
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
from google import genai
|
|
9
|
+
from google.genai import errors
|
|
10
|
+
|
|
11
|
+
from ..models.model_response import ModelResponse
|
|
12
|
+
from ..models.usage import Usage
|
|
13
|
+
from ..utils.const import (
|
|
14
|
+
GEMINI_ENDPOINT,
|
|
15
|
+
GEMINI_LOGGING,
|
|
16
|
+
GEMINI_PROVIDER,
|
|
17
|
+
MAX_OUTPUT_TOKENS,
|
|
18
|
+
REQUEST_TIMEOUT_MS,
|
|
19
|
+
STATUS_UNKNOWN,
|
|
20
|
+
UPLOAD_POLL_S,
|
|
21
|
+
UPLOAD_TIMEOUT_S,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
_LOGGER = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class GeminiBackend:
|
|
28
|
+
"""Transcribe and complete text with Gemini models.
|
|
29
|
+
|
|
30
|
+
Audio goes through the Files API and is deleted right after each call."""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
api_key: str | None = None,
|
|
35
|
+
thinking_level: str = "low",
|
|
36
|
+
client: genai.Client | None = None,
|
|
37
|
+
):
|
|
38
|
+
"""Initialize the backend.
|
|
39
|
+
|
|
40
|
+
api_key: the paid-tier AI Studio key (default: None, only with a client)
|
|
41
|
+
thinking_level: the Gemini thinking level (default: low)
|
|
42
|
+
client: a preconfigured client, used by tests (default: None)"""
|
|
43
|
+
self.thinking_level = thinking_level
|
|
44
|
+
# the client also reads the key from the environment; key routing is tested
|
|
45
|
+
# in the registry
|
|
46
|
+
# pragma: no mutate start (client setup)
|
|
47
|
+
self._client = client or genai.Client(
|
|
48
|
+
api_key=api_key, http_options={"timeout": REQUEST_TIMEOUT_MS}
|
|
49
|
+
)
|
|
50
|
+
# pragma: no mutate end
|
|
51
|
+
self._uploads = set()
|
|
52
|
+
|
|
53
|
+
def describe(self) -> dict:
|
|
54
|
+
"""Return provider, endpoint and logging setting for run.json."""
|
|
55
|
+
return {
|
|
56
|
+
"provider": GEMINI_PROVIDER,
|
|
57
|
+
"endpoint": GEMINI_ENDPOINT,
|
|
58
|
+
"logging": GEMINI_LOGGING,
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
def check(self, model: str) -> bool:
|
|
62
|
+
"""Return whether the key works and the model exists.
|
|
63
|
+
|
|
64
|
+
model: the model id"""
|
|
65
|
+
try:
|
|
66
|
+
self._client.models.get(model=model)
|
|
67
|
+
except (errors.APIError, httpx.HTTPError) as error:
|
|
68
|
+
_LOGGER.error("Could not access Gemini model %s: %s.", model, error)
|
|
69
|
+
return False
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
def transcribe(
|
|
73
|
+
self, audio: Path, prompt: str, schema: dict, model: str
|
|
74
|
+
) -> ModelResponse | None:
|
|
75
|
+
"""Upload the audio, transcribe it and delete the upload.
|
|
76
|
+
|
|
77
|
+
audio: the mp3 file
|
|
78
|
+
prompt: the instructions
|
|
79
|
+
schema: the JSON schema of the answer
|
|
80
|
+
model: the model id"""
|
|
81
|
+
uploaded = self._upload(audio)
|
|
82
|
+
if uploaded is None:
|
|
83
|
+
return None
|
|
84
|
+
try:
|
|
85
|
+
content = [
|
|
86
|
+
{"type": "text", "text": prompt},
|
|
87
|
+
{
|
|
88
|
+
"type": "audio",
|
|
89
|
+
"uri": uploaded.uri,
|
|
90
|
+
"mime_type": "audio/mpeg",
|
|
91
|
+
},
|
|
92
|
+
]
|
|
93
|
+
return self._stream(model, content, schema)
|
|
94
|
+
finally:
|
|
95
|
+
self._delete(uploaded.name)
|
|
96
|
+
|
|
97
|
+
def complete(self, prompt: str, schema: dict, model: str) -> ModelResponse | None:
|
|
98
|
+
"""Return the JSON answer to a text prompt.
|
|
99
|
+
|
|
100
|
+
prompt: the instructions
|
|
101
|
+
schema: the JSON schema of the answer
|
|
102
|
+
model: the model id"""
|
|
103
|
+
return self._stream(model, prompt, schema)
|
|
104
|
+
|
|
105
|
+
def close(self) -> None:
|
|
106
|
+
"""Delete uploads left by an interrupted call."""
|
|
107
|
+
for name in list(self._uploads):
|
|
108
|
+
self._delete(name)
|
|
109
|
+
|
|
110
|
+
def _upload(self, audio: Path):
|
|
111
|
+
"""Upload a file and wait until it is usable; return None on failure.
|
|
112
|
+
|
|
113
|
+
audio: the mp3 file"""
|
|
114
|
+
try:
|
|
115
|
+
uploaded = self._client.files.upload(
|
|
116
|
+
file=audio, # pragma: no mutate (request wiring)
|
|
117
|
+
config={"mime_type": "audio/mpeg"},
|
|
118
|
+
)
|
|
119
|
+
self._uploads.add(uploaded.name)
|
|
120
|
+
deadline = time.monotonic() + UPLOAD_TIMEOUT_S
|
|
121
|
+
# state is an SDK enum (FileState.ACTIVE); its string form is stable
|
|
122
|
+
while not str(uploaded.state).endswith("ACTIVE"):
|
|
123
|
+
if (
|
|
124
|
+
str(uploaded.state).endswith("FAILED")
|
|
125
|
+
or time.monotonic() > deadline # pragma: no mutate (threshold)
|
|
126
|
+
):
|
|
127
|
+
_LOGGER.error("Upload of %s did not become active.", audio.name)
|
|
128
|
+
self._delete(uploaded.name)
|
|
129
|
+
return None
|
|
130
|
+
time.sleep(UPLOAD_POLL_S)
|
|
131
|
+
uploaded = self._client.files.get(name=uploaded.name)
|
|
132
|
+
except (errors.APIError, httpx.HTTPError, OSError) as error:
|
|
133
|
+
_LOGGER.error("Could not upload %s to Gemini: %s.", audio.name, error)
|
|
134
|
+
return None
|
|
135
|
+
return uploaded
|
|
136
|
+
|
|
137
|
+
def _delete(self, name: str) -> None:
|
|
138
|
+
"""Delete an uploaded file; failures are logged (Google deletes it after 48 h).
|
|
139
|
+
|
|
140
|
+
name: the file resource name"""
|
|
141
|
+
try:
|
|
142
|
+
self._client.files.delete(name=name)
|
|
143
|
+
self._uploads.discard(name)
|
|
144
|
+
except (errors.APIError, httpx.HTTPError) as error:
|
|
145
|
+
_LOGGER.error("Could not delete Gemini upload %s: %s.", name, error)
|
|
146
|
+
|
|
147
|
+
def _stream(self, model: str, content, schema: dict) -> ModelResponse | None:
|
|
148
|
+
"""Run a streaming interaction and collect the text and final status.
|
|
149
|
+
|
|
150
|
+
model: the model id
|
|
151
|
+
content: the input (text or a list of content parts)
|
|
152
|
+
schema: the JSON schema of the answer"""
|
|
153
|
+
parts = []
|
|
154
|
+
status = STATUS_UNKNOWN
|
|
155
|
+
usage = Usage()
|
|
156
|
+
try:
|
|
157
|
+
# pragma: no mutate start (wire format)
|
|
158
|
+
response_format = {"mime_type": "application/json", "schema": schema}
|
|
159
|
+
# pragma: no mutate end
|
|
160
|
+
stream = self._client.interactions.create(
|
|
161
|
+
model=model,
|
|
162
|
+
input=content,
|
|
163
|
+
stream=True,
|
|
164
|
+
generation_config={
|
|
165
|
+
"thinking_level": self.thinking_level,
|
|
166
|
+
"max_output_tokens": MAX_OUTPUT_TOKENS,
|
|
167
|
+
},
|
|
168
|
+
response_format=response_format,
|
|
169
|
+
)
|
|
170
|
+
for event in stream:
|
|
171
|
+
# events always carry a type; the default is defensive
|
|
172
|
+
event_type = getattr(event, "event_type", None) # pragma: no mutate
|
|
173
|
+
if event_type == "step.delta" and event.delta.type == "text":
|
|
174
|
+
parts.append(event.delta.text)
|
|
175
|
+
elif event_type == "interaction.status_update":
|
|
176
|
+
status = event.status
|
|
177
|
+
elif event_type == "interaction.completed":
|
|
178
|
+
status = event.interaction.status or status
|
|
179
|
+
usage = interaction_usage(event.interaction.usage)
|
|
180
|
+
elif event_type == "error":
|
|
181
|
+
_LOGGER.error("Gemini stream error for %s: %s.", model, event)
|
|
182
|
+
return None
|
|
183
|
+
except (errors.APIError, httpx.HTTPError) as error:
|
|
184
|
+
_LOGGER.error("Could not call Gemini model %s: %s.", model, error)
|
|
185
|
+
return None
|
|
186
|
+
return ModelResponse(text="".join(parts), status=status, usage=usage)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def interaction_usage(usage) -> Usage:
|
|
190
|
+
"""Convert the Interactions API usage into Usage.
|
|
191
|
+
|
|
192
|
+
usage: the usage object of the completed interaction (may be None)"""
|
|
193
|
+
if usage is None:
|
|
194
|
+
return Usage()
|
|
195
|
+
by_modality = {
|
|
196
|
+
str(item.modality).lower(): item.tokens or 0
|
|
197
|
+
for item in usage.input_tokens_by_modality or []
|
|
198
|
+
}
|
|
199
|
+
return Usage(
|
|
200
|
+
input_text_tokens=by_modality.get("text", 0),
|
|
201
|
+
input_audio_tokens=by_modality.get("audio", 0),
|
|
202
|
+
output_tokens=usage.total_output_tokens or 0,
|
|
203
|
+
thought_tokens=usage.total_thought_tokens or 0,
|
|
204
|
+
)
|