pytest-audioeval 0.1.0__py3-none-any.whl

pytest_audioeval/client.py

@@ -0,0 +1,27 @@
+"""AudioEval facade — main fixture interface."""
+
+from __future__ import annotations
+
+import contextlib
+
+from pytest_audioeval.samples.registry import SampleRegistry
+from pytest_audioeval.stt import STTClient
+from pytest_audioeval.tts import TTSClient
+
+
+class AudioEval:
+    """Pytest fixture facade: audioeval.stt / audioeval.tts / audioeval.samples."""
+
+    __slots__ = ("samples", "stt", "tts")
+
+    def __init__(self, *, stt_url: str | None = None, tts_url: str | None = None) -> None:
+        self.samples = SampleRegistry()
+        self.stt: STTClient | None = STTClient(url=stt_url) if stt_url else None
+        self.tts: TTSClient | None = TTSClient(url=tts_url) if tts_url else None
+
+    async def aclose(self) -> None:
+        """Cleanup clients."""
+        for client in (self.stt, self.tts):
+            if client is not None:
+                with contextlib.suppress(RuntimeError):
+                    await client.aclose()

pytest_audioeval/metrics/audio.py

@@ -0,0 +1,28 @@
+"""Perceptual audio quality via PESQ."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Self
+
+import numpy as np
+from pesq import pesq as pesq_score
+
+
+@dataclass(slots=True, frozen=True)
+class AudioMetrics:
+    """PESQ MOS (1-5 scale)."""
+
+    mos: float
+    sample_rate: int
+
+    @classmethod
+    def compute(cls, reference: np.ndarray, hypothesis: np.ndarray, *, sample_rate: int = 16_000) -> Self:
+        """PESQ wideband comparison."""
+        score = pesq_score(sample_rate, reference, hypothesis, "wb")
+        return cls(mos=float(score), sample_rate=sample_rate)
+
+    def assert_quality(self, *, min_mos: float = 3.0) -> None:
+        """Raise AssertionError if MOS below threshold."""
+        if self.mos < min_mos:
+            raise AssertionError(f"PESQ MOS {self.mos:.2f} < {min_mos}")

pytest_audioeval/metrics/text.py

@@ -0,0 +1,45 @@
+"""Word/character-level transcription quality metrics."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Self
+
+from jiwer import cer, process_words, wer
+
+
+@dataclass(slots=True, frozen=True)
+class TextMetrics:
+    """WER/CER transcription quality. O(n) via jiwer."""
+
+    wer: float
+    cer: float
+    substitutions: int
+    insertions: int
+    deletions: int
+
+    @classmethod
+    def compute(cls, reference: str, hypothesis: str) -> Self:
+        """Compute all text metrics in a single pass."""
+        output = process_words(reference, hypothesis)
+        return cls(
+            wer=wer(reference, hypothesis),
+            cer=cer(reference, hypothesis),
+            substitutions=output.substitutions,
+            insertions=output.insertions,
+            deletions=output.deletions,
+        )
+
+    def assert_quality(self, *, max_wer: float = 0.2, max_cer: float = 0.15) -> None:
+        """Raise AssertionError with detailed breakdown on failure."""
+        violations: list[str] = []
+        if self.wer > max_wer:
+            violations.append(f"WER {self.wer:.3f} > {max_wer}")
+        if self.cer > max_cer:
+            violations.append(f"CER {self.cer:.3f} > {max_cer}")
+        if violations:
+            msg = (
+                f"Audio quality assertion failed: {', '.join(violations)} | "
+                f"subs={self.substitutions} ins={self.insertions} del={self.deletions}"
+            )
+            raise AssertionError(msg)

pytest_audioeval/plugin.py

@@ -0,0 +1,41 @@
+"""pytest entry point — fixtures and hooks."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import Any
+
+import pytest
+
+from pytest_audioeval.client import AudioEval
+
+
+def pytest_addoption(parser: Any) -> None:
+    """Register CLI options for audioeval."""
+    group = parser.getgroup("audioeval", "Audio evaluation options")
+    group.addoption("--stt-url", default=None, help="STT service WebSocket URL")
+    group.addoption("--tts-url", default=None, help="TTS service HTTP URL")
+    group.addoption("--audioeval-wer", type=float, default=0.2, help="Max WER threshold")
+    group.addoption("--audioeval-cer", type=float, default=0.15, help="Max CER threshold")
+    group.addoption("--audioeval-mos", type=float, default=3.0, help="Min PESQ MOS threshold")
+
+
+@pytest.fixture(scope="session")
+async def audioeval(request: pytest.FixtureRequest) -> AsyncIterator[AudioEval]:
+    """Session-scoped AudioEval fixture with CLI-driven configuration."""
+    stt_url = request.config.getoption("--stt-url")
+    tts_url = request.config.getoption("--tts-url")
+
+    ae = AudioEval(stt_url=stt_url, tts_url=tts_url)
+    yield ae
+    await ae.aclose()
+
+
+@pytest.fixture
+def audioeval_thresholds(request: pytest.FixtureRequest) -> dict[str, float]:
+    """Per-test threshold overrides from CLI."""
+    return {
+        "max_wer": request.config.getoption("--audioeval-wer"),
+        "max_cer": request.config.getoption("--audioeval-cer"),
+        "min_mos": request.config.getoption("--audioeval-mos"),
+    }

pytest_audioeval/samples/audio/en/counting.txt

@@ -0,0 +1 @@
+One two three four five six seven eight nine ten.

pytest_audioeval/samples/audio/en/hello_world.txt

@@ -0,0 +1 @@
+Hello world.

pytest_audioeval/samples/audio/en/quick_brown_fox.txt

@@ -0,0 +1 @@
+The quick brown fox jumps over the lazy dog.

pytest_audioeval/samples/registry.py

@@ -0,0 +1,114 @@
+"""Sample catalog and ground-truth registry."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum, auto
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+
+
+class SampleLang(StrEnum):
+    """Supported sample languages."""
+
+    EN = auto()
+    ES = auto()
+    FR = auto()
+    DE = auto()
+
+
+@dataclass(slots=True, frozen=True)
+class AudioSample:
+    """Ground-truth pair: audio + expected transcription."""
+
+    name: str
+    lang: SampleLang
+    reference_text: str
+    audio_path: Path
+    sample_rate: int = 16_000
+    duration_ms: int = 0
+
+    def audio_bytes(self) -> bytes:
+        """Raw file bytes."""
+        return self.audio_path.read_bytes()
+
+    def audio_numpy(self) -> np.ndarray:
+        """Load as float32 numpy array."""
+        data, _ = sf.read(self.audio_path, dtype="float32")
+        return data
+
+    def chunks(self, chunk_ms: int = 200) -> list[bytes]:
+        """Split audio into float32 PCM chunks for streaming. O(n/chunk_size)."""
+        data = self.audio_numpy()
+        samples_per_chunk = (self.sample_rate * chunk_ms) // 1000
+        return [data[i : i + samples_per_chunk].tobytes() for i in range(0, len(data), samples_per_chunk)]
+
+
+_SAMPLES_DIR = Path(__file__).resolve().parent / "audio"
+
+
+class SampleRegistry:
+    """Catalog of embedded audio fixtures. O(1) lookup by name."""
+
+    __slots__ = ("_catalog",)
+
+    def __init__(self) -> None:
+        self._catalog: dict[str, AudioSample] = {}
+        self._discover()
+
+    def _discover(self) -> None:
+        """Auto-register samples from directory convention: {lang}/{name}.wav + .txt."""
+        if not _SAMPLES_DIR.exists():
+            return
+        for lang_dir in sorted(_SAMPLES_DIR.iterdir()):
+            if not lang_dir.is_dir():
+                continue
+            lang_code = lang_dir.name
+            if lang_code not in SampleLang.__members__.values():
+                continue
+            lang = SampleLang(lang_code)
+            for wav_path in sorted(lang_dir.glob("*.wav")):
+                txt_path = wav_path.with_suffix(".txt")
+                if not txt_path.exists():
+                    continue
+                name = wav_path.stem
+                key = f"{lang_code}_{name}"
+                info = sf.info(wav_path)
+                self._catalog[key] = AudioSample(
+                    name=name,
+                    lang=lang,
+                    reference_text=txt_path.read_text().strip(),
+                    audio_path=wav_path,
+                    sample_rate=int(info.samplerate),
+                    duration_ms=int(info.duration * 1000),
+                )
+
+    def __getattr__(self, name: str) -> AudioSample:
+        """Attribute-style access: samples.en_hello_world."""
+        if name in self._catalog:
+            return self._catalog[name]
+        available = ", ".join(sorted(self._catalog))
+        raise AttributeError(f"Sample '{name}' not found. Available: {available}")
+
+    def all(self) -> list[AudioSample]:
+        """All registered samples."""
+        return list(self._catalog.values())
+
+    def by_lang(self, lang: SampleLang) -> list[AudioSample]:
+        """Filter samples by language. O(n)."""
+        return [s for s in self._catalog.values() if s.lang == lang]
+
+    def register(self, sample: AudioSample) -> None:
+        """Register custom project-specific samples."""
+        self._catalog[f"{sample.lang}_{sample.name}"] = sample
+
+    def __len__(self) -> int:
+        return len(self._catalog)
+
+    def __contains__(self, name: str) -> bool:
+        return name in self._catalog
+
+    def __repr__(self) -> str:
+        return f"SampleRegistry({len(self._catalog)} samples)"

pytest_audioeval/stt.py

@@ -0,0 +1,135 @@
+"""STT evaluation client — httpx + httpx-ws + httpx-sse under the hood."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from typing import Any, Self
+
+import httpx
+from httpx_sse import EventSource
+from httpx_ws import AsyncWebSocketClient, AsyncWebSocketSession
+
+from pytest_audioeval.metrics.text import TextMetrics
+from pytest_audioeval.samples.registry import AudioSample
+
+
+@dataclass(slots=True)
+class STTResult:
+    """STT evaluation result with optional metrics."""
+
+    hypothesis_text: str = ""
+    text_metrics: TextMetrics | None = None
+    latency_ms: float = 0.0
+    chunks_received: int = 0
+    fragments: list[str] = field(default_factory=list)
+
+    def assert_quality(self, *, max_wer: float = 0.2, max_cer: float = 0.15) -> Self:
+        """Assert STT quality. Chainable."""
+        if self.text_metrics is None:
+            raise AssertionError("No text metrics — call compute_metrics() first or provide a sample")
+        self.text_metrics.assert_quality(max_wer=max_wer, max_cer=max_cer)
+        return self
+
+    def compute_metrics(self, reference: str) -> Self:
+        """Compute WER/CER against reference. Chainable."""
+        self.text_metrics = TextMetrics.compute(reference, self.hypothesis_text)
+        return self
+
+
+class STTSession:
+    """Active WebSocket session for STT evaluation."""
+
+    __slots__ = ("_result", "_sample", "_session", "_t0")
+
+    def __init__(self, *, session: AsyncWebSocketSession, sample: AudioSample | None) -> None:
+        self._session = session
+        self._sample = sample
+        self._t0 = time.perf_counter()
+        self._result = STTResult()
+
+    async def send_bytes(self, data: bytes) -> None:
+        """Send binary audio data."""
+        await self._session.send_bytes(data)
+
+    async def send_text(self, data: str) -> None:
+        """Send text (JSON config, END_OF_AUDIO, etc.)."""
+        await self._session.send_text(data)
+
+    async def send_sample(self, sample: AudioSample, *, chunk_ms: int = 200) -> None:
+        """Stream sample in chunks with realistic pacing."""
+        for chunk in sample.chunks(chunk_ms):
+            await self._session.send_bytes(chunk)
+            await asyncio.sleep(chunk_ms / 1000)
+
+    async def receive_text(self, *, timeout: float | None = None) -> str:
+        """Receive text frame and accumulate as fragment."""
+        text = await self._session.receive_text(timeout=timeout)
+        self._result.fragments.append(text)
+        self._result.chunks_received += 1
+        return text
+
+    async def receive_bytes(self, *, timeout: float | None = None) -> bytes:
+        """Receive binary frame."""
+        return await self._session.receive_bytes(timeout=timeout)
+
+    def result(self) -> STTResult:
+        """Build STTResult from accumulated fragments."""
+        self._result.latency_ms = (time.perf_counter() - self._t0) * 1000
+        self._result.hypothesis_text = " ".join(self._result.fragments)
+        if self._sample and self._result.hypothesis_text:
+            self._result.compute_metrics(self._sample.reference_text)
+        return self._result
+
+
+class STTClient:
+    """STT evaluation client — HTTP batch + WebSocket streaming."""
+
+    __slots__ = ("_timeout", "_url")
+
+    def __init__(self, *, url: str, timeout: float = 30.0) -> None:
+        self._url = url
+        self._timeout = timeout
+
+    async def post(self, *, data: bytes | None = None, **kwargs: Any) -> httpx.Response:
+        """Batch POST audio to STT endpoint (e.g. OpenAI Whisper API). Returns raw httpx.Response."""
+        async with httpx.AsyncClient(timeout=self._timeout) as client:
+            response = await client.post(self._url, content=data, **kwargs)
+        response.raise_for_status()
+        return response
+
+    @asynccontextmanager
+    async def stream(self, *, data: bytes | None = None, **kwargs: Any) -> AsyncIterator[httpx.Response]:
+        """Chunked streaming POST. Yields httpx.Response for aiter_bytes/aiter_lines."""
+        async with (
+            httpx.AsyncClient(timeout=self._timeout) as client,
+            client.stream("POST", self._url, content=data, **kwargs) as response,
+        ):
+            response.raise_for_status()
+            yield response
+
+    @asynccontextmanager
+    async def sse(self, *, data: bytes | None = None, **kwargs: Any) -> AsyncIterator[EventSource]:
+        """SSE streaming POST. Yields EventSource for aiter_sse()."""
+        headers = kwargs.pop("headers", {})
+        headers["Accept"] = "text/event-stream"
+        async with (
+            httpx.AsyncClient(timeout=self._timeout) as client,
+            client.stream("POST", self._url, content=data, headers=headers, **kwargs) as response,
+        ):
+            response.raise_for_status()
+            yield EventSource(response)
+
+    @asynccontextmanager
+    async def ws(self, *, sample: AudioSample | None = None, **kwargs: Any) -> AsyncIterator[STTSession]:
+        """Open WebSocket session for STT streaming (e.g. WhisperLive)."""
+        async with httpx.AsyncClient() as client:
+            ws_client = AsyncWebSocketClient(client, keepalive_ping_interval_seconds=None)
+            async with ws_client.connect(self._url, **kwargs) as session:
+                yield STTSession(session=session, sample=sample)
+
+    async def aclose(self) -> None:
+        """No-op — clients are created per-call."""

pytest_audioeval/tts.py

@@ -0,0 +1,61 @@
+"""TTS evaluation client — httpx + httpx-sse + httpx-ws under the hood."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any
+
+import httpx
+from httpx_sse import EventSource
+from httpx_ws import AsyncWebSocketClient, AsyncWebSocketSession
+
+
+class TTSClient:
+    """TTS evaluation client — HTTP batch, streaming, and SSE."""
+
+    __slots__ = ("_timeout", "_url")
+
+    def __init__(self, *, url: str, timeout: float = 30.0) -> None:
+        self._url = url
+        self._timeout = timeout
+
+    async def post(self, *, json: dict[str, Any] | None = None, **kwargs: Any) -> httpx.Response:
+        """Batch POST to TTS endpoint. Returns raw httpx.Response."""
+        async with httpx.AsyncClient(timeout=self._timeout) as client:
+            response = await client.post(self._url, json=json, **kwargs)
+        response.raise_for_status()
+        return response
+
+    @asynccontextmanager
+    async def stream(self, *, json: dict[str, Any] | None = None, **kwargs: Any) -> AsyncIterator[httpx.Response]:
+        """Chunked streaming POST. Yields httpx.Response for aiter_bytes/aiter_lines."""
+        async with (
+            httpx.AsyncClient(timeout=self._timeout) as client,
+            client.stream("POST", self._url, json=json, **kwargs) as response,
+        ):
+            response.raise_for_status()
+            yield response
+
+    @asynccontextmanager
+    async def sse(self, *, json: dict[str, Any] | None = None, **kwargs: Any) -> AsyncIterator[EventSource]:
+        """SSE streaming POST. Yields EventSource for aiter_sse()."""
+        headers = kwargs.pop("headers", {})
+        headers["Accept"] = "text/event-stream"
+        async with (
+            httpx.AsyncClient(timeout=self._timeout) as client,
+            client.stream("POST", self._url, json=json, headers=headers, **kwargs) as response,
+        ):
+            response.raise_for_status()
+            yield EventSource(response)
+
+    @asynccontextmanager
+    async def ws(self, **kwargs: Any) -> AsyncIterator[AsyncWebSocketSession]:
+        """Open WebSocket session for TTS streaming (e.g. WebSocket-based TTS servers)."""
+        async with httpx.AsyncClient() as client:
+            ws_client = AsyncWebSocketClient(client, keepalive_ping_interval_seconds=None)
+            async with ws_client.connect(self._url, **kwargs) as session:
+                yield session
+
+    async def aclose(self) -> None:
+        """No-op — clients are created per-call."""

pytest_audioeval-0.1.0.dist-info/METADATA

@@ -0,0 +1,282 @@
+Metadata-Version: 2.4
+Name: pytest-audioeval
+Version: 0.1.0
+Summary: Pytest plugin for STT/TTS integration testing with httpx, metrics, and embedded audio samples.
+Project-URL: Homepage, https://damvolkov.github.io/pytest-audioeval
+Project-URL: Documentation, https://damvolkov.github.io/pytest-audioeval
+Project-URL: Repository, https://github.com/damvolkov/pytest-audioeval
+Project-URL: Changelog, https://damvolkov.github.io/pytest-audioeval/changelog/
+Author-email: damvolkov <damvolkovv@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: audio,evaluation,metrics,pesq,pytest,speech,stt,testing,tts,wer
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.13
+Requires-Dist: httpx-sse>=0.4.3
+Requires-Dist: httpx-ws>=0.8.2
+Requires-Dist: httpx>=0.27
+Requires-Dist: jiwer>=3.0
+Requires-Dist: numpy>=2.0
+Requires-Dist: pesq>=0.0.4
+Requires-Dist: pytest>=8.0
+Requires-Dist: soundfile>=0.12
+Description-Content-Type: text/markdown
+
+# pytest-audioeval
+
+Pytest plugin for STT/TTS integration testing. Built on the httpx ecosystem (`httpx`, `httpx-ws`, `httpx-sse`) with built-in metrics, embedded ground-truth audio samples, and chainable assertions.
+
+## Features
+
+- **STT via WebSocket** — `audioeval.stt.ws()` streams audio, collects transcription
+- **TTS via HTTP** — `audioeval.tts.post()` batch, `.stream()` chunked, `.sse()` Server-Sent Events
+- **Text metrics** — WER, CER, substitutions, insertions, deletions (via `jiwer`)
+- **Audio metrics** — PESQ MOS 1–5 scale (via `pesq`)
+- **Embedded samples** — ground-truth audio + reference text pairs, multi-language ready
+- **Chainable assertions** — `result.compute_metrics(ref).assert_quality(max_wer=0.2)`
+- **CLI thresholds** — `--audioeval-wer`, `--audioeval-cer`, `--audioeval-mos`
+
+## Install
+
+```bash
+uv add pytest-audioeval
+```
+
+## Quick Start
+
+### STT — WebSocket
+
+```python
+import asyncio
+import uuid
+import orjson as json
+from pytest_audioeval.client import AudioEval
+
+
+async def test_user_stt_ws(audioeval: AudioEval) -> None:
+    sample = audioeval.samples.en_hello_world
+
+    async with audioeval.stt.ws(sample=sample) as session:
+        config = json.dumps(
+            {"uid": str(uuid.uuid4()), "language": "en", "task": "transcribe",
+             "model": "large-v3-turbo", "use_vad": True}
+        ).decode()
+        await session.send_text(config)
+
+        ready = await session.receive_text()
+        assert "SERVER_READY" in ready
+
+        await session.send_sample(sample, chunk_ms=200)
+        await asyncio.sleep(2)
+        await session.send_text("END_OF_AUDIO")
+
+        # Collect transcription segments...
+```
+
+### TTS — Batch POST
+
+```python
+import io
+import soundfile as sf
+from pytest_audioeval.client import AudioEval
+
+
+async def test_user_tts_batch(audioeval: AudioEval) -> None:
+    response = await audioeval.tts.post(
+        json={"input": "Hello world.", "model": "kokoro",
+              "voice": "af_heart", "response_format": "wav", "stream": False},
+    )
+    data, rate = sf.read(io.BytesIO(response.content), dtype="float32")
+    assert rate == 24_000
+    assert len(data) > 0
+```
+
+### TTS — Chunked Streaming
+
+```python
+async def test_user_tts_streaming(audioeval: AudioEval) -> None:
+    chunks = []
+    async with audioeval.tts.stream(json={"input": "Hello.", ...}) as response:
+        async for chunk in response.aiter_bytes():
+            chunks.append(chunk)
+    assert len(chunks) > 0
+```
+
+### TTS — Server-Sent Events
+
+```python
+async def test_user_tts_sse(audioeval: AudioEval) -> None:
+    async with audioeval.tts.sse(json={"input": "Hello.", ...}) as event_source:
+        async for sse in event_source.aiter_sse():
+            print(sse.data)
+```
+
+### Text Metrics
+
+```python
+from pytest_audioeval.metrics.text import TextMetrics
+
+
+async def test_user_metrics_text() -> None:
+    metrics = TextMetrics.compute(
+        reference="the quick brown fox jumps over the lazy dog",
+        hypothesis="the quick brown fox jumps over the lazy dock",
+    )
+    assert metrics.wer < 0.15
+    assert metrics.substitutions == 1
+```
+
+### STT Result — Chainable Assertions
+
+```python
+from pytest_audioeval.stt import STTResult
+
+
+async def test_user_stt_result() -> None:
+    result = STTResult(hypothesis_text="Hello world.")
+    result.compute_metrics("Hello world.")
+    result.assert_quality(max_wer=0.2, max_cer=0.15)
+```
+
+### Sample Registry
+
+```python
+from pytest_audioeval.samples.registry import SampleLang
+
+
+async def test_user_samples_browse(audioeval: AudioEval) -> None:
+    # All samples
+    assert len(audioeval.samples) >= 3
+
+    # Filter by language
+    en_samples = audioeval.samples.by_lang(SampleLang.EN)
+
+    # Attribute access: {lang}_{name}
+    sample = audioeval.samples.en_hello_world
+    assert sample.reference_text == "Hello world."
+
+    # Audio access
+    audio_f32 = sample.audio_numpy()        # numpy float32 array
+    audio_raw = sample.audio_bytes()         # raw bytes
+    chunks = sample.chunks(chunk_ms=200)     # chunked for streaming
+```
+
+### CLI Thresholds
+
+```python
+async def test_user_thresholds(audioeval_thresholds: dict[str, float]) -> None:
+    assert audioeval_thresholds["max_wer"] == 0.2
+    assert audioeval_thresholds["max_cer"] == 0.15
+    assert audioeval_thresholds["min_mos"] == 3.0
+```
+
+## CLI Options
+
+```bash
+pytest --stt-url=ws://localhost:45120 --tts-url=http://localhost:45130/v1/audio/speech
+pytest --audioeval-wer=0.15 --audioeval-cer=0.10 --audioeval-mos=3.5
+```
+
+| Option | Default | Description |
+|---|---|---|
+| `--stt-url` | `None` | STT service WebSocket URL |
+| `--tts-url` | `None` | TTS service HTTP URL |
+| `--audioeval-wer` | `0.2` | Max WER threshold |
+| `--audioeval-cer` | `0.15` | Max CER threshold |
+| `--audioeval-mos` | `3.0` | Min PESQ MOS threshold |
+
+## Fixtures
+
+| Fixture | Scope | Type | Description |
+|---|---|---|---|
+| `audioeval` | session | `AudioEval` | Main facade — `audioeval.stt`, `audioeval.tts`, `audioeval.samples` |
+| `audioeval_thresholds` | function | `dict[str, float]` | CLI-driven threshold dict |
+
+## Architecture
+
+```
+src/pytest_audioeval/
+├── plugin.py              # pytest entry point (fixtures, CLI options)
+├── client.py              # AudioEval facade
+├── stt.py                 # STTClient (httpx-ws), STTSession, STTResult
+├── tts.py                 # TTSClient (httpx + httpx-sse)
+├── metrics/
+│   ├── text.py            # TextMetrics — WER, CER via jiwer
+│   └── audio.py           # AudioMetrics — PESQ MOS via pesq
+└── samples/
+    ├── registry.py        # SampleRegistry + AudioSample + SampleLang
+    └── audio/en/          # Embedded ground-truth WAV + TXT pairs
+```
+
+### Clients
+
+| Client | Transport | Methods |
+|---|---|---|
+| `STTClient` | `httpx-ws` | `.ws()` — WebSocket context manager yielding `STTSession` |
+| `TTSClient` | `httpx` + `httpx-sse` | `.post()` batch, `.stream()` chunked, `.sse()` SSE |
+
+### Metrics
+
+| Metric | Class | Source | Range |
+|---|---|---|---|
+| Word Error Rate (WER) | `TextMetrics` | `jiwer` | 0.0 – 1.0+ |
+| Character Error Rate (CER) | `TextMetrics` | `jiwer` | 0.0 – 1.0+ |
+| Substitutions / Insertions / Deletions | `TextMetrics` | `jiwer` | 0 – N |
+| PESQ MOS | `AudioMetrics` | `pesq` | 1.0 – 5.0 |
+
+### Samples
+
+Embedded ground-truth audio with reference transcriptions:
+
+```
+samples/audio/
+└── en/                    # English (16kHz, float32)
+    ├── hello_world.wav    # "Hello world."
+    ├── quick_brown_fox.wav
+    └── counting.wav       # "One, two, three, four, five."
+```
+
+Access: `audioeval.samples.en_hello_world`, `audioeval.samples.en_counting`, etc.
+
+## Infrastructure
+
+Integration tests require GPU-accelerated TTS/STT services:
+
+```bash
+make infra-up       # Start TTS (Kokoro) + STT (WhisperLive)
+make infra-status   # Check health
+make infra-logs     # View logs
+make infra-down     # Stop services
+```
+
+| Service | Image | Port | Protocol |
+|---|---|---|---|
+| TTS (Kokoro) | `ghcr.io/remsky/kokoro-fastapi-gpu` | `45130` | HTTP |
+| STT (WhisperLive) | `ghcr.io/collabora/whisperlive-gpu` | `45120` | WebSocket |
+
+## Development
+
+```bash
+make install            # uv sync --dev
+make lint               # ruff check + format
+make test-unit          # unit tests (no services)
+make test-integration   # integration tests (requires services)
+make coverage           # coverage report (>90%)
+```
+
+## Requirements
+
+- Python >= 3.13
+- NVIDIA GPU + Docker with nvidia-container-toolkit (for integration tests)
+
+## License
+
+MIT

pytest_audioeval-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+pytest_audioeval/client.py,sha256=q2-Wc4t-Y973mMKAjcx4i63LoKm51ItrPxm2AbgKBas,939
+pytest_audioeval/plugin.py,sha256=GQsmvWjgJGkVDVg63WJpXIg3bzEy_VAzy3SH6UBYT5Y,1545
+pytest_audioeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pytest_audioeval/stt.py,sha256=swdlUwfk6nbxULzpp9Ln5J1FDxma3rjSxV4xeFZp_Qk,5484
+pytest_audioeval/tts.py,sha256=YWETOLXsAjFSV0DOZ5XjD5btelLUF7Ze5nmz7WYeyy0,2531
+pytest_audioeval/metrics/audio.py,sha256=4XWYXd6YRoAiYnlFDjTFdmg6vtLmhaG56fMYcC1x2cg,842
+pytest_audioeval/metrics/text.py,sha256=Koisq7M8D4k-AUA_OiCj9TG0cb6EawWL8OJbHsNMD8w,1489
+pytest_audioeval/samples/registry.py,sha256=j_L8sVONaY_syx_8KX9iqcNCNkkLA1hcKxX3EqBAJmM,3690
+pytest_audioeval/samples/audio/en/counting.txt,sha256=Drz5VehysxuVt0jTmqDmGI-3mBIXNEIZPy8VXdLrCww,49
+pytest_audioeval/samples/audio/en/counting.wav,sha256=ReH23MoTEk5-2Wg7vq6PQJ-dPNjQBN5nR9nQL8GvEoc,191056
+pytest_audioeval/samples/audio/en/hello_world.txt,sha256=qj7BbmrMgJ2LKBhmInYlar_S8bRBy1FXSTPz1L0RXRE,12
+pytest_audioeval/samples/audio/en/hello_world.wav,sha256=1podm-hax7z8VE-nIRW2dQZgGSvh83F9DfJTLjQJ_8U,73256
+pytest_audioeval/samples/audio/en/quick_brown_fox.txt,sha256=71N_JciVv6eCUmUpqbY9l6pjFWTV14nCt2VEjIY1-2w,44
+pytest_audioeval/samples/audio/en/quick_brown_fox.wav,sha256=x8yH2Da4BggWU04pxsXWpDrmLu5pmy8dFVmLhmra_tY,184952
+pytest_audioeval-0.1.0.dist-info/METADATA,sha256=qnbfw-erpOwHqz3esGgx5umMGuNJc9PYbR0ThBuHDPg,8922
+pytest_audioeval-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+pytest_audioeval-0.1.0.dist-info/entry_points.txt,sha256=yDBwGzawDne6_1gZ6zS6Mv6jUVm8IHdI084rWB2nlcY,47
+pytest_audioeval-0.1.0.dist-info/licenses/LICENSE,sha256=4K5VRrESrMWeDdFrydaJe4ZtvcVCfF3FSMdxK6893cE,1070
+pytest_audioeval-0.1.0.dist-info/RECORD,,

pytest_audioeval-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pytest_audioeval-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+audioeval = pytest_audioeval.plugin

pytest_audioeval-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Damien Volkov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.