pipecat-backchannel 0.2.0__py3-none-any.whl

pipecat_backchannel/__init__.py

@@ -0,0 +1,49 @@
+"""Backchannels for Pipecat voice agents.
+
+Injects natural continuers ("mhm", "yeah", "right") while the user is still
+talking, so the agent sounds like it's listening instead of waiting. Clips are
+recorded once from your own TTS service and played from disk, so saying "mhm"
+costs no LLM call and no network round trip.
+
+Basic usage — no keys, no sample rates, no wiring::
+
+    from pipecat_backchannel import Backchannel
+
+    backchannel = Backchannel()
+
+    pipeline = Pipeline(backchannel([
+        transport.input(),
+        stt,
+        context_aggregator.user(),
+        llm,
+        tts,
+        transport.output(),
+        context_aggregator.assistant(),
+    ]))
+
+This module holds what you need to *use* it. What you need to *change* it lives
+one import deeper:
+
+===================================  ========================================
+``pipecat_backchannel.clips``        the inventory and how a clip is chosen
+``pipecat_backchannel.synth``        producing clips without the pipeline's TTS
+``pipecat_backchannel.cache``        where clips are kept between runs
+``pipecat_backchannel.library``      what clips exist and what they sound like
+``pipecat_backchannel.placement``    how the processors find their positions
+``pipecat_backchannel.turn``         the end-of-turn model behind the gate
+``pipecat_backchannel.processor``    the gate that decides when to fire
+``pipecat_backchannel.player``       turning decisions into audio
+``pipecat_backchannel.recorder``     recording clips from the pipeline's TTS
+``pipecat_backchannel.frames``       the marker frame the two exchange
+===================================  ========================================
+"""
+
+from pipecat_backchannel.plugin import Backchannel
+from pipecat_backchannel.processor import BackchannelParams
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "Backchannel",
+    "BackchannelParams",
+]

pipecat_backchannel/cache.py

@@ -0,0 +1,114 @@
+"""On-disk clip cache.
+
+Clips are produced at most once, then read from disk forever after. Nothing here
+runs on the hot path: the cache is filled at pipeline start, and the player only
+ever plays bytes that are already in memory. That's the whole point — no LLM, no
+per-utterance network round trip, no touching the live TTS service's state
+machinery to say "mhm".
+
+The :class:`ClipCache` protocol is two methods on purpose. Storage is the least
+interesting decision in this library, and the consumer only ever needs "do you
+have this?" and "keep this".
+"""
+
+from __future__ import annotations
+
+import hashlib
+import wave
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+
+def clip_filename(text: str, sample_rate: int) -> str:
+    """Return the cache filename for a clip.
+
+    Exported so you can pre-record clips by hand: name your ``.wav`` files with
+    this, drop them in the cache directory, and they will be used without ever
+    calling a synthesizer or recording anything.
+
+    The name carries a short fingerprint of the exact text, because the readable
+    part alone cannot: ``"Hm."``, ``"Hm,"`` and ``"Hm..."`` are three different
+    prosodies — the whole point of the inventory — and all collapse to ``hm``.
+
+    Args:
+        text: The clip text, e.g. ``"Mhm."``.
+        sample_rate: Sample rate in Hz.
+
+    Returns:
+        A filename such as ``"mhm_5c9e6a_24000hz.wav"``.
+    """
+    safe = "".join(c if c.isalnum() else "_" for c in text.lower()).strip("_")
+    tag = hashlib.sha1(text.encode()).hexdigest()[:6]
+    return f"{safe}_{tag}_{sample_rate}hz.wav"
+
+
+@runtime_checkable
+class ClipCache(Protocol):
+    """Keeps clip audio between runs."""
+
+    def get(self, text: str, sample_rate: int) -> bytes | None:
+        """Return cached PCM for a clip, or ``None`` if it isn't stored.
+
+        Args:
+            text: The clip text.
+            sample_rate: Sample rate in Hz. Part of the key — clips stored at one
+                rate must never be handed back for another.
+
+        Returns:
+            Raw mono 16-bit little-endian PCM, or ``None``.
+        """
+        ...
+
+    def put(self, text: str, sample_rate: int, pcm: bytes) -> None:
+        """Store PCM for a clip.
+
+        Args:
+            text: The clip text.
+            sample_rate: Sample rate in Hz.
+            pcm: Raw mono 16-bit little-endian PCM.
+        """
+        ...
+
+
+class FileClipCache:
+    """Stores clips as ``.wav`` files in a directory.
+
+    Delete the directory to force everything to be produced again, e.g. after
+    changing voice.
+    """
+
+    def __init__(self, directory: str | Path = ".clip_cache"):
+        """Initialize the cache.
+
+        Args:
+            directory: Where the ``.wav`` files live. Created if missing.
+        """
+        self._directory = Path(directory)
+
+    @property
+    def directory(self) -> Path:
+        """The directory holding the cached ``.wav`` files."""
+        return self._directory
+
+    def path_for(self, text: str, sample_rate: int) -> Path:
+        """Return the full path this cache would use for a clip."""
+        return self._directory / clip_filename(text, sample_rate)
+
+    def get(self, text: str, sample_rate: int) -> bytes | None:
+        """Read a clip from disk. See :class:`ClipCache`."""
+        path = self.path_for(text, sample_rate)
+        if not path.exists():
+            return None
+        with wave.open(str(path), "rb") as wf:
+            pcm = wf.readframes(wf.getnframes())
+        return pcm
+
+    def put(self, text: str, sample_rate: int, pcm: bytes) -> None:
+        """Write a clip to disk. See :class:`ClipCache`."""
+        self._directory.mkdir(parents=True, exist_ok=True)
+        path = self.path_for(text, sample_rate)
+        with wave.open(str(path), "wb") as wf:
+            wf.setnchannels(1)
+            wf.setsampwidth(2)
+            wf.setframerate(sample_rate)
+            wf.writeframes(pcm)

pipecat_backchannel/clips.py

@@ -0,0 +1,288 @@
+"""Backchannel clip inventory and clip-choice strategies.
+
+Clip *choice* is a separate, much lower-stakes decision than the fire/no-fire
+gate in :mod:`pipecat_backchannel.processor`: which flavor of "mhm" fits the
+last thing the user said, and don't repeat the same one twice in a row. A wrong
+guess here just sounds slightly off; it never affects *whether* a clip plays.
+
+Clips are grouped by conversational function. The default
+:class:`HeuristicClipSelector` picks a group with two regexes over the latest
+ASR partial, then avoids recently-played clips within that group.
+"""
+
+from __future__ import annotations
+
+import random
+import re
+from collections.abc import Mapping, Sequence
+from typing import Protocol, runtime_checkable
+
+from loguru import logger
+
+#: Clips grouped by conversational function. Pass a modified copy as
+#: ``Backchannel(clip_groups=...)`` to change the inventory; the audio for
+#: whatever you list is produced at startup.
+#:
+#: Real backchannels are a small, closed set of vocalizations — *uh-huh, yeah,
+#: mm-hmm, right, okay, oh, huh, hm* covers nearly all of them. Assessments like
+#: "absolutely" or "makes sense" read as backchannels on the page but in real
+#: speech they arrive at turn boundaries, not over someone mid-sentence, and a
+#: bot that says them while you're still talking sounds like it's answering.
+#:
+#: So variety comes from **prosody, not vocabulary**: the same few sounds spelled
+#: and punctuated differently, because that is the handle a TTS gives you.
+#: ``"Mhm."`` falls, ``"Mhm,"`` stays up and hands the floor back, ``"Hm..."``
+#: draws out. One sound, three meanings.
+#:
+#: Every group needs at least two clips, or it repeats itself no matter what the
+#: selector does.
+DEFAULT_CLIP_GROUPS: dict[str, list[str]] = {
+    # Plain continuer — the safe default, fits almost any mid-thought pause.
+    "continue": [
+        "Mhm.",
+        "Mhm,",
+        "Mm-hmm.",
+        "Mm-hmm,",
+        "Mm.",
+        "Mm,",
+        "Right.",
+        "Okay.",
+    ],
+    # Agreement/confirmation — fits when they just stated something concrete.
+    "affirm": [
+        "Yeah.",
+        "Yeah,",
+        "Yep.",
+        "Yup.",
+        "Uh-huh, yeah.",
+        "Yeah, yeah.",
+        "Okay, yeah.",
+    ],
+    # Hesitation-empathy — fits when they're visibly searching for words. The
+    # trailing ellipsis is doing the work: it keeps the sound unresolved, which
+    # is what makes it read as thinking along rather than answering.
+    "thinking": [
+        "Hmm.",
+        "Hm.",
+        "Hm...",
+        "Hmm...",
+        "Mm...",
+        "Yeah...",
+        "Right...",
+    ],
+    # Realization/surprise — fits when they're introducing something notable.
+    # Kept mild on purpose: "wow" and "no way" are strong enough to read as
+    # taking the floor, which is the one thing a backchannel must never do.
+    "surprise": [
+        "Oh.",
+        "Oh yeah.",
+        "Oh, yeah?",
+        "Oh really.",
+        "Really.",
+        "Huh.",
+        "Oh wow.",
+    ],
+}
+
+
+def flatten_clips(groups: Mapping[str, Sequence[str]]) -> list[str]:
+    """Flatten grouped clips into the deduplicated list of texts to synthesize.
+
+    Args:
+        groups: Clip groups, e.g. :data:`DEFAULT_CLIP_GROUPS`.
+
+    Returns:
+        Every clip text across all groups, in order, without duplicates.
+    """
+    seen: dict[str, None] = {}
+    for group in groups.values():
+        for clip in group:
+            seen.setdefault(clip, None)
+    return list(seen)
+
+
+@runtime_checkable
+class ClipSelector(Protocol):
+    """Chooses which clip to play for a given moment."""
+
+    def select(
+        self,
+        text: str,
+        groups: Mapping[str, Sequence[str]],
+        recent: Sequence[str],
+    ) -> str:
+        """Pick a clip.
+
+        Called at the moment the clip fires, so implementations must not do
+        network or disk I/O here.
+
+        Args:
+            text: The latest ASR partial for the user's current turn ("" if none).
+            groups: The processor's clip groups.
+            recent: Recently played clips, most recent last. Avoid these where
+                you can, and avoid ``recent[-1]`` unless the group holds nothing
+                else — saying the same thing twice in a row is the only
+                repetition a listener reliably notices.
+
+        Returns:
+            A clip text present in ``groups``.
+        """
+        ...
+
+
+# Hesitation markers are discourse fillers ("um", "like", "I mean"), not content
+# words — WordNet has no synonym set for them, so this stays a hand list rather
+# than a library expansion.
+_HESITATION_RE = re.compile(
+    r"\b(um+|uh+|like|you know|i mean|i don'?t know|not sure|kind of|sort of)\b", re.I
+)
+
+_SURPRISE_SEED_WORDS = [
+    "surprising",
+    "shocking",
+    "unexpected",
+    "astonishing",
+    "remarkable",
+]
+_SURPRISE_MARKERS = {
+    "actually",
+    "turns out",
+    "apparently",
+    "wow",
+    "no way",
+    "really",
+    "honestly",
+}
+# WordNet's similar-adjective graph drifts into unrelated territory from these
+# seeds (e.g. "outrage", "scandalous", "disgraceful" — shock-adjacent, not
+# surprise). Automatic expansion still needs this manual denylist.
+_SURPRISE_DENYLIST = {
+    "storm",
+    "floor",
+    "disgraceful",
+    "appal",
+    "appall",
+    "offend",
+    "outrage",
+    "scandalise",
+    "scandalize",
+    "scandalous",
+    "shameful",
+    "traumatise",
+    "traumatize",
+    "upset",
+    "unthought",
+    "unthought-of",
+    "unprovided for",
+    "unhoped",
+    "lurid",
+    "singular",
+}
+
+
+def expand_via_wordnet(
+    seed_words: Sequence[str], denylist: set[str] = _SURPRISE_DENYLIST
+) -> set[str]:
+    """Expand content-word seeds through WordNet's synonym/similarity graph.
+
+    Only worth doing for real content words with real synonym sets. On the first
+    call this downloads the WordNet corpus (~10MB, cached under ``~/nltk_data/``);
+    if that fails (offline, sandboxed), it falls back to the seeds alone rather
+    than raising.
+
+    Args:
+        seed_words: Words to expand from.
+        denylist: Words to drop from the result — WordNet drifts, and expansion
+            is not fire-and-forget.
+
+    Returns:
+        The expanded word set, minus ``denylist``.
+    """
+    try:
+        from nltk.corpus import wordnet as wn
+
+        try:
+            wn.synsets("test")
+        except LookupError:
+            import nltk
+
+            nltk.download("wordnet", quiet=True)
+
+        words = set()
+        for seed in seed_words:
+            for synset in wn.synsets(seed):
+                for lemma in synset.lemmas():
+                    words.add(lemma.name().replace("_", " ").lower())
+                if synset.pos() == "a":
+                    for similar in synset.similar_tos():
+                        for lemma in similar.lemmas():
+                            words.add(lemma.name().replace("_", " ").lower())
+        return words - denylist
+    except Exception as e:
+        logger.warning(f"WordNet expansion unavailable ({e}), using seed words only")
+        return set(seed_words)
+
+
+class HeuristicClipSelector:
+    """Picks a clip group with two regexes over the latest ASR partial.
+
+    A rough proxy for the real problem (which filler fits this context). Good
+    enough because the stakes are low — see the module docstring.
+    """
+
+    def __init__(
+        self, *, affirm_probability: float = 0.4, expand_synonyms: bool = True
+    ):
+        """Initialize the selector.
+
+        Any synonym expansion happens here, at construction time — never during
+        :meth:`select`, which runs at the moment the clip fires.
+
+        Args:
+            affirm_probability: Chance of "affirm" over "continue" when neither
+                regex matches.
+            expand_synonyms: Expand the surprise vocabulary via WordNet. Costs a
+                one-time ~10MB corpus download on first use; set ``False`` to
+                use the seed words alone.
+        """
+        self._affirm_probability = affirm_probability
+        surprise_words = set(_SURPRISE_SEED_WORDS)
+        if expand_synonyms:
+            surprise_words = expand_via_wordnet(_SURPRISE_SEED_WORDS)
+        surprise_words |= _SURPRISE_MARKERS
+        self._surprise_re = re.compile(
+            r"\b(" + "|".join(re.escape(w) for w in sorted(surprise_words)) + r")\b",
+            re.I,
+        )
+
+    def pick_group(self, text: str) -> str:
+        """Choose a clip group name for the given ASR partial."""
+        if self._surprise_re.search(text):
+            return "surprise"
+        if _HESITATION_RE.search(text):
+            return "thinking"
+        return "affirm" if random.random() < self._affirm_probability else "continue"
+
+    def select(
+        self,
+        text: str,
+        groups: Mapping[str, Sequence[str]],
+        recent: Sequence[str],
+    ) -> str:
+        """Pick a clip, avoiding recent ones within the chosen group."""
+        group_name = self.pick_group(text)
+        # Custom inventories may not define every default group name.
+        if group_name not in groups:
+            group_name = next(iter(groups))
+        group = groups[group_name]
+
+        candidates = [c for c in group if c not in recent]
+        if not candidates:
+            # Every clip in this group has been used recently. Widen back to the
+            # whole group, but never onto the one that just played: a repeat two
+            # clips apart passes unnoticed, back to back never does.
+            last = recent[-1] if recent else None
+            candidates = [c for c in group if c != last] or list(group)
+        clip = random.choice(candidates)
+        logger.debug(f"Backchannel: picked {clip!r} (group={group_name!r})")
+        return clip

pipecat_backchannel/frames.py

@@ -0,0 +1,17 @@
+"""Frames emitted by the backchannel processor."""
+
+from dataclasses import dataclass
+
+from pipecat.frames.frames import DataFrame
+
+
+@dataclass
+class PlayCachedClipFrame(DataFrame):
+    """Tells the downstream :class:`~pipecat_backchannel.player.ClipPlayer` to
+    play a pre-cached backchannel clip.
+
+    Parameters:
+        text: The clip text, used as the key into the loaded clip dict.
+    """
+
+    text: str

pipecat_backchannel/library.py

@@ -0,0 +1,165 @@
+"""The clip library: what clips exist, and what they sound like.
+
+One object owns both halves, because they are one fact. Splitting them is how a
+custom inventory silently drifts out of sync with the audio on disk — the
+processor picks a clip that was never produced, and the bot goes quiet with no
+error anywhere.
+
+Nothing here needs a sample rate from the caller. The library learns it from the
+pipeline's ``StartFrame`` and keys the cache on it, so clips can never be played
+back at a rate they weren't produced at.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Mapping, Sequence
+
+from loguru import logger
+
+from pipecat_backchannel.cache import ClipCache, FileClipCache
+from pipecat_backchannel.clips import DEFAULT_CLIP_GROUPS, flatten_clips
+from pipecat_backchannel.synth import ClipSynthesizer
+
+
+def _validate(groups: Mapping[str, Sequence[str]]) -> None:
+    if not groups:
+        raise ValueError("clip_groups is empty — there would be nothing to say.")
+    for name, clips in groups.items():
+        if not clips:
+            raise ValueError(f"clip group {name!r} is empty — remove it or give it clips.")
+        for clip in clips:
+            if not clip or not clip.strip():
+                raise ValueError(f"clip group {name!r} contains an empty clip text.")
+
+
+class ClipLibrary:
+    """The clips a backchannel can play, and the audio behind them.
+
+    Filled at pipeline start, from whichever source is available: the cache
+    first, then a :class:`~pipecat_backchannel.synth.ClipSynthesizer` if one was
+    given. With neither, clips stay missing until something calls :meth:`store`
+    — which is what :class:`~pipecat_backchannel.recorder.ClipRecorder` does,
+    using the pipeline's own TTS service.
+    """
+
+    def __init__(
+        self,
+        *,
+        groups: Mapping[str, Sequence[str]] | None = None,
+        synthesizer: ClipSynthesizer | None = None,
+        cache: ClipCache | None = None,
+    ):
+        """Initialize the library.
+
+        Args:
+            groups: Clip inventory grouped by conversational function. Defaults
+                to :data:`~pipecat_backchannel.clips.DEFAULT_CLIP_GROUPS`.
+            synthesizer: Produces clips the cache doesn't have. ``None`` means
+                the clips are recorded from the pipeline's TTS instead.
+            cache: Where clips are kept between runs. Defaults to
+                :class:`~pipecat_backchannel.cache.FileClipCache`.
+        """
+        groups = DEFAULT_CLIP_GROUPS if groups is None else groups
+        _validate(groups)
+        self._groups = groups
+        self._texts = flatten_clips(groups)
+        self._synthesizer = synthesizer
+        self._cache = cache or FileClipCache()
+
+        self._clips: dict[str, bytes] = {}
+        self._sample_rate: int | None = None
+        self._lock = asyncio.Lock()
+
+    @property
+    def groups(self) -> Mapping[str, Sequence[str]]:
+        """The clip inventory, grouped by conversational function."""
+        return self._groups
+
+    @property
+    def texts(self) -> list[str]:
+        """Every clip text, deduplicated."""
+        return list(self._texts)
+
+    @property
+    def synthesizer(self) -> ClipSynthesizer | None:
+        """What produces clips the cache doesn't have, if anything."""
+        return self._synthesizer
+
+    @property
+    def sample_rate(self) -> int | None:
+        """The rate the loaded clips are at, or ``None`` before the first load."""
+        return self._sample_rate
+
+    @property
+    def ready(self) -> bool:
+        """Whether every clip has audio. The gate stays shut until this is true."""
+        return bool(self._sample_rate) and len(self._clips) == len(self._texts)
+
+    def missing(self) -> list[str]:
+        """Clip texts that still have no audio."""
+        return [t for t in self._texts if t not in self._clips]
+
+    def get(self, text: str) -> bytes | None:
+        """Return a clip's PCM, or ``None`` if it isn't loaded."""
+        return self._clips.get(text)
+
+    def store(self, text: str, pcm: bytes) -> None:
+        """Add a clip's audio and write it to the cache.
+
+        Args:
+            text: The clip text. Must be part of this library's inventory.
+            pcm: Raw mono 16-bit little-endian PCM at :attr:`sample_rate`.
+
+        Raises:
+            RuntimeError: Called before the library knows its sample rate.
+            KeyError: ``text`` is not in the inventory.
+        """
+        if self._sample_rate is None:
+            raise RuntimeError("ClipLibrary.store() called before load().")
+        if text not in self._texts:
+            raise KeyError(f"{text!r} is not in this library's clip inventory.")
+        self._clips[text] = pcm
+        self._cache.put(text, self._sample_rate, pcm)
+
+    async def load(self, sample_rate: int) -> None:
+        """Fill the library at ``sample_rate``, using the cache then a synthesizer.
+
+        Idempotent: calling it again at the same rate does nothing. Calling it at
+        a *different* rate discards what's loaded and reloads, because a clip
+        produced at one rate must never be played at another.
+
+        Missing clips are left missing rather than raising — the library reports
+        that through :attr:`ready`, and something else may fill them in later.
+        """
+        async with self._lock:
+            if sample_rate == self._sample_rate:
+                if not self.missing():
+                    return
+            else:
+                if self._sample_rate is not None:
+                    logger.debug(
+                        f"Backchannel clips: sample rate changed "
+                        f"{self._sample_rate} -> {sample_rate}, reloading"
+                    )
+                self._clips.clear()
+                self._sample_rate = sample_rate
+
+            from_cache = 0
+            synthesized = 0
+            for text in self.missing():
+                pcm = self._cache.get(text, sample_rate)
+                if pcm is not None:
+                    from_cache += 1
+                elif self._synthesizer is not None:
+                    pcm = await self._synthesizer(text, sample_rate)
+                    self._cache.put(text, sample_rate, pcm)
+                    synthesized += 1
+                if pcm is not None:
+                    self._clips[text] = pcm
+            if synthesized:
+                logger.info(
+                    f"Backchannel: synthesized {synthesized} clip(s) (one-time, cached after)"
+                )
+            if from_cache:
+                logger.debug(f"Backchannel: loaded {from_cache} clip(s) from cache")