pipecat-behavioralsignals 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,12 @@
1
+ #
2
+ # Copyright (c) 2026, Behavioral Signal Technologies, Inc.
3
+ #
4
+ # SPDX-License-Identifier: BSD 2-Clause License
5
+ #
6
+
7
+ """Behavioral Signals voice readings for Pipecat agents, as RCS perception."""
8
+
9
+ from pipecat_behavioralsignals.perception import BehavioralSignalsPerceptionService
10
+
11
+
12
+ __all__ = ["BehavioralSignalsPerceptionService"]
@@ -0,0 +1,186 @@
1
+ #
2
+ # Copyright (c) 2026, Behavioral Signal Technologies, Inc.
3
+ #
4
+ # SPDX-License-Identifier: BSD 2-Clause License
5
+ #
6
+
7
+ """The service that reads how the user spoke with Behavioral Signals."""
8
+
9
+ import asyncio
10
+
11
+ from behavioralsignals import Behavioral
12
+ from pipecat.frames.frames import (
13
+ Frame,
14
+ EndFrame,
15
+ CancelFrame,
16
+ InputAudioRawFrame,
17
+ TranscriptionFrame,
18
+ EagerTranscriptionFrame,
19
+ PerceptionMetadataFrame,
20
+ )
21
+ from pipecat.services.settings import ServiceSettings
22
+ from pipecat.services.ai_service import AIService
23
+ from pipecat.expressive.perception import PerceptionSignal
24
+ from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup
25
+
26
+ from pipecat_behavioralsignals.stream import BehavioralSignalsStream
27
+ from pipecat_behavioralsignals.signals import CAPABILITIES, Reading
28
+
29
+
30
+ class BehavioralSignalsPerceptionService(AIService):
31
+ """Reads how the user spoke from their voice with Behavioral Signals.
32
+
33
+ Goes between the STT service and the user aggregator. It streams the user's
34
+ audio, which the STT service passes on, to the Behavioral Signals streaming
35
+ API. On each final transcription it sets ``perception`` to the latest
36
+ reading of each signal that overlaps the transcription's audio: the audio
37
+ since the previous final transcription, or that of the previous one when
38
+ no audio came in between. It also sets it on an eager transcription,
39
+ without ending the span. Signals with no such reading are left out, and
40
+ with none at all ``perception`` stays None. Transcriptions with no text
41
+ pass through unread.
42
+
43
+ It reads mono audio only. On audio with more channels it reports an error
44
+ and stops reading. When the stream keeps failing, it reports an error
45
+ once and keeps retrying.
46
+
47
+ At start it broadcasts a ``PerceptionMetadataFrame`` that declares its
48
+ signals. It logs in to Behavioral Signals while the pipeline is set up; if
49
+ that fails, it reports an error that leaves it unusable, and frames pass
50
+ through unread.
51
+
52
+ Part of expressive conversation (alpha): turn it on with
53
+ ``PipelineParams(enable_expressive_conversation=True)`` for the LLM to read
54
+ the turns. Use it instead of ``PipecatToneReader``, not with it: a pipeline
55
+ has one perception producer.
56
+
57
+ Example::
58
+
59
+ pipeline = Pipeline([transport.input(), stt, BehavioralSignalsPerceptionService(),
60
+ user_aggregator, llm, ...])
61
+ """
62
+
63
+ def __init__(self, *, cid: str | None = None, api_key: str | None = None, **kwargs):
64
+ """Initialize the service.
65
+
66
+ Args:
67
+ cid: Your Behavioral Signals client ID. Defaults to the
68
+ ``BEHAVIORALSIGNALS_CID`` environment variable.
69
+ api_key: Your Behavioral Signals API key. Defaults to the
70
+ ``BEHAVIORALSIGNALS_API_KEY`` environment variable.
71
+ **kwargs: Additional arguments passed to the parent.
72
+ """
73
+ super().__init__(settings=ServiceSettings(model=None), **kwargs)
74
+ self._cid = cid
75
+ self._api_key = api_key
76
+ self._client: Behavioral | None = None
77
+ self._stream: BehavioralSignalsStream | None = None
78
+ self._seconds = 0.0 # the user's audio so far
79
+ self._span_start = 0.0 # where the next transcription's audio starts
80
+ self._span = (0.0, 0.0) # the audio of the last final transcription
81
+ self._latest: dict[str, Reading] = {} # the latest reading of each signal
82
+
83
+ async def setup(self, setup: FrameProcessorSetup):
84
+ """Set up the service and log in to Behavioral Signals.
85
+
86
+ Args:
87
+ setup: Configuration object containing setup parameters.
88
+ """
89
+ await super().setup(setup)
90
+ try:
91
+ self._client = await asyncio.to_thread(Behavioral, self._cid, self._api_key)
92
+ except Exception as e: # noqa: BLE001 - any login failure makes it unusable
93
+ await self.push_error(
94
+ f"{self}: couldn't log in to Behavioral Signals: {e}",
95
+ e,
96
+ force_treat_as_permanent=True,
97
+ )
98
+ return
99
+ self._stream = BehavioralSignalsStream(self._client, self._report)
100
+
101
+ def service_metadata_frame(self) -> PerceptionMetadataFrame | None:
102
+ """The signals this service reports, or None when it couldn't log in.
103
+
104
+ Returns:
105
+ The metadata frame to broadcast at start.
106
+ """
107
+ if not self._stream:
108
+ return None
109
+ return PerceptionMetadataFrame(service_name=self.name, perception=CAPABILITIES)
110
+
111
+ async def process_frame(self, frame: Frame, direction: FrameDirection):
112
+ """Stream the user's audio, and set the perception of transcriptions.
113
+
114
+ Args:
115
+ frame: The frame to process.
116
+ direction: The direction of frame processing.
117
+ """
118
+ await super().process_frame(frame, direction)
119
+
120
+ if isinstance(frame, InputAudioRawFrame):
121
+ await self._send(frame)
122
+ elif isinstance(frame, EagerTranscriptionFrame):
123
+ frame.perception = self._perceive(self._span_start, self._seconds)
124
+ elif isinstance(frame, TranscriptionFrame) and frame.text.strip():
125
+ self._end_span()
126
+ frame.perception = self._perceive(*self._span)
127
+ await self.push_frame(frame, direction)
128
+
129
+ async def stop(self, frame: EndFrame):
130
+ """Close the stream.
131
+
132
+ Args:
133
+ frame: The end frame.
134
+ """
135
+ await super().stop(frame)
136
+ self._close()
137
+
138
+ async def cancel(self, frame: CancelFrame):
139
+ """Close the stream.
140
+
141
+ Args:
142
+ frame: The cancel frame.
143
+ """
144
+ await super().cancel(frame)
145
+ self._close()
146
+
147
+ async def cleanup(self):
148
+ """Close the stream and the client."""
149
+ await super().cleanup()
150
+ self._close()
151
+ if self._client:
152
+ self._client.close()
153
+ self._client = None
154
+
155
+ async def _send(self, frame: InputAudioRawFrame) -> None:
156
+ if self._stream and frame.num_channels != 1:
157
+ self._close() # its readings would be mistimed
158
+ await self.push_error(
159
+ f"{self}: Behavioral Signals reads mono audio only, "
160
+ f"but got {frame.num_channels} channels; stopped reading"
161
+ )
162
+ if self._stream:
163
+ self._stream.send(frame.audio, frame.sample_rate, self._seconds)
164
+ self._seconds += frame.num_frames / frame.sample_rate
165
+
166
+ def _end_span(self) -> None:
167
+ """Make the audio since the last final transcription its span, if there is any."""
168
+ if self._seconds > self._span_start:
169
+ self._span = (self._span_start, self._seconds)
170
+ self._span_start = self._seconds
171
+
172
+ def _perceive(self, start: float, end: float) -> list[PerceptionSignal] | None:
173
+ if self._stream:
174
+ self._latest.update({r.signal.name: r for r in self._stream.drain()})
175
+ signals = [r.signal for r in self._latest.values() if r.overlaps(start, end)]
176
+ return signals or None
177
+
178
+ def _report(self, message: str) -> None:
179
+ """Report a stream error upstream. Called from the stream's thread."""
180
+ coroutine = self.push_error(f"{self}: {message}")
181
+ asyncio.run_coroutine_threadsafe(coroutine, self.get_event_loop())
182
+
183
+ def _close(self) -> None:
184
+ if self._stream:
185
+ self._stream.close()
186
+ self._stream = None
File without changes
@@ -0,0 +1,119 @@
1
+ #
2
+ # Copyright (c) 2026, Behavioral Signal Technologies, Inc.
3
+ #
4
+ # SPDX-License-Identifier: BSD 2-Clause License
5
+ #
6
+
7
+ """The signals sent to the LLM, and how a Behavioral Signals result becomes one."""
8
+
9
+ from dataclasses import dataclass
10
+
11
+ from behavioralsignals.models import ResultItem
12
+ from pipecat.expressive.perception import (
13
+ PerceptionTerm,
14
+ PerceptionSignal,
15
+ PerceptionCapabilities,
16
+ )
17
+
18
+
19
+ _FROM_VOICE = "Read from the voice, not the words, over the latest part of the turn."
20
+
21
+ # Each signal has the name of its Behavioral Signals task.
22
+ CAPABILITIES = PerceptionCapabilities(
23
+ signals=[
24
+ PerceptionTerm(
25
+ "emotion",
26
+ f"The emotion in the voice. {_FROM_VOICE}",
27
+ ["happy", "angry", "sad", "neutral"],
28
+ ),
29
+ PerceptionTerm(
30
+ "strength",
31
+ f"Arousal: the energy in the voice. {_FROM_VOICE}",
32
+ ["strong", "weak", "neutral"],
33
+ ),
34
+ PerceptionTerm(
35
+ "positivity",
36
+ f"Valence: how positive the tone of voice is. {_FROM_VOICE}",
37
+ ["positive", "negative", "neutral"],
38
+ ),
39
+ PerceptionTerm(
40
+ "speaking_rate",
41
+ f"How fast the user speaks, compared to speakers in general. {_FROM_VOICE}",
42
+ ["fast", "slow", "normal"],
43
+ ),
44
+ PerceptionTerm(
45
+ "hesitation",
46
+ f"Whether the user sounds hesitant. {_FROM_VOICE}",
47
+ ["yes", "no"],
48
+ ),
49
+ PerceptionTerm(
50
+ "engagement",
51
+ f"Whether the user sounds involved or detached. {_FROM_VOICE}",
52
+ ["engaged", "withdrawn", "neutral"],
53
+ ),
54
+ PerceptionTerm(
55
+ "intensity",
56
+ f"How intense the emotion is, from 0 to 1, whichever emotion it is. {_FROM_VOICE}",
57
+ ),
58
+ ]
59
+ )
60
+
61
+ _TASKS = {term.name for term in CAPABILITIES.signals}
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class Reading:
66
+ """One signal read over a span of the user's audio, in seconds from the first audio.
67
+
68
+ Parameters:
69
+ start: Where the span starts.
70
+ end: Where the span ends.
71
+ signal: The signal read over the span.
72
+ """
73
+
74
+ start: float
75
+ end: float
76
+ signal: PerceptionSignal
77
+
78
+ def overlaps(self, start: float, end: float) -> bool:
79
+ """Whether this reading's span overlaps the span from `start` to `end`.
80
+
81
+ Args:
82
+ start: Where the other span starts.
83
+ end: Where the other span ends.
84
+
85
+ Returns:
86
+ True if the spans share some audio.
87
+ """
88
+ return self.start < end and self.end > start
89
+
90
+
91
+ def reading_of(item: ResultItem, offset: float) -> Reading | None:
92
+ """The reading of a result, moved by `offset` seconds.
93
+
94
+ Args:
95
+ item: A result of the stream.
96
+ offset: Where the stream's audio starts, in seconds from the first audio.
97
+
98
+ Returns:
99
+ The reading, or None for a task that is not sent, or a result with a value missing.
100
+ """
101
+ if item.task not in _TASKS or item.st is None or item.et is None or not _complete(item):
102
+ return None
103
+ return Reading(offset + item.st, offset + item.et, _signal_of(item))
104
+
105
+
106
+ def _complete(item: ResultItem) -> bool:
107
+ """Whether the result has every value its signal needs."""
108
+ if not item.prediction:
109
+ return False
110
+ if item.task == "intensity":
111
+ return item.prediction[0].score is not None
112
+ return all(p.label is not None and p.posterior is not None for p in item.prediction)
113
+
114
+
115
+ def _signal_of(item: ResultItem) -> PerceptionSignal:
116
+ if item.task == "intensity":
117
+ return PerceptionSignal("intensity", value=round(float(item.prediction[0].score), 2))
118
+ labels = [(p.label, float(p.posterior)) for p in item.prediction]
119
+ return PerceptionSignal(item.task, labels=sorted(labels, key=lambda label: -label[1]))
@@ -0,0 +1,161 @@
1
+ #
2
+ # Copyright (c) 2026, Behavioral Signal Technologies, Inc.
3
+ #
4
+ # SPDX-License-Identifier: BSD 2-Clause License
5
+ #
6
+
7
+ """One Behavioral Signals stream per call, in a thread."""
8
+
9
+ import queue
10
+ import threading
11
+ from collections.abc import Callable, Iterator
12
+
13
+ from loguru import logger
14
+ from behavioralsignals import Behavioral
15
+ from behavioralsignals.models import StreamingOptions
16
+
17
+ from pipecat_behavioralsignals.signals import Reading, reading_of
18
+
19
+
20
+ FIRST_RETRY_S = 0.5
21
+ MAX_RETRY_S = 8.0
22
+ # After this many streams in a row with no results, it reports an error. It keeps retrying, since
23
+ # the cause may pass, like a network outage or a credit top-up.
24
+ FAILURES_TO_REPORT = 3
25
+ # The most audio chunks waiting to be sent: about 20 s of 20 ms chunks. When the stream stops
26
+ # reading audio and this fills, the queued audio is dropped and the stream restarts.
27
+ MAX_CHUNKS = 1000
28
+
29
+ # One chunk of audio: where it starts in seconds from the first audio, its sample rate, its bytes.
30
+ Chunk = tuple[float, int, bytes]
31
+
32
+
33
+ def _audio(first: Chunk, chunks: queue.Queue) -> Iterator[bytes]:
34
+ """The audio of one stream, until None."""
35
+ chunk = first
36
+ while chunk is not None:
37
+ yield chunk[2]
38
+ chunk = chunks.get()
39
+
40
+
41
+ def _end(chunks: queue.Queue) -> None:
42
+ """End the audio of a stream: drop the queued audio, then add None."""
43
+ try:
44
+ while True:
45
+ chunks.get_nowait()
46
+ except queue.Empty:
47
+ chunks.put_nowait(None)
48
+
49
+
50
+ class BehavioralSignalsStream:
51
+ """Audio in with `send`, readings out with `drain`.
52
+
53
+ The stream opens on the first audio. When it fails, a new one opens after a pause, from the
54
+ audio sent after it failed. When streams keep failing, it calls `on_failing` once.
55
+ """
56
+
57
+ def __init__(self, client: Behavioral, on_failing: Callable[[str], None]):
58
+ """Start the thread that runs the stream.
59
+
60
+ Args:
61
+ client: A Behavioral Signals client that has logged in.
62
+ on_failing: Called, from the stream's thread, when streams keep failing.
63
+ """
64
+ self._client = client
65
+ self._on_failing = on_failing
66
+ self._chunks: queue.Queue = queue.Queue(MAX_CHUNKS)
67
+ self._readings: queue.Queue = queue.Queue()
68
+ self._closed = threading.Event()
69
+ threading.Thread(target=self._run, daemon=True).start()
70
+
71
+ def send(self, audio: bytes, sample_rate: int, at: float) -> None:
72
+ """Send 16-bit mono audio.
73
+
74
+ Args:
75
+ audio: The audio bytes.
76
+ sample_rate: The audio's sample rate.
77
+ at: Where the audio starts, in seconds from the first audio.
78
+ """
79
+ if self._closed.is_set():
80
+ return
81
+ try:
82
+ self._chunks.put_nowait((at, sample_rate, audio))
83
+ except queue.Full:
84
+ logger.warning(
85
+ "Behavioral Signals stream is not reading audio: dropping it and restarting"
86
+ )
87
+ _end(self._chunks)
88
+
89
+ def drain(self) -> list[Reading]:
90
+ """Take the readings that arrived since the last call.
91
+
92
+ Returns:
93
+ The readings, oldest first.
94
+ """
95
+ readings = []
96
+ while not self._readings.empty():
97
+ readings.append(self._readings.get_nowait())
98
+ return readings
99
+
100
+ def close(self) -> None:
101
+ """Stop streaming: end the audio, and open no new stream."""
102
+ self._closed.set()
103
+ _end(self._chunks)
104
+
105
+ def _run(self) -> None:
106
+ retry, failures = FIRST_RETRY_S, 0
107
+ while not self._closed.is_set():
108
+ answered = self._stream_or_fail()
109
+ if self._closed.is_set():
110
+ return
111
+ retry = FIRST_RETRY_S if answered else min(retry * 2, MAX_RETRY_S)
112
+ failures = 0 if answered else failures + 1
113
+ if failures == FAILURES_TO_REPORT:
114
+ self._on_failing(
115
+ f"Behavioral Signals stream failed {failures} times in a row, still retrying"
116
+ )
117
+ logger.warning(f"Behavioral Signals stream ended, opening a new one in {retry:.1f}s")
118
+ self._closed.wait(retry)
119
+
120
+ def _stream_or_fail(self) -> bool:
121
+ """Run one stream; a failure to open it counts as a stream with no results."""
122
+ try:
123
+ return self._stream()
124
+ except Exception as error: # noqa: BLE001 - reconnects on any failure
125
+ logger.warning(f"Behavioral Signals stream failed: {error!r}")
126
+ return False
127
+
128
+ def _stream(self) -> bool:
129
+ """Run one stream from a new queue, so audio sent while it was down is dropped.
130
+
131
+ Returns:
132
+ True if the stream sent any results.
133
+ """
134
+ chunks = self._chunks = queue.Queue(MAX_CHUNKS)
135
+ if self._closed.is_set() or (first := chunks.get()) is None:
136
+ return False
137
+ options = StreamingOptions(sample_rate=first[1], encoding="LINEAR_PCM")
138
+ responses = self._client.stream_audio(_audio(first, chunks), options)
139
+ answered = False
140
+ while (results := _next_results(responses)) is not None:
141
+ answered = answered or bool(results)
142
+ self._keep(results, first[0])
143
+ _end(chunks) # ends the audio if the stream is still reading it
144
+ return answered
145
+
146
+ def _keep(self, items: list, offset: float) -> None:
147
+ for item in items:
148
+ if reading := reading_of(item, offset):
149
+ self._readings.put(reading)
150
+
151
+
152
+ def _next_results(responses: Iterator) -> list | None:
153
+ """The results of the next response, or None when the stream ends or fails."""
154
+ try:
155
+ response = next(responses)
156
+ except StopIteration:
157
+ return None
158
+ except Exception as error: # noqa: BLE001 - reconnects on any failure
159
+ logger.warning(f"Behavioral Signals stream failed: {error!r}")
160
+ return None
161
+ return response.results or []
@@ -0,0 +1,213 @@
1
+ Metadata-Version: 2.4
2
+ Name: pipecat-behavioralsignals
3
+ Version: 0.1.0
4
+ Summary: Behavioral Signals voice readings for Pipecat agents, as RCS perception
5
+ Keywords: pipecat,voice,emotion-recognition,speech,behavioral-signals
6
+ Author: Behavioral Signal Technologies
7
+ License-Expression: BSD-2-Clause
8
+ License-File: LICENSE
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Programming Language :: Python :: 3.13
15
+ Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Classifier: Typing :: Typed
18
+ Requires-Dist: behavioralsignals>=1.0.0,<2
19
+ Requires-Dist: pipecat-ai>=1.15.0,<1.16
20
+ Requires-Python: >=3.12
21
+ Project-URL: Homepage, https://behavioralsignals.com/
22
+ Project-URL: Documentation, https://behavioralsignals.readme.io/
23
+ Project-URL: Repository, https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals
24
+ Project-URL: Issues, https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/issues
25
+ Project-URL: Changelog, https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/CHANGELOG.md
26
+ Description-Content-Type: text/markdown
27
+
28
+ # pipecat-behavioralsignals
29
+
30
+ <p align="center">
31
+ <img src="https://raw.githubusercontent.com/BehavioralSignalTechnologies/behavioralsignals-python/main/assets/logo.png" alt="Behavioral Signal Technologies"/>
32
+ </p>
33
+
34
+ <div align="center">
35
+
36
+ [![Discord](https://badgen.net/discord/members/fxjRrbMH3Q/?color=8978cc&icon=discord)](https://discord.com/invite/fxjRrbMH3Q)
37
+ [![readme.io](https://badgen.net/badge/readme.io/Documentation/?color=black)](https://behavioralsignals.readme.io/)
38
+ [![CI](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/actions/workflows/ci.yml/badge.svg)](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/actions/workflows/ci.yml)
39
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
40
+ [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
41
+ [![PyPI](https://badgen.net/pypi/v/pipecat-behavioralsignals)](https://pypi.org/project/pipecat-behavioralsignals/)
42
+ [![Python](https://badgen.net/pypi/python/pipecat-behavioralsignals)](https://pypi.org/project/pipecat-behavioralsignals/)
43
+ [![Pipecat](https://badgen.net/badge/Pipecat/1.15/blue)](https://github.com/pipecat-ai/pipecat)
44
+ [![License](https://badgen.net/badge/license/BSD-2-Clause/blue)](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/LICENSE)
45
+ [![Contributor Covenant](https://badgen.net/badge/Contributor%20Covenant/2.1/4baaaa)](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/CODE_OF_CONDUCT.md)
46
+
47
+ </div>
48
+
49
+ Lets a [Pipecat](https://github.com/pipecat-ai/pipecat) voice agent hear how the user speaks, not
50
+ just what they say. It streams the user's voice to the
51
+ [Behavioral Signals](https://behavioralsignals.com/) API and adds what it reads, such as emotion,
52
+ energy and hesitation, to each user turn. The LLM reads it with the user's words.
53
+
54
+ It uses Pipecat's Real-Time Context Standard (RCS): it is a perception producer. Pipecat's own
55
+ `PipecatToneReader` reads the words; this reads the voice.
56
+
57
+ Maintained by Behavioral Signal Technologies, the company behind the Behavioral Signals API.
58
+
59
+ ## Installation
60
+
61
+ ```bash
62
+ uv add pipecat-behavioralsignals
63
+ ```
64
+
65
+ Or with pip:
66
+
67
+ ```bash
68
+ pip install pipecat-behavioralsignals
69
+ ```
70
+
71
+ Requires Python 3.12 or later and Pipecat 1.15.
72
+
73
+ ## Prerequisites
74
+
75
+ A Behavioral Signals account. Create one and an API key in the
76
+ [Behavioral Signals portal](https://portal.behavioralsignals.com/). Set your client ID (CID) and API key:
77
+
78
+ ```bash
79
+ export BEHAVIORALSIGNALS_CID=...
80
+ export BEHAVIORALSIGNALS_API_KEY=...
81
+ ```
82
+
83
+ You can also pass them to the service as `cid` and `api_key`.
84
+
85
+ The service streams all of the user's input audio for the whole call, silence included, and
86
+ streaming is billed by the minute. See the portal for prices.
87
+
88
+ ## Usage
89
+
90
+ Put `BehavioralSignalsPerceptionService` right after your STT service, and turn on expressive
91
+ conversation:
92
+
93
+ ```python
94
+ from pipecat.pipeline.pipeline import Pipeline
95
+ from pipecat.pipeline.worker import PipelineParams, PipelineWorker
96
+
97
+ from pipecat_behavioralsignals import BehavioralSignalsPerceptionService
98
+
99
+ perception = BehavioralSignalsPerceptionService()
100
+
101
+ pipeline = Pipeline([
102
+ transport.input(),
103
+ stt,
104
+ perception,
105
+ user_aggregator,
106
+ llm,
107
+ tts,
108
+ transport.output(),
109
+ assistant_aggregator,
110
+ ])
111
+
112
+ worker = PipelineWorker(pipeline, params=PipelineParams(enable_expressive_conversation=True))
113
+ ```
114
+
115
+ Things to know:
116
+
117
+ - It reads the audio that the STT service passes on. STT services do that by default; do not set
118
+ `audio_passthrough=False`.
119
+ - It expects 16-bit mono audio, which is what Pipecat transports give by default. On audio with
120
+ more channels it reports an error and stops reading.
121
+ - A pipeline has one perception producer. Use this instead of `PipecatToneReader`, not with it.
122
+ - Without `enable_expressive_conversation=True`, the LLM does not see the readings.
123
+
124
+ ## What the LLM sees
125
+
126
+ At start, the service declares its signals in a `PerceptionMetadataFrame`. Pipecat uses it to tell
127
+ the LLM how to read them. Then each user message starts with the reading, like this:
128
+
129
+ ```
130
+ ([emotion angry=0.71, neutral=0.20, sad=0.06, happy=0.03],[strength strong=0.64, neutral=0.30, weak=0.06],[positivity negative=0.72, neutral=0.20, positive=0.08],[speaking_rate fast=0.50, normal=0.40, slow=0.10],[hesitation no=0.80, yes=0.20],[engagement engaged=0.55, neutral=0.35, withdrawn=0.10],[intensity value=0.61]) I already told you, I was charged twice.
131
+ ```
132
+
133
+ The signals have the names of the Behavioral Signals tasks:
134
+
135
+ | Signal | What it reads | Labels |
136
+ |---|---|---|
137
+ | `emotion` | The emotion in the voice | `happy`, `angry`, `sad`, `neutral` |
138
+ | `strength` | Arousal: the energy in the voice | `strong`, `weak`, `neutral` |
139
+ | `positivity` | Valence: how positive the tone of voice is | `positive`, `negative`, `neutral` |
140
+ | `speaking_rate` | How fast the user speaks, compared to speakers in general | `fast`, `slow`, `normal` |
141
+ | `hesitation` | Whether the user sounds hesitant | `yes`, `no` |
142
+ | `engagement` | Whether the user sounds involved or detached | `engaged`, `withdrawn`, `neutral` |
143
+ | `intensity` | How intense the emotion is, whichever emotion it is | a value from 0 to 1 |
144
+
145
+ Each label has its probability, likeliest first.
146
+ The API's speaker signals (age and gender) are not sent.
147
+
148
+ ## How it works
149
+
150
+ Behavioral Signals reads the voice in short windows of about 2 seconds, and each reading arrives
151
+ about 1 to 2.5 seconds after its audio. On each final transcription, the service takes the latest
152
+ reading of each signal that overlaps the transcription's audio: the audio since the previous
153
+ final transcription. So a reading usually covers the start or middle of a turn rather than its
154
+ last words, and a very short turn may have no reading. Then the message has no reading at the start.
155
+
156
+ ## Your users' audio
157
+
158
+ The service sends the user's audio to the Behavioral Signals API. The
159
+ [SDK README](https://github.com/BehavioralSignalTechnologies/behavioralsignals-python) says a
160
+ stream's results are saved there and can be fetched later. Tell your users, and check that this
161
+ fits your privacy terms.
162
+
163
+ ## Known limits
164
+
165
+ - The service logs in when the pipeline starts. With wrong or missing credentials it reports one
166
+ error and turns itself off for the call. The pipeline keeps running, with no readings.
167
+ - If the stream keeps failing later, for example with no credits left or no network, the service
168
+ reports one error upstream and keeps retrying, in case the cause passes. Turns get no readings
169
+ meanwhile.
170
+ - If the network drops without closing the connection, the stream can hang for minutes before it
171
+ fails and a new one opens. Turns get no readings meanwhile.
172
+ - If the API stops reading audio, the service keeps up to about 20 seconds of it. Beyond that it
173
+ drops the audio and starts a new stream.
174
+ - When the pipeline ends, the service closes its stream. But it cannot stop a call to the API that
175
+ hangs: the SDK has no way to cancel it. Then the stream's thread lives on until the call returns.
176
+ It is a daemon thread, so it does not keep the process from exiting.
177
+
178
+ ## Running the example
179
+
180
+ [examples/bot.py](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/examples/bot.py) is a voice agent you talk to in the browser. It uses Deepgram
181
+ for speech to text and speech, and OpenAI for the LLM.
182
+
183
+ ```bash
184
+ git clone https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals.git
185
+ cd pipecat-behavioralsignals
186
+ uv sync
187
+ cp examples/env.example .env # then fill in the keys
188
+ uv run examples/bot.py
189
+ ```
190
+
191
+ Open http://localhost:7860/client and talk. The bot logs each turn's reading, as `Voice: ...`.
192
+
193
+ ## Compatibility
194
+
195
+ Tested with Pipecat v1.15.0 and behavioralsignals 1.0.0. RCS is an alpha in Pipecat, so its types
196
+ may change between releases.
197
+
198
+ ## Development
199
+
200
+ ```bash
201
+ uv sync
202
+ uv run pytest
203
+ uv run ruff check
204
+ uv run ruff format --check
205
+ ```
206
+
207
+ The tests need no credentials: the Behavioral Signals client is faked. See
208
+ [CONTRIBUTING.md](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/CONTRIBUTING.md) for more, and
209
+ [SECURITY.md](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/SECURITY.md) to report a security problem.
210
+
211
+ ## License
212
+
213
+ BSD 2-Clause. See [LICENSE](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/LICENSE).
@@ -0,0 +1,9 @@
1
+ pipecat_behavioralsignals/__init__.py,sha256=EJwP8g8QEa76A2ddp1yl8eCtD83PsILKxRxS_38tglQ,329
2
+ pipecat_behavioralsignals/perception.py,sha256=hSu69XklWXOA12k4vaQjSafR3PFLnzCC70WiVYEZA1I,7430
3
+ pipecat_behavioralsignals/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
+ pipecat_behavioralsignals/signals.py,sha256=qDogruB77vd6Va9aur_dpeM92M8wE-gv6rYhuY0Cm0w,3808
5
+ pipecat_behavioralsignals/stream.py,sha256=vMX0cHFKNoEuloXKgZsIA5Xed4z3jGGhaaNb5Ar4fJk,5837
6
+ pipecat_behavioralsignals-0.1.0.dist-info/licenses/LICENSE,sha256=f_MN8CgH9MusqU8eQzncy6DSaixDVXoZfn7HUW3C3xw,1325
7
+ pipecat_behavioralsignals-0.1.0.dist-info/WHEEL,sha256=MWx7psoF7KNST9G72pGtiBYuaXRXQXBk9pw5R6FzYko,80
8
+ pipecat_behavioralsignals-0.1.0.dist-info/METADATA,sha256=ikTRr6jdlj6QA2j4DSUjAOvpU2O-56KCkBuSI7VPB_g,9905
9
+ pipecat_behavioralsignals-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.13.7
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,24 @@
1
+ BSD 2-Clause License
2
+
3
+ Copyright (c) 2026, Behavioral Signal Technologies, Inc.
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
19
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
22
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.