pipecat-behavioralsignals 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipecat_behavioralsignals/__init__.py +12 -0
- pipecat_behavioralsignals/perception.py +186 -0
- pipecat_behavioralsignals/py.typed +0 -0
- pipecat_behavioralsignals/signals.py +119 -0
- pipecat_behavioralsignals/stream.py +161 -0
- pipecat_behavioralsignals-0.1.0.dist-info/METADATA +213 -0
- pipecat_behavioralsignals-0.1.0.dist-info/RECORD +9 -0
- pipecat_behavioralsignals-0.1.0.dist-info/WHEEL +4 -0
- pipecat_behavioralsignals-0.1.0.dist-info/licenses/LICENSE +24 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2026, Behavioral Signal Technologies, Inc.
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: BSD 2-Clause License
|
|
5
|
+
#
|
|
6
|
+
|
|
7
|
+
"""Behavioral Signals voice readings for Pipecat agents, as RCS perception."""
|
|
8
|
+
|
|
9
|
+
from pipecat_behavioralsignals.perception import BehavioralSignalsPerceptionService
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
__all__ = ["BehavioralSignalsPerceptionService"]
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2026, Behavioral Signal Technologies, Inc.
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: BSD 2-Clause License
|
|
5
|
+
#
|
|
6
|
+
|
|
7
|
+
"""The service that reads how the user spoke with Behavioral Signals."""
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
|
|
11
|
+
from behavioralsignals import Behavioral
|
|
12
|
+
from pipecat.frames.frames import (
|
|
13
|
+
Frame,
|
|
14
|
+
EndFrame,
|
|
15
|
+
CancelFrame,
|
|
16
|
+
InputAudioRawFrame,
|
|
17
|
+
TranscriptionFrame,
|
|
18
|
+
EagerTranscriptionFrame,
|
|
19
|
+
PerceptionMetadataFrame,
|
|
20
|
+
)
|
|
21
|
+
from pipecat.services.settings import ServiceSettings
|
|
22
|
+
from pipecat.services.ai_service import AIService
|
|
23
|
+
from pipecat.expressive.perception import PerceptionSignal
|
|
24
|
+
from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup
|
|
25
|
+
|
|
26
|
+
from pipecat_behavioralsignals.stream import BehavioralSignalsStream
|
|
27
|
+
from pipecat_behavioralsignals.signals import CAPABILITIES, Reading
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class BehavioralSignalsPerceptionService(AIService):
|
|
31
|
+
"""Reads how the user spoke from their voice with Behavioral Signals.
|
|
32
|
+
|
|
33
|
+
Goes between the STT service and the user aggregator. It streams the user's
|
|
34
|
+
audio, which the STT service passes on, to the Behavioral Signals streaming
|
|
35
|
+
API. On each final transcription it sets ``perception`` to the latest
|
|
36
|
+
reading of each signal that overlaps the transcription's audio: the audio
|
|
37
|
+
since the previous final transcription, or that of the previous one when
|
|
38
|
+
no audio came in between. It also sets it on an eager transcription,
|
|
39
|
+
without ending the span. Signals with no such reading are left out, and
|
|
40
|
+
with none at all ``perception`` stays None. Transcriptions with no text
|
|
41
|
+
pass through unread.
|
|
42
|
+
|
|
43
|
+
It reads mono audio only. On audio with more channels it reports an error
|
|
44
|
+
and stops reading. When the stream keeps failing, it reports an error
|
|
45
|
+
once and keeps retrying.
|
|
46
|
+
|
|
47
|
+
At start it broadcasts a ``PerceptionMetadataFrame`` that declares its
|
|
48
|
+
signals. It logs in to Behavioral Signals while the pipeline is set up; if
|
|
49
|
+
that fails, it reports an error that leaves it unusable, and frames pass
|
|
50
|
+
through unread.
|
|
51
|
+
|
|
52
|
+
Part of expressive conversation (alpha): turn it on with
|
|
53
|
+
``PipelineParams(enable_expressive_conversation=True)`` for the LLM to read
|
|
54
|
+
the turns. Use it instead of ``PipecatToneReader``, not with it: a pipeline
|
|
55
|
+
has one perception producer.
|
|
56
|
+
|
|
57
|
+
Example::
|
|
58
|
+
|
|
59
|
+
pipeline = Pipeline([transport.input(), stt, BehavioralSignalsPerceptionService(),
|
|
60
|
+
user_aggregator, llm, ...])
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(self, *, cid: str | None = None, api_key: str | None = None, **kwargs):
|
|
64
|
+
"""Initialize the service.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
cid: Your Behavioral Signals client ID. Defaults to the
|
|
68
|
+
``BEHAVIORALSIGNALS_CID`` environment variable.
|
|
69
|
+
api_key: Your Behavioral Signals API key. Defaults to the
|
|
70
|
+
``BEHAVIORALSIGNALS_API_KEY`` environment variable.
|
|
71
|
+
**kwargs: Additional arguments passed to the parent.
|
|
72
|
+
"""
|
|
73
|
+
super().__init__(settings=ServiceSettings(model=None), **kwargs)
|
|
74
|
+
self._cid = cid
|
|
75
|
+
self._api_key = api_key
|
|
76
|
+
self._client: Behavioral | None = None
|
|
77
|
+
self._stream: BehavioralSignalsStream | None = None
|
|
78
|
+
self._seconds = 0.0 # the user's audio so far
|
|
79
|
+
self._span_start = 0.0 # where the next transcription's audio starts
|
|
80
|
+
self._span = (0.0, 0.0) # the audio of the last final transcription
|
|
81
|
+
self._latest: dict[str, Reading] = {} # the latest reading of each signal
|
|
82
|
+
|
|
83
|
+
async def setup(self, setup: FrameProcessorSetup):
|
|
84
|
+
"""Set up the service and log in to Behavioral Signals.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
setup: Configuration object containing setup parameters.
|
|
88
|
+
"""
|
|
89
|
+
await super().setup(setup)
|
|
90
|
+
try:
|
|
91
|
+
self._client = await asyncio.to_thread(Behavioral, self._cid, self._api_key)
|
|
92
|
+
except Exception as e: # noqa: BLE001 - any login failure makes it unusable
|
|
93
|
+
await self.push_error(
|
|
94
|
+
f"{self}: couldn't log in to Behavioral Signals: {e}",
|
|
95
|
+
e,
|
|
96
|
+
force_treat_as_permanent=True,
|
|
97
|
+
)
|
|
98
|
+
return
|
|
99
|
+
self._stream = BehavioralSignalsStream(self._client, self._report)
|
|
100
|
+
|
|
101
|
+
def service_metadata_frame(self) -> PerceptionMetadataFrame | None:
|
|
102
|
+
"""The signals this service reports, or None when it couldn't log in.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
The metadata frame to broadcast at start.
|
|
106
|
+
"""
|
|
107
|
+
if not self._stream:
|
|
108
|
+
return None
|
|
109
|
+
return PerceptionMetadataFrame(service_name=self.name, perception=CAPABILITIES)
|
|
110
|
+
|
|
111
|
+
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
|
112
|
+
"""Stream the user's audio, and set the perception of transcriptions.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
frame: The frame to process.
|
|
116
|
+
direction: The direction of frame processing.
|
|
117
|
+
"""
|
|
118
|
+
await super().process_frame(frame, direction)
|
|
119
|
+
|
|
120
|
+
if isinstance(frame, InputAudioRawFrame):
|
|
121
|
+
await self._send(frame)
|
|
122
|
+
elif isinstance(frame, EagerTranscriptionFrame):
|
|
123
|
+
frame.perception = self._perceive(self._span_start, self._seconds)
|
|
124
|
+
elif isinstance(frame, TranscriptionFrame) and frame.text.strip():
|
|
125
|
+
self._end_span()
|
|
126
|
+
frame.perception = self._perceive(*self._span)
|
|
127
|
+
await self.push_frame(frame, direction)
|
|
128
|
+
|
|
129
|
+
async def stop(self, frame: EndFrame):
|
|
130
|
+
"""Close the stream.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
frame: The end frame.
|
|
134
|
+
"""
|
|
135
|
+
await super().stop(frame)
|
|
136
|
+
self._close()
|
|
137
|
+
|
|
138
|
+
async def cancel(self, frame: CancelFrame):
|
|
139
|
+
"""Close the stream.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
frame: The cancel frame.
|
|
143
|
+
"""
|
|
144
|
+
await super().cancel(frame)
|
|
145
|
+
self._close()
|
|
146
|
+
|
|
147
|
+
async def cleanup(self):
|
|
148
|
+
"""Close the stream and the client."""
|
|
149
|
+
await super().cleanup()
|
|
150
|
+
self._close()
|
|
151
|
+
if self._client:
|
|
152
|
+
self._client.close()
|
|
153
|
+
self._client = None
|
|
154
|
+
|
|
155
|
+
async def _send(self, frame: InputAudioRawFrame) -> None:
|
|
156
|
+
if self._stream and frame.num_channels != 1:
|
|
157
|
+
self._close() # its readings would be mistimed
|
|
158
|
+
await self.push_error(
|
|
159
|
+
f"{self}: Behavioral Signals reads mono audio only, "
|
|
160
|
+
f"but got {frame.num_channels} channels; stopped reading"
|
|
161
|
+
)
|
|
162
|
+
if self._stream:
|
|
163
|
+
self._stream.send(frame.audio, frame.sample_rate, self._seconds)
|
|
164
|
+
self._seconds += frame.num_frames / frame.sample_rate
|
|
165
|
+
|
|
166
|
+
def _end_span(self) -> None:
|
|
167
|
+
"""Make the audio since the last final transcription its span, if there is any."""
|
|
168
|
+
if self._seconds > self._span_start:
|
|
169
|
+
self._span = (self._span_start, self._seconds)
|
|
170
|
+
self._span_start = self._seconds
|
|
171
|
+
|
|
172
|
+
def _perceive(self, start: float, end: float) -> list[PerceptionSignal] | None:
|
|
173
|
+
if self._stream:
|
|
174
|
+
self._latest.update({r.signal.name: r for r in self._stream.drain()})
|
|
175
|
+
signals = [r.signal for r in self._latest.values() if r.overlaps(start, end)]
|
|
176
|
+
return signals or None
|
|
177
|
+
|
|
178
|
+
def _report(self, message: str) -> None:
|
|
179
|
+
"""Report a stream error upstream. Called from the stream's thread."""
|
|
180
|
+
coroutine = self.push_error(f"{self}: {message}")
|
|
181
|
+
asyncio.run_coroutine_threadsafe(coroutine, self.get_event_loop())
|
|
182
|
+
|
|
183
|
+
def _close(self) -> None:
|
|
184
|
+
if self._stream:
|
|
185
|
+
self._stream.close()
|
|
186
|
+
self._stream = None
|
|
File without changes
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2026, Behavioral Signal Technologies, Inc.
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: BSD 2-Clause License
|
|
5
|
+
#
|
|
6
|
+
|
|
7
|
+
"""The signals sent to the LLM, and how a Behavioral Signals result becomes one."""
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
from behavioralsignals.models import ResultItem
|
|
12
|
+
from pipecat.expressive.perception import (
|
|
13
|
+
PerceptionTerm,
|
|
14
|
+
PerceptionSignal,
|
|
15
|
+
PerceptionCapabilities,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
_FROM_VOICE = "Read from the voice, not the words, over the latest part of the turn."
|
|
20
|
+
|
|
21
|
+
# Each signal has the name of its Behavioral Signals task.
|
|
22
|
+
CAPABILITIES = PerceptionCapabilities(
|
|
23
|
+
signals=[
|
|
24
|
+
PerceptionTerm(
|
|
25
|
+
"emotion",
|
|
26
|
+
f"The emotion in the voice. {_FROM_VOICE}",
|
|
27
|
+
["happy", "angry", "sad", "neutral"],
|
|
28
|
+
),
|
|
29
|
+
PerceptionTerm(
|
|
30
|
+
"strength",
|
|
31
|
+
f"Arousal: the energy in the voice. {_FROM_VOICE}",
|
|
32
|
+
["strong", "weak", "neutral"],
|
|
33
|
+
),
|
|
34
|
+
PerceptionTerm(
|
|
35
|
+
"positivity",
|
|
36
|
+
f"Valence: how positive the tone of voice is. {_FROM_VOICE}",
|
|
37
|
+
["positive", "negative", "neutral"],
|
|
38
|
+
),
|
|
39
|
+
PerceptionTerm(
|
|
40
|
+
"speaking_rate",
|
|
41
|
+
f"How fast the user speaks, compared to speakers in general. {_FROM_VOICE}",
|
|
42
|
+
["fast", "slow", "normal"],
|
|
43
|
+
),
|
|
44
|
+
PerceptionTerm(
|
|
45
|
+
"hesitation",
|
|
46
|
+
f"Whether the user sounds hesitant. {_FROM_VOICE}",
|
|
47
|
+
["yes", "no"],
|
|
48
|
+
),
|
|
49
|
+
PerceptionTerm(
|
|
50
|
+
"engagement",
|
|
51
|
+
f"Whether the user sounds involved or detached. {_FROM_VOICE}",
|
|
52
|
+
["engaged", "withdrawn", "neutral"],
|
|
53
|
+
),
|
|
54
|
+
PerceptionTerm(
|
|
55
|
+
"intensity",
|
|
56
|
+
f"How intense the emotion is, from 0 to 1, whichever emotion it is. {_FROM_VOICE}",
|
|
57
|
+
),
|
|
58
|
+
]
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
_TASKS = {term.name for term in CAPABILITIES.signals}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Reading:
|
|
66
|
+
"""One signal read over a span of the user's audio, in seconds from the first audio.
|
|
67
|
+
|
|
68
|
+
Parameters:
|
|
69
|
+
start: Where the span starts.
|
|
70
|
+
end: Where the span ends.
|
|
71
|
+
signal: The signal read over the span.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
start: float
|
|
75
|
+
end: float
|
|
76
|
+
signal: PerceptionSignal
|
|
77
|
+
|
|
78
|
+
def overlaps(self, start: float, end: float) -> bool:
|
|
79
|
+
"""Whether this reading's span overlaps the span from `start` to `end`.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
start: Where the other span starts.
|
|
83
|
+
end: Where the other span ends.
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
True if the spans share some audio.
|
|
87
|
+
"""
|
|
88
|
+
return self.start < end and self.end > start
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def reading_of(item: ResultItem, offset: float) -> Reading | None:
|
|
92
|
+
"""The reading of a result, moved by `offset` seconds.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
item: A result of the stream.
|
|
96
|
+
offset: Where the stream's audio starts, in seconds from the first audio.
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
The reading, or None for a task that is not sent, or a result with a value missing.
|
|
100
|
+
"""
|
|
101
|
+
if item.task not in _TASKS or item.st is None or item.et is None or not _complete(item):
|
|
102
|
+
return None
|
|
103
|
+
return Reading(offset + item.st, offset + item.et, _signal_of(item))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _complete(item: ResultItem) -> bool:
|
|
107
|
+
"""Whether the result has every value its signal needs."""
|
|
108
|
+
if not item.prediction:
|
|
109
|
+
return False
|
|
110
|
+
if item.task == "intensity":
|
|
111
|
+
return item.prediction[0].score is not None
|
|
112
|
+
return all(p.label is not None and p.posterior is not None for p in item.prediction)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _signal_of(item: ResultItem) -> PerceptionSignal:
|
|
116
|
+
if item.task == "intensity":
|
|
117
|
+
return PerceptionSignal("intensity", value=round(float(item.prediction[0].score), 2))
|
|
118
|
+
labels = [(p.label, float(p.posterior)) for p in item.prediction]
|
|
119
|
+
return PerceptionSignal(item.task, labels=sorted(labels, key=lambda label: -label[1]))
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2026, Behavioral Signal Technologies, Inc.
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: BSD 2-Clause License
|
|
5
|
+
#
|
|
6
|
+
|
|
7
|
+
"""One Behavioral Signals stream per call, in a thread."""
|
|
8
|
+
|
|
9
|
+
import queue
|
|
10
|
+
import threading
|
|
11
|
+
from collections.abc import Callable, Iterator
|
|
12
|
+
|
|
13
|
+
from loguru import logger
|
|
14
|
+
from behavioralsignals import Behavioral
|
|
15
|
+
from behavioralsignals.models import StreamingOptions
|
|
16
|
+
|
|
17
|
+
from pipecat_behavioralsignals.signals import Reading, reading_of
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
FIRST_RETRY_S = 0.5
|
|
21
|
+
MAX_RETRY_S = 8.0
|
|
22
|
+
# After this many streams in a row with no results, it reports an error. It keeps retrying, since
|
|
23
|
+
# the cause may pass, like a network outage or a credit top-up.
|
|
24
|
+
FAILURES_TO_REPORT = 3
|
|
25
|
+
# The most audio chunks waiting to be sent: about 20 s of 20 ms chunks. When the stream stops
|
|
26
|
+
# reading audio and this fills, the queued audio is dropped and the stream restarts.
|
|
27
|
+
MAX_CHUNKS = 1000
|
|
28
|
+
|
|
29
|
+
# One chunk of audio: where it starts in seconds from the first audio, its sample rate, its bytes.
|
|
30
|
+
Chunk = tuple[float, int, bytes]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _audio(first: Chunk, chunks: queue.Queue) -> Iterator[bytes]:
|
|
34
|
+
"""The audio of one stream, until None."""
|
|
35
|
+
chunk = first
|
|
36
|
+
while chunk is not None:
|
|
37
|
+
yield chunk[2]
|
|
38
|
+
chunk = chunks.get()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _end(chunks: queue.Queue) -> None:
|
|
42
|
+
"""End the audio of a stream: drop the queued audio, then add None."""
|
|
43
|
+
try:
|
|
44
|
+
while True:
|
|
45
|
+
chunks.get_nowait()
|
|
46
|
+
except queue.Empty:
|
|
47
|
+
chunks.put_nowait(None)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class BehavioralSignalsStream:
|
|
51
|
+
"""Audio in with `send`, readings out with `drain`.
|
|
52
|
+
|
|
53
|
+
The stream opens on the first audio. When it fails, a new one opens after a pause, from the
|
|
54
|
+
audio sent after it failed. When streams keep failing, it calls `on_failing` once.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(self, client: Behavioral, on_failing: Callable[[str], None]):
|
|
58
|
+
"""Start the thread that runs the stream.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
client: A Behavioral Signals client that has logged in.
|
|
62
|
+
on_failing: Called, from the stream's thread, when streams keep failing.
|
|
63
|
+
"""
|
|
64
|
+
self._client = client
|
|
65
|
+
self._on_failing = on_failing
|
|
66
|
+
self._chunks: queue.Queue = queue.Queue(MAX_CHUNKS)
|
|
67
|
+
self._readings: queue.Queue = queue.Queue()
|
|
68
|
+
self._closed = threading.Event()
|
|
69
|
+
threading.Thread(target=self._run, daemon=True).start()
|
|
70
|
+
|
|
71
|
+
def send(self, audio: bytes, sample_rate: int, at: float) -> None:
|
|
72
|
+
"""Send 16-bit mono audio.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
audio: The audio bytes.
|
|
76
|
+
sample_rate: The audio's sample rate.
|
|
77
|
+
at: Where the audio starts, in seconds from the first audio.
|
|
78
|
+
"""
|
|
79
|
+
if self._closed.is_set():
|
|
80
|
+
return
|
|
81
|
+
try:
|
|
82
|
+
self._chunks.put_nowait((at, sample_rate, audio))
|
|
83
|
+
except queue.Full:
|
|
84
|
+
logger.warning(
|
|
85
|
+
"Behavioral Signals stream is not reading audio: dropping it and restarting"
|
|
86
|
+
)
|
|
87
|
+
_end(self._chunks)
|
|
88
|
+
|
|
89
|
+
def drain(self) -> list[Reading]:
|
|
90
|
+
"""Take the readings that arrived since the last call.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
The readings, oldest first.
|
|
94
|
+
"""
|
|
95
|
+
readings = []
|
|
96
|
+
while not self._readings.empty():
|
|
97
|
+
readings.append(self._readings.get_nowait())
|
|
98
|
+
return readings
|
|
99
|
+
|
|
100
|
+
def close(self) -> None:
|
|
101
|
+
"""Stop streaming: end the audio, and open no new stream."""
|
|
102
|
+
self._closed.set()
|
|
103
|
+
_end(self._chunks)
|
|
104
|
+
|
|
105
|
+
def _run(self) -> None:
|
|
106
|
+
retry, failures = FIRST_RETRY_S, 0
|
|
107
|
+
while not self._closed.is_set():
|
|
108
|
+
answered = self._stream_or_fail()
|
|
109
|
+
if self._closed.is_set():
|
|
110
|
+
return
|
|
111
|
+
retry = FIRST_RETRY_S if answered else min(retry * 2, MAX_RETRY_S)
|
|
112
|
+
failures = 0 if answered else failures + 1
|
|
113
|
+
if failures == FAILURES_TO_REPORT:
|
|
114
|
+
self._on_failing(
|
|
115
|
+
f"Behavioral Signals stream failed {failures} times in a row, still retrying"
|
|
116
|
+
)
|
|
117
|
+
logger.warning(f"Behavioral Signals stream ended, opening a new one in {retry:.1f}s")
|
|
118
|
+
self._closed.wait(retry)
|
|
119
|
+
|
|
120
|
+
def _stream_or_fail(self) -> bool:
|
|
121
|
+
"""Run one stream; a failure to open it counts as a stream with no results."""
|
|
122
|
+
try:
|
|
123
|
+
return self._stream()
|
|
124
|
+
except Exception as error: # noqa: BLE001 - reconnects on any failure
|
|
125
|
+
logger.warning(f"Behavioral Signals stream failed: {error!r}")
|
|
126
|
+
return False
|
|
127
|
+
|
|
128
|
+
def _stream(self) -> bool:
|
|
129
|
+
"""Run one stream from a new queue, so audio sent while it was down is dropped.
|
|
130
|
+
|
|
131
|
+
Returns:
|
|
132
|
+
True if the stream sent any results.
|
|
133
|
+
"""
|
|
134
|
+
chunks = self._chunks = queue.Queue(MAX_CHUNKS)
|
|
135
|
+
if self._closed.is_set() or (first := chunks.get()) is None:
|
|
136
|
+
return False
|
|
137
|
+
options = StreamingOptions(sample_rate=first[1], encoding="LINEAR_PCM")
|
|
138
|
+
responses = self._client.stream_audio(_audio(first, chunks), options)
|
|
139
|
+
answered = False
|
|
140
|
+
while (results := _next_results(responses)) is not None:
|
|
141
|
+
answered = answered or bool(results)
|
|
142
|
+
self._keep(results, first[0])
|
|
143
|
+
_end(chunks) # ends the audio if the stream is still reading it
|
|
144
|
+
return answered
|
|
145
|
+
|
|
146
|
+
def _keep(self, items: list, offset: float) -> None:
|
|
147
|
+
for item in items:
|
|
148
|
+
if reading := reading_of(item, offset):
|
|
149
|
+
self._readings.put(reading)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _next_results(responses: Iterator) -> list | None:
|
|
153
|
+
"""The results of the next response, or None when the stream ends or fails."""
|
|
154
|
+
try:
|
|
155
|
+
response = next(responses)
|
|
156
|
+
except StopIteration:
|
|
157
|
+
return None
|
|
158
|
+
except Exception as error: # noqa: BLE001 - reconnects on any failure
|
|
159
|
+
logger.warning(f"Behavioral Signals stream failed: {error!r}")
|
|
160
|
+
return None
|
|
161
|
+
return response.results or []
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pipecat-behavioralsignals
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Behavioral Signals voice readings for Pipecat agents, as RCS perception
|
|
5
|
+
Keywords: pipecat,voice,emotion-recognition,speech,behavioral-signals
|
|
6
|
+
Author: Behavioral Signal Technologies
|
|
7
|
+
License-Expression: BSD-2-Clause
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Dist: behavioralsignals>=1.0.0,<2
|
|
19
|
+
Requires-Dist: pipecat-ai>=1.15.0,<1.16
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Project-URL: Homepage, https://behavioralsignals.com/
|
|
22
|
+
Project-URL: Documentation, https://behavioralsignals.readme.io/
|
|
23
|
+
Project-URL: Repository, https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals
|
|
24
|
+
Project-URL: Issues, https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/issues
|
|
25
|
+
Project-URL: Changelog, https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/CHANGELOG.md
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# pipecat-behavioralsignals
|
|
29
|
+
|
|
30
|
+
<p align="center">
|
|
31
|
+
<img src="https://raw.githubusercontent.com/BehavioralSignalTechnologies/behavioralsignals-python/main/assets/logo.png" alt="Behavioral Signal Technologies"/>
|
|
32
|
+
</p>
|
|
33
|
+
|
|
34
|
+
<div align="center">
|
|
35
|
+
|
|
36
|
+
[](https://discord.com/invite/fxjRrbMH3Q)
|
|
37
|
+
[](https://behavioralsignals.readme.io/)
|
|
38
|
+
[](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/actions/workflows/ci.yml)
|
|
39
|
+
[](https://github.com/astral-sh/ruff)
|
|
40
|
+
[](https://github.com/astral-sh/uv)
|
|
41
|
+
[](https://pypi.org/project/pipecat-behavioralsignals/)
|
|
42
|
+
[](https://pypi.org/project/pipecat-behavioralsignals/)
|
|
43
|
+
[](https://github.com/pipecat-ai/pipecat)
|
|
44
|
+
[](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/LICENSE)
|
|
45
|
+
[](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/CODE_OF_CONDUCT.md)
|
|
46
|
+
|
|
47
|
+
</div>
|
|
48
|
+
|
|
49
|
+
Lets a [Pipecat](https://github.com/pipecat-ai/pipecat) voice agent hear how the user speaks, not
|
|
50
|
+
just what they say. It streams the user's voice to the
|
|
51
|
+
[Behavioral Signals](https://behavioralsignals.com/) API and adds what it reads, such as emotion,
|
|
52
|
+
energy and hesitation, to each user turn. The LLM reads it with the user's words.
|
|
53
|
+
|
|
54
|
+
It uses Pipecat's Real-Time Context Standard (RCS): it is a perception producer. Pipecat's own
|
|
55
|
+
`PipecatToneReader` reads the words; this reads the voice.
|
|
56
|
+
|
|
57
|
+
Maintained by Behavioral Signal Technologies, the company behind the Behavioral Signals API.
|
|
58
|
+
|
|
59
|
+
## Installation
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
uv add pipecat-behavioralsignals
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Or with pip:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install pipecat-behavioralsignals
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Requires Python 3.12 or later and Pipecat 1.15.
|
|
72
|
+
|
|
73
|
+
## Prerequisites
|
|
74
|
+
|
|
75
|
+
A Behavioral Signals account. Create one and an API key in the
|
|
76
|
+
[Behavioral Signals portal](https://portal.behavioralsignals.com/). Set your client ID (CID) and API key:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
export BEHAVIORALSIGNALS_CID=...
|
|
80
|
+
export BEHAVIORALSIGNALS_API_KEY=...
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
You can also pass them to the service as `cid` and `api_key`.
|
|
84
|
+
|
|
85
|
+
The service streams all of the user's input audio for the whole call, silence included, and
|
|
86
|
+
streaming is billed by the minute. See the portal for prices.
|
|
87
|
+
|
|
88
|
+
## Usage
|
|
89
|
+
|
|
90
|
+
Put `BehavioralSignalsPerceptionService` right after your STT service, and turn on expressive
|
|
91
|
+
conversation:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from pipecat.pipeline.pipeline import Pipeline
|
|
95
|
+
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
|
|
96
|
+
|
|
97
|
+
from pipecat_behavioralsignals import BehavioralSignalsPerceptionService
|
|
98
|
+
|
|
99
|
+
perception = BehavioralSignalsPerceptionService()
|
|
100
|
+
|
|
101
|
+
pipeline = Pipeline([
|
|
102
|
+
transport.input(),
|
|
103
|
+
stt,
|
|
104
|
+
perception,
|
|
105
|
+
user_aggregator,
|
|
106
|
+
llm,
|
|
107
|
+
tts,
|
|
108
|
+
transport.output(),
|
|
109
|
+
assistant_aggregator,
|
|
110
|
+
])
|
|
111
|
+
|
|
112
|
+
worker = PipelineWorker(pipeline, params=PipelineParams(enable_expressive_conversation=True))
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Things to know:
|
|
116
|
+
|
|
117
|
+
- It reads the audio that the STT service passes on. STT services do that by default; do not set
|
|
118
|
+
`audio_passthrough=False`.
|
|
119
|
+
- It expects 16-bit mono audio, which is what Pipecat transports give by default. On audio with
|
|
120
|
+
more channels it reports an error and stops reading.
|
|
121
|
+
- A pipeline has one perception producer. Use this instead of `PipecatToneReader`, not with it.
|
|
122
|
+
- Without `enable_expressive_conversation=True`, the LLM does not see the readings.
|
|
123
|
+
|
|
124
|
+
## What the LLM sees
|
|
125
|
+
|
|
126
|
+
At start, the service declares its signals in a `PerceptionMetadataFrame`. Pipecat uses it to tell
|
|
127
|
+
the LLM how to read them. Then each user message starts with the reading, like this:
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
([emotion angry=0.71, neutral=0.20, sad=0.06, happy=0.03],[strength strong=0.64, neutral=0.30, weak=0.06],[positivity negative=0.72, neutral=0.20, positive=0.08],[speaking_rate fast=0.50, normal=0.40, slow=0.10],[hesitation no=0.80, yes=0.20],[engagement engaged=0.55, neutral=0.35, withdrawn=0.10],[intensity value=0.61]) I already told you, I was charged twice.
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
The signals have the names of the Behavioral Signals tasks:
|
|
134
|
+
|
|
135
|
+
| Signal | What it reads | Labels |
|
|
136
|
+
|---|---|---|
|
|
137
|
+
| `emotion` | The emotion in the voice | `happy`, `angry`, `sad`, `neutral` |
|
|
138
|
+
| `strength` | Arousal: the energy in the voice | `strong`, `weak`, `neutral` |
|
|
139
|
+
| `positivity` | Valence: how positive the tone of voice is | `positive`, `negative`, `neutral` |
|
|
140
|
+
| `speaking_rate` | How fast the user speaks, compared to speakers in general | `fast`, `slow`, `normal` |
|
|
141
|
+
| `hesitation` | Whether the user sounds hesitant | `yes`, `no` |
|
|
142
|
+
| `engagement` | Whether the user sounds involved or detached | `engaged`, `withdrawn`, `neutral` |
|
|
143
|
+
| `intensity` | How intense the emotion is, whichever emotion it is | a value from 0 to 1 |
|
|
144
|
+
|
|
145
|
+
Each label has its probability, likeliest first.
|
|
146
|
+
The API's speaker signals (age and gender) are not sent.
|
|
147
|
+
|
|
148
|
+
## How it works
|
|
149
|
+
|
|
150
|
+
Behavioral Signals reads the voice in short windows of about 2 seconds, and each reading arrives
|
|
151
|
+
about 1 to 2.5 seconds after its audio. On each final transcription, the service takes the latest
|
|
152
|
+
reading of each signal that overlaps the transcription's audio: the audio since the previous
|
|
153
|
+
final transcription. So a reading usually covers the start or middle of a turn rather than its
|
|
154
|
+
last words, and a very short turn may have no reading. Then the message has no reading at the start.
|
|
155
|
+
|
|
156
|
+
## Your users' audio
|
|
157
|
+
|
|
158
|
+
The service sends the user's audio to the Behavioral Signals API. The
|
|
159
|
+
[SDK README](https://github.com/BehavioralSignalTechnologies/behavioralsignals-python) says a
|
|
160
|
+
stream's results are saved there and can be fetched later. Tell your users, and check that this
|
|
161
|
+
fits your privacy terms.
|
|
162
|
+
|
|
163
|
+
## Known limits
|
|
164
|
+
|
|
165
|
+
- The service logs in when the pipeline starts. With wrong or missing credentials it reports one
|
|
166
|
+
error and turns itself off for the call. The pipeline keeps running, with no readings.
|
|
167
|
+
- If the stream keeps failing later, for example with no credits left or no network, the service
|
|
168
|
+
reports one error upstream and keeps retrying, in case the cause passes. Turns get no readings
|
|
169
|
+
meanwhile.
|
|
170
|
+
- If the network drops without closing the connection, the stream can hang for minutes before it
|
|
171
|
+
fails and a new one opens. Turns get no readings meanwhile.
|
|
172
|
+
- If the API stops reading audio, the service keeps up to about 20 seconds of it. Beyond that it
|
|
173
|
+
drops the audio and starts a new stream.
|
|
174
|
+
- When the pipeline ends, the service closes its stream. But it cannot stop a call to the API that
|
|
175
|
+
hangs: the SDK has no way to cancel it. Then the stream's thread lives on until the call returns.
|
|
176
|
+
It is a daemon thread, so it does not keep the process from exiting.
|
|
177
|
+
|
|
178
|
+
## Running the example
|
|
179
|
+
|
|
180
|
+
[examples/bot.py](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/examples/bot.py) is a voice agent you talk to in the browser. It uses Deepgram
|
|
181
|
+
for speech to text and speech, and OpenAI for the LLM.
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
git clone https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals.git
|
|
185
|
+
cd pipecat-behavioralsignals
|
|
186
|
+
uv sync
|
|
187
|
+
cp examples/env.example .env # then fill in the keys
|
|
188
|
+
uv run examples/bot.py
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Open http://localhost:7860/client and talk. The bot logs each turn's reading, as `Voice: ...`.
|
|
192
|
+
|
|
193
|
+
## Compatibility
|
|
194
|
+
|
|
195
|
+
Tested with Pipecat v1.15.0 and behavioralsignals 1.0.0. RCS is an alpha in Pipecat, so its types
|
|
196
|
+
may change between releases.
|
|
197
|
+
|
|
198
|
+
## Development
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
uv sync
|
|
202
|
+
uv run pytest
|
|
203
|
+
uv run ruff check
|
|
204
|
+
uv run ruff format --check
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
The tests need no credentials: the Behavioral Signals client is faked. See
|
|
208
|
+
[CONTRIBUTING.md](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/CONTRIBUTING.md) for more, and
|
|
209
|
+
[SECURITY.md](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/SECURITY.md) to report a security problem.
|
|
210
|
+
|
|
211
|
+
## License
|
|
212
|
+
|
|
213
|
+
BSD 2-Clause. See [LICENSE](https://github.com/BehavioralSignalTechnologies/pipecat-behavioralsignals/blob/main/LICENSE).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
pipecat_behavioralsignals/__init__.py,sha256=EJwP8g8QEa76A2ddp1yl8eCtD83PsILKxRxS_38tglQ,329
|
|
2
|
+
pipecat_behavioralsignals/perception.py,sha256=hSu69XklWXOA12k4vaQjSafR3PFLnzCC70WiVYEZA1I,7430
|
|
3
|
+
pipecat_behavioralsignals/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
pipecat_behavioralsignals/signals.py,sha256=qDogruB77vd6Va9aur_dpeM92M8wE-gv6rYhuY0Cm0w,3808
|
|
5
|
+
pipecat_behavioralsignals/stream.py,sha256=vMX0cHFKNoEuloXKgZsIA5Xed4z3jGGhaaNb5Ar4fJk,5837
|
|
6
|
+
pipecat_behavioralsignals-0.1.0.dist-info/licenses/LICENSE,sha256=f_MN8CgH9MusqU8eQzncy6DSaixDVXoZfn7HUW3C3xw,1325
|
|
7
|
+
pipecat_behavioralsignals-0.1.0.dist-info/WHEEL,sha256=MWx7psoF7KNST9G72pGtiBYuaXRXQXBk9pw5R6FzYko,80
|
|
8
|
+
pipecat_behavioralsignals-0.1.0.dist-info/METADATA,sha256=ikTRr6jdlj6QA2j4DSUjAOvpU2O-56KCkBuSI7VPB_g,9905
|
|
9
|
+
pipecat_behavioralsignals-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
BSD 2-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Behavioral Signal Technologies, Inc.
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
16
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
17
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
18
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
19
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
20
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
21
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
22
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
23
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
24
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|