behavioralsignals 0.7.1__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {behavioralsignals-0.7.1/src/behavioralsignals.egg-info → behavioralsignals-0.7.2}/PKG-INFO +52 -6
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/README.md +51 -5
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/pyproject.toml +1 -1
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/mcp_server.py +14 -4
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/models.py +10 -6
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/utils.py +3 -2
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2/src/behavioralsignals.egg-info}/PKG-INFO +52 -6
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/tests/test_mcp_server.py +17 -5
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/tests/test_utils.py +6 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/LICENSE +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/setup.cfg +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/__init__.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/base.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/behavioral.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/client.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/configuration.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/deepfakes.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/__init__.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/api_pb2.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/api_pb2.pyi +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/api_pb2_grpc.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/py.typed +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/SOURCES.txt +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/dependency_links.txt +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/entry_points.txt +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/requires.txt +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/top_level.txt +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/tests/test_client.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/tests/test_models.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/tests/test_package.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/tests/test_streaming.py +0 -0
- {behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/tests/test_wait_for_result.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: behavioralsignals
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: Official Python SDK for the Behavioral Signals API: voice emotion/behavior analysis and audio/video deepfake detection
|
|
5
5
|
Author: Behavioral Signal Technologies
|
|
6
6
|
License: Apache License
|
|
@@ -271,7 +271,7 @@ Dynamic: license-file
|
|
|
271
271
|
|
|
272
272
|
Official Python SDK for the [Behavioral Signals API](https://behavioralsignals.readme.io/).
|
|
273
273
|
|
|
274
|
-
|
|
274
|
+
Estimate behavioral signals from speech and detect deepfake speech using batch and real-time audio APIs. Experimental video deepfake detection is also available in batch mode.
|
|
275
275
|
|
|
276
276
|
[Python SDK Documentation](https://behavioralsignals.readme.io/docs/behavioral-signals-python-sdk) ·
|
|
277
277
|
[Examples](examples/) ·
|
|
@@ -280,7 +280,7 @@ Analyze human behavior and detect deepfake speech using batch and real-time audi
|
|
|
280
280
|
|
|
281
281
|
## Features
|
|
282
282
|
|
|
283
|
-
- **Behavioral Analysis** —
|
|
283
|
+
- **Behavioral Analysis** — estimate behavioral signals from speech in batch and real-time streaming modes
|
|
284
284
|
- **Deepfake Detection** — detect synthetic or manipulated speech in batch and real-time streaming modes
|
|
285
285
|
- **Video Deepfake Detection (Experimental, Batch Only)** — analyze both the video frames and audio track of supported video files
|
|
286
286
|
- **Core Speech Attributes (Batch Only)** — automatic speech recognition (ASR), speaker diarization, and language identification
|
|
@@ -342,7 +342,7 @@ print_results(result.results)
|
|
|
342
342
|
|
|
343
343
|
`upload_audio()` returns a process with a unique process ID (`pid`). `wait_for_result()` polls until processing completes and returns the analysis result.
|
|
344
344
|
|
|
345
|
-
`print_results()` prints one line per result: the start and end time in seconds, the task, and the
|
|
345
|
+
`print_results()` prints one line per result: the start and end time in seconds, the task, and the final label with its probability.
|
|
346
346
|
Continuous tasks such as `intensity` have no label, only a score, and `asr` and `diarization` have a label without a probability.
|
|
347
347
|
For a 10-second clip of one speaker, the output starts like this:
|
|
348
348
|
|
|
@@ -368,6 +368,52 @@ process = client.behavioral.upload_audio(
|
|
|
368
368
|
)
|
|
369
369
|
```
|
|
370
370
|
|
|
371
|
+
### Behavioral outputs
|
|
372
|
+
|
|
373
|
+
The Behavioral API returns **24 signals** across 8 dimensions: 18 behavioral signals and 6 speaker signals. Each label is its own signal with its own probability.
|
|
374
|
+
All eight dimensions are scored together on every utterance longer than 1 second, so one utterance can be *sad + weak + negative + slow + hesitating + withdrawn*, with a voice estimated as female and aged 31 - 45. That gives 4 × 3 × 3 × 3 × 2 × 3 × 2 × 4 = **5,184 possible profiles** per utterance.
|
|
375
|
+
|
|
376
|
+
| Task | What it measures | Labels | # |
|
|
377
|
+
|---|---|---|---|
|
|
378
|
+
| `emotion` | Basic emotion in the voice | `happy`, `angry`, `sad`, `neutral` | 4 |
|
|
379
|
+
| `strength` | Arousal: energy in the voice | `strong`, `weak`, `neutral` | 3 |
|
|
380
|
+
| `positivity` | Valence: sentiment of the tone | `positive`, `negative`, `neutral` | 3 |
|
|
381
|
+
| `speaking_rate` | How fast the speaker talks, compared to speakers in general, not to their own pace | `fast`, `slow`, `normal` | 3 |
|
|
382
|
+
| `hesitation` | Signs of hesitation | `yes`, `no` | 2 |
|
|
383
|
+
| `engagement` | Involved or detached tone | `engaged`, `withdrawn`, `neutral` | 3 |
|
|
384
|
+
| `gender` | Sex of the speaker, estimated from the voice | `female`, `male` | 2 |
|
|
385
|
+
| `age` | Age range of the speaker, estimated from the voice | `18 - 22`, `23 - 30`, `31 - 45`, `46 - 65` | 4 |
|
|
386
|
+
| **Total** | | | **24** |
|
|
387
|
+
|
|
388
|
+
Plus `intensity`: how intense the emotion is, as one score between 0 and 1. It is not a score for any one emotion, even when `emotion` is `angry`. It has no labels and is not counted in the 24.
|
|
389
|
+
In batch results, utterances of 1 second or less return only `diarization`, `asr` and `language`. When streaming, every segment gets all 24 signals and `intensity`, however short it is.
|
|
390
|
+
In batch results, `gender` and `age` are given per speaker: every utterance of a speaker longer than 1 second gets the same values, averaged over the file. When streaming, `gender` and `age` are estimated on each segment.
|
|
391
|
+
The other tasks have open-ended values, not fixed labels: `diarization` (a speaker ID such as `SPEAKER_00`), `asr` (the transcript) and `language`. With `embeddings=True`, `features` holds the behavioral embedding.
|
|
392
|
+
All results of one utterance share its `id`, `startTime` and `endTime`. To split results by speaker, use the `diarization` label of each utterance. Speaker IDs are generic: the API does not say who is the agent and who is the customer.
|
|
393
|
+
|
|
394
|
+
`neutral` means something different in each task, so always read a label together with its task: `emotion: neutral` means no clear emotion, `strength: neutral` normal energy, `positivity: neutral` neither positive nor negative, and `engagement: neutral` neither engaged nor withdrawn.
|
|
395
|
+
|
|
396
|
+
`finalLabel` gives one answer per task. For behavioral tasks it is the label with the highest probability.
|
|
397
|
+
When you need a score instead of a label, use the probabilities in `prediction`. They are strings, so convert them with `float()`.
|
|
398
|
+
For each task in the table they add up to 1, so each label's probability is a score from 0 to 1 that you can track over time, average over a call, or compare against your own threshold.
|
|
399
|
+
For example, the `angry` probability of `emotion` is a score for anger. `intensity` is not: it does not say which emotion is intense.
|
|
400
|
+
|
|
401
|
+
`positivity` is valence and `strength` is arousal, the two dimensions often used in emotion research. The API gives them as probabilities from 0 to 1. To put them on the usual scale from -1 to 1, subtract the probabilities of the two opposite labels:
|
|
402
|
+
|
|
403
|
+
```python
|
|
404
|
+
for item in result.results:
|
|
405
|
+
if item.task in ("positivity", "strength"):
|
|
406
|
+
scores = {p.label: float(p.posterior) for p in item.prediction}
|
|
407
|
+
if item.task == "positivity":
|
|
408
|
+
print(item.st, "valence", scores["positive"] - scores["negative"])
|
|
409
|
+
else:
|
|
410
|
+
print(item.st, "arousal", scores["strong"] - scores["weak"])
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
All probabilities are model estimates: use them to compare utterances and follow trends, not as exact measurements.
|
|
414
|
+
|
|
415
|
+
See [Definition of behaviors](https://behavioralsignals.readme.io/docs/definition-of-behaviors) for what each signal means.
|
|
416
|
+
|
|
371
417
|
## Deepfake Detection
|
|
372
418
|
|
|
373
419
|
### Audio
|
|
@@ -629,8 +675,8 @@ The server exposes four tools:
|
|
|
629
675
|
| Tool | What it does |
|
|
630
676
|
|---|---|
|
|
631
677
|
| `analyze_behavior` | Uploads an audio file or S3 presigned URL for behavioral analysis and returns the results |
|
|
632
|
-
| `detect_deepfake` | Uploads an audio or video file, or an S3 presigned URL, for deepfake detection and returns the results |
|
|
633
|
-
| `get_result` | Returns the results of a process, in pages, optionally filtered to specific tasks |
|
|
678
|
+
| `detect_deepfake` | Uploads an audio or video file, or an S3 presigned URL, for deepfake detection (video is experimental) and returns the results |
|
|
679
|
+
| `get_result` | Returns the results of a process, in pages, optionally filtered to specific tasks. Each row has the final label and the probability of every label |
|
|
634
680
|
| `list_processes` | Lists processes, newest first, including the failure reason when available |
|
|
635
681
|
|
|
636
682
|
Notes:
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
|
|
26
26
|
Official Python SDK for the [Behavioral Signals API](https://behavioralsignals.readme.io/).
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
Estimate behavioral signals from speech and detect deepfake speech using batch and real-time audio APIs. Experimental video deepfake detection is also available in batch mode.
|
|
29
29
|
|
|
30
30
|
[Python SDK Documentation](https://behavioralsignals.readme.io/docs/behavioral-signals-python-sdk) ·
|
|
31
31
|
[Examples](examples/) ·
|
|
@@ -34,7 +34,7 @@ Analyze human behavior and detect deepfake speech using batch and real-time audi
|
|
|
34
34
|
|
|
35
35
|
## Features
|
|
36
36
|
|
|
37
|
-
- **Behavioral Analysis** —
|
|
37
|
+
- **Behavioral Analysis** — estimate behavioral signals from speech in batch and real-time streaming modes
|
|
38
38
|
- **Deepfake Detection** — detect synthetic or manipulated speech in batch and real-time streaming modes
|
|
39
39
|
- **Video Deepfake Detection (Experimental, Batch Only)** — analyze both the video frames and audio track of supported video files
|
|
40
40
|
- **Core Speech Attributes (Batch Only)** — automatic speech recognition (ASR), speaker diarization, and language identification
|
|
@@ -96,7 +96,7 @@ print_results(result.results)
|
|
|
96
96
|
|
|
97
97
|
`upload_audio()` returns a process with a unique process ID (`pid`). `wait_for_result()` polls until processing completes and returns the analysis result.
|
|
98
98
|
|
|
99
|
-
`print_results()` prints one line per result: the start and end time in seconds, the task, and the
|
|
99
|
+
`print_results()` prints one line per result: the start and end time in seconds, the task, and the final label with its probability.
|
|
100
100
|
Continuous tasks such as `intensity` have no label, only a score, and `asr` and `diarization` have a label without a probability.
|
|
101
101
|
For a 10-second clip of one speaker, the output starts like this:
|
|
102
102
|
|
|
@@ -122,6 +122,52 @@ process = client.behavioral.upload_audio(
|
|
|
122
122
|
)
|
|
123
123
|
```
|
|
124
124
|
|
|
125
|
+
### Behavioral outputs
|
|
126
|
+
|
|
127
|
+
The Behavioral API returns **24 signals** across 8 dimensions: 18 behavioral signals and 6 speaker signals. Each label is its own signal with its own probability.
|
|
128
|
+
All eight dimensions are scored together on every utterance longer than 1 second, so one utterance can be *sad + weak + negative + slow + hesitating + withdrawn*, with a voice estimated as female and aged 31 - 45. That gives 4 × 3 × 3 × 3 × 2 × 3 × 2 × 4 = **5,184 possible profiles** per utterance.
|
|
129
|
+
|
|
130
|
+
| Task | What it measures | Labels | # |
|
|
131
|
+
|---|---|---|---|
|
|
132
|
+
| `emotion` | Basic emotion in the voice | `happy`, `angry`, `sad`, `neutral` | 4 |
|
|
133
|
+
| `strength` | Arousal: energy in the voice | `strong`, `weak`, `neutral` | 3 |
|
|
134
|
+
| `positivity` | Valence: sentiment of the tone | `positive`, `negative`, `neutral` | 3 |
|
|
135
|
+
| `speaking_rate` | How fast the speaker talks, compared to speakers in general, not to their own pace | `fast`, `slow`, `normal` | 3 |
|
|
136
|
+
| `hesitation` | Signs of hesitation | `yes`, `no` | 2 |
|
|
137
|
+
| `engagement` | Involved or detached tone | `engaged`, `withdrawn`, `neutral` | 3 |
|
|
138
|
+
| `gender` | Sex of the speaker, estimated from the voice | `female`, `male` | 2 |
|
|
139
|
+
| `age` | Age range of the speaker, estimated from the voice | `18 - 22`, `23 - 30`, `31 - 45`, `46 - 65` | 4 |
|
|
140
|
+
| **Total** | | | **24** |
|
|
141
|
+
|
|
142
|
+
Plus `intensity`: how intense the emotion is, as one score between 0 and 1. It is not a score for any one emotion, even when `emotion` is `angry`. It has no labels and is not counted in the 24.
|
|
143
|
+
In batch results, utterances of 1 second or less return only `diarization`, `asr` and `language`. When streaming, every segment gets all 24 signals and `intensity`, however short it is.
|
|
144
|
+
In batch results, `gender` and `age` are given per speaker: every utterance of a speaker longer than 1 second gets the same values, averaged over the file. When streaming, `gender` and `age` are estimated on each segment.
|
|
145
|
+
The other tasks have open-ended values, not fixed labels: `diarization` (a speaker ID such as `SPEAKER_00`), `asr` (the transcript) and `language`. With `embeddings=True`, `features` holds the behavioral embedding.
|
|
146
|
+
All results of one utterance share its `id`, `startTime` and `endTime`. To split results by speaker, use the `diarization` label of each utterance. Speaker IDs are generic: the API does not say who is the agent and who is the customer.
|
|
147
|
+
|
|
148
|
+
`neutral` means something different in each task, so always read a label together with its task: `emotion: neutral` means no clear emotion, `strength: neutral` normal energy, `positivity: neutral` neither positive nor negative, and `engagement: neutral` neither engaged nor withdrawn.
|
|
149
|
+
|
|
150
|
+
`finalLabel` gives one answer per task. For behavioral tasks it is the label with the highest probability.
|
|
151
|
+
When you need a score instead of a label, use the probabilities in `prediction`. They are strings, so convert them with `float()`.
|
|
152
|
+
For each task in the table they add up to 1, so each label's probability is a score from 0 to 1 that you can track over time, average over a call, or compare against your own threshold.
|
|
153
|
+
For example, the `angry` probability of `emotion` is a score for anger. `intensity` is not: it does not say which emotion is intense.
|
|
154
|
+
|
|
155
|
+
`positivity` is valence and `strength` is arousal, the two dimensions often used in emotion research. The API gives them as probabilities from 0 to 1. To put them on the usual scale from -1 to 1, subtract the probabilities of the two opposite labels:
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
for item in result.results:
|
|
159
|
+
if item.task in ("positivity", "strength"):
|
|
160
|
+
scores = {p.label: float(p.posterior) for p in item.prediction}
|
|
161
|
+
if item.task == "positivity":
|
|
162
|
+
print(item.st, "valence", scores["positive"] - scores["negative"])
|
|
163
|
+
else:
|
|
164
|
+
print(item.st, "arousal", scores["strong"] - scores["weak"])
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
All probabilities are model estimates: use them to compare utterances and follow trends, not as exact measurements.
|
|
168
|
+
|
|
169
|
+
See [Definition of behaviors](https://behavioralsignals.readme.io/docs/definition-of-behaviors) for what each signal means.
|
|
170
|
+
|
|
125
171
|
## Deepfake Detection
|
|
126
172
|
|
|
127
173
|
### Audio
|
|
@@ -383,8 +429,8 @@ The server exposes four tools:
|
|
|
383
429
|
| Tool | What it does |
|
|
384
430
|
|---|---|
|
|
385
431
|
| `analyze_behavior` | Uploads an audio file or S3 presigned URL for behavioral analysis and returns the results |
|
|
386
|
-
| `detect_deepfake` | Uploads an audio or video file, or an S3 presigned URL, for deepfake detection and returns the results |
|
|
387
|
-
| `get_result` | Returns the results of a process, in pages, optionally filtered to specific tasks |
|
|
432
|
+
| `detect_deepfake` | Uploads an audio or video file, or an S3 presigned URL, for deepfake detection (video is experimental) and returns the results |
|
|
433
|
+
| `get_result` | Returns the results of a process, in pages, optionally filtered to specific tasks. Each row has the final label and the probability of every label |
|
|
388
434
|
| `list_processes` | Lists processes, newest first, including the failure reason when available |
|
|
389
435
|
|
|
390
436
|
Notes:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "behavioralsignals"
|
|
3
|
-
version = "0.7.
|
|
3
|
+
version = "0.7.2"
|
|
4
4
|
description = "Official Python SDK for the Behavioral Signals API: voice emotion/behavior analysis and audio/video deepfake detection"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "Behavioral Signal Technologies" }]
|
|
@@ -36,7 +36,7 @@ WaitSeconds = Annotated[float, Field(ge=0, le=50)]
|
|
|
36
36
|
|
|
37
37
|
DEFAULT_WAIT = 45
|
|
38
38
|
DEFAULT_LIMIT = 300
|
|
39
|
-
RESULT_COLUMNS = ["start", "end", "task", "label", "
|
|
39
|
+
RESULT_COLUMNS = ["start", "end", "task", "label", "probabilities"]
|
|
40
40
|
PROCESS_COLUMNS = ["pid", "name", "status", "duration", "created", "reason"]
|
|
41
41
|
READ_ONLY = ToolAnnotations(read_only_hint=True)
|
|
42
42
|
|
|
@@ -71,7 +71,7 @@ def detect_deepfake(
|
|
|
71
71
|
generator_detection: bool = False,
|
|
72
72
|
wait_seconds: WaitSeconds = DEFAULT_WAIT,
|
|
73
73
|
) -> str:
|
|
74
|
-
"""Detects whether speech (media="audio") or a video (media="video") is a deepfake.
|
|
74
|
+
"""Detects whether speech (media="audio") or a video (media="video", experimental) is a deepfake.
|
|
75
75
|
|
|
76
76
|
`source` is an absolute path to a local file, or an S3 presigned URL. With
|
|
77
77
|
generator_detection=True, the result also names the likely generator (experimental). Each
|
|
@@ -100,6 +100,9 @@ def get_result(
|
|
|
100
100
|
`analysis` is the kind of job: "behavioral" (analyze_behavior), "deepfake_audio" or
|
|
101
101
|
"deepfake_video" (detect_deepfake). Results come as tab-separated rows, `limit` rows from
|
|
102
102
|
`offset`. `tasks` keeps only those tasks, e.g. ["emotion"]; the header lists the task names.
|
|
103
|
+
Each row has the final label and the probability of every label of the task. Use the
|
|
104
|
+
probabilities as scores. positivity is valence and strength is arousal; for a scale from -1
|
|
105
|
+
to 1, use P(positive) - P(negative) and P(strong) - P(weak).
|
|
103
106
|
"""
|
|
104
107
|
with _tool_errors(), _api(analysis) as api:
|
|
105
108
|
return _result_text(api, analysis, pid, wait_seconds, tasks, offset, limit)
|
|
@@ -238,8 +241,15 @@ def _result_row(item: ResultItem) -> list[str] | None:
|
|
|
238
241
|
label = item.finalLabel or (top.score if top else None)
|
|
239
242
|
if not label:
|
|
240
243
|
return None
|
|
241
|
-
|
|
242
|
-
return [_cell(value) for value in
|
|
244
|
+
values = (item.startTime, item.endTime, item.task, label, _probabilities(item))
|
|
245
|
+
return [_cell(value) for value in values]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _probabilities(item: ResultItem) -> str:
|
|
249
|
+
"""Returns every label of the item with its probability, e.g. "sad=0.70, neutral=0.30"."""
|
|
250
|
+
return ", ".join(
|
|
251
|
+
f"{p.label}={float(p.posterior):.2f}" for p in item.prediction or [] if p.posterior
|
|
252
|
+
)
|
|
243
253
|
|
|
244
254
|
|
|
245
255
|
def _format_result(
|
|
@@ -215,7 +215,9 @@ class _SerializableModel(BaseModel):
|
|
|
215
215
|
class ModelPredictions(_SerializableModel):
|
|
216
216
|
label: str | None = Field(None, description="The name of the class", examples=["happy"])
|
|
217
217
|
posterior: str | None = Field(
|
|
218
|
-
None,
|
|
218
|
+
None,
|
|
219
|
+
description="The probability of this class, as a string. The posteriors of one result add up to about 1, except for language, which lists only the top 3 languages. Use float(posterior) as a score when finalLabel is not enough",
|
|
220
|
+
examples=["0.754"],
|
|
219
221
|
)
|
|
220
222
|
score: str | None = Field(
|
|
221
223
|
None,
|
|
@@ -223,7 +225,7 @@ class ModelPredictions(_SerializableModel):
|
|
|
223
225
|
examples=["0.62"],
|
|
224
226
|
)
|
|
225
227
|
dominantInSegments: list[int] | None = Field(
|
|
226
|
-
None, description="
|
|
228
|
+
None, description="Internal field, usually empty. You can ignore it."
|
|
227
229
|
)
|
|
228
230
|
|
|
229
231
|
|
|
@@ -237,22 +239,24 @@ class ResultItem(_SerializableModel):
|
|
|
237
239
|
)
|
|
238
240
|
task: str | None = Field(
|
|
239
241
|
None,
|
|
240
|
-
description="The
|
|
242
|
+
description="The task. Behavioral: emotion, strength, positivity, speaking_rate, hesitation, engagement, intensity. Speaker: gender, age. Other: diarization, asr, language, features. Deepfakes API: deepfake, visual_deepfake, generator. "
|
|
241
243
|
"Consider visiting the guides in behavioralsignals.readme.io for the latest examples.",
|
|
242
244
|
examples=["emotion"],
|
|
243
245
|
)
|
|
244
246
|
prediction: list[ModelPredictions] | None = None
|
|
245
247
|
finalLabel: str | None = Field(
|
|
246
|
-
None,
|
|
248
|
+
None,
|
|
249
|
+
description="The label the API picks for this task. For behavioral tasks it is the label with the highest posterior.",
|
|
250
|
+
examples=["happy"],
|
|
247
251
|
)
|
|
248
252
|
level: str | None = Field(
|
|
249
253
|
None,
|
|
250
|
-
description="
|
|
254
|
+
description="'utterance' in batch results. Streaming can also return 'segment'.",
|
|
251
255
|
examples=["utterance"],
|
|
252
256
|
)
|
|
253
257
|
embedding: str | None = Field(
|
|
254
258
|
None,
|
|
255
|
-
description="The
|
|
259
|
+
description="The embedding as a stringified array: the speaker embedding (192 values) on the diarization row of a speaker's first utterance, or the behavioral embedding (768 values) on the features row.",
|
|
256
260
|
examples=["[11.614513397216797, -15.228992462158203, -4.92175817489624, ...]"],
|
|
257
261
|
)
|
|
258
262
|
|
|
@@ -27,7 +27,7 @@ def make_audio_stream(file_path: str, chunk_size: float = 0.25) -> tuple[Iterato
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
def print_results(items: Iterable[ResultItem] | None) -> None:
|
|
30
|
-
"""Print one line per result: start time, end time, task,
|
|
30
|
+
"""Print one line per result: start time, end time, task, final label and its probability.
|
|
31
31
|
|
|
32
32
|
Continuous tasks (e.g. intensity) have no label, so their score is printed instead.
|
|
33
33
|
Rows without a label or score (e.g. the features row, which carries embeddings) are skipped.
|
|
@@ -40,5 +40,6 @@ def print_results(items: Iterable[ResultItem] | None) -> None:
|
|
|
40
40
|
label = item.finalLabel or (top.score if top else None)
|
|
41
41
|
if not label:
|
|
42
42
|
continue
|
|
43
|
-
|
|
43
|
+
posterior = next((p.posterior for p in item.prediction or [] if p.label == label), None)
|
|
44
|
+
confidence = f" ({float(posterior):.1%})" if posterior else ""
|
|
44
45
|
print(f"{item.st} {item.et} {item.task} {label.strip()}{confidence}")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: behavioralsignals
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: Official Python SDK for the Behavioral Signals API: voice emotion/behavior analysis and audio/video deepfake detection
|
|
5
5
|
Author: Behavioral Signal Technologies
|
|
6
6
|
License: Apache License
|
|
@@ -271,7 +271,7 @@ Dynamic: license-file
|
|
|
271
271
|
|
|
272
272
|
Official Python SDK for the [Behavioral Signals API](https://behavioralsignals.readme.io/).
|
|
273
273
|
|
|
274
|
-
|
|
274
|
+
Estimate behavioral signals from speech and detect deepfake speech using batch and real-time audio APIs. Experimental video deepfake detection is also available in batch mode.
|
|
275
275
|
|
|
276
276
|
[Python SDK Documentation](https://behavioralsignals.readme.io/docs/behavioral-signals-python-sdk) ·
|
|
277
277
|
[Examples](examples/) ·
|
|
@@ -280,7 +280,7 @@ Analyze human behavior and detect deepfake speech using batch and real-time audi
|
|
|
280
280
|
|
|
281
281
|
## Features
|
|
282
282
|
|
|
283
|
-
- **Behavioral Analysis** —
|
|
283
|
+
- **Behavioral Analysis** — estimate behavioral signals from speech in batch and real-time streaming modes
|
|
284
284
|
- **Deepfake Detection** — detect synthetic or manipulated speech in batch and real-time streaming modes
|
|
285
285
|
- **Video Deepfake Detection (Experimental, Batch Only)** — analyze both the video frames and audio track of supported video files
|
|
286
286
|
- **Core Speech Attributes (Batch Only)** — automatic speech recognition (ASR), speaker diarization, and language identification
|
|
@@ -342,7 +342,7 @@ print_results(result.results)
|
|
|
342
342
|
|
|
343
343
|
`upload_audio()` returns a process with a unique process ID (`pid`). `wait_for_result()` polls until processing completes and returns the analysis result.
|
|
344
344
|
|
|
345
|
-
`print_results()` prints one line per result: the start and end time in seconds, the task, and the
|
|
345
|
+
`print_results()` prints one line per result: the start and end time in seconds, the task, and the final label with its probability.
|
|
346
346
|
Continuous tasks such as `intensity` have no label, only a score, and `asr` and `diarization` have a label without a probability.
|
|
347
347
|
For a 10-second clip of one speaker, the output starts like this:
|
|
348
348
|
|
|
@@ -368,6 +368,52 @@ process = client.behavioral.upload_audio(
|
|
|
368
368
|
)
|
|
369
369
|
```
|
|
370
370
|
|
|
371
|
+
### Behavioral outputs
|
|
372
|
+
|
|
373
|
+
The Behavioral API returns **24 signals** across 8 dimensions: 18 behavioral signals and 6 speaker signals. Each label is its own signal with its own probability.
|
|
374
|
+
All eight dimensions are scored together on every utterance longer than 1 second, so one utterance can be *sad + weak + negative + slow + hesitating + withdrawn*, with a voice estimated as female and aged 31 - 45. That gives 4 × 3 × 3 × 3 × 2 × 3 × 2 × 4 = **5,184 possible profiles** per utterance.
|
|
375
|
+
|
|
376
|
+
| Task | What it measures | Labels | # |
|
|
377
|
+
|---|---|---|---|
|
|
378
|
+
| `emotion` | Basic emotion in the voice | `happy`, `angry`, `sad`, `neutral` | 4 |
|
|
379
|
+
| `strength` | Arousal: energy in the voice | `strong`, `weak`, `neutral` | 3 |
|
|
380
|
+
| `positivity` | Valence: sentiment of the tone | `positive`, `negative`, `neutral` | 3 |
|
|
381
|
+
| `speaking_rate` | How fast the speaker talks, compared to speakers in general, not to their own pace | `fast`, `slow`, `normal` | 3 |
|
|
382
|
+
| `hesitation` | Signs of hesitation | `yes`, `no` | 2 |
|
|
383
|
+
| `engagement` | Involved or detached tone | `engaged`, `withdrawn`, `neutral` | 3 |
|
|
384
|
+
| `gender` | Sex of the speaker, estimated from the voice | `female`, `male` | 2 |
|
|
385
|
+
| `age` | Age range of the speaker, estimated from the voice | `18 - 22`, `23 - 30`, `31 - 45`, `46 - 65` | 4 |
|
|
386
|
+
| **Total** | | | **24** |
|
|
387
|
+
|
|
388
|
+
Plus `intensity`: how intense the emotion is, as one score between 0 and 1. It is not a score for any one emotion, even when `emotion` is `angry`. It has no labels and is not counted in the 24.
|
|
389
|
+
In batch results, utterances of 1 second or less return only `diarization`, `asr` and `language`. When streaming, every segment gets all 24 signals and `intensity`, however short it is.
|
|
390
|
+
In batch results, `gender` and `age` are given per speaker: every utterance of a speaker longer than 1 second gets the same values, averaged over the file. When streaming, `gender` and `age` are estimated on each segment.
|
|
391
|
+
The other tasks have open-ended values, not fixed labels: `diarization` (a speaker ID such as `SPEAKER_00`), `asr` (the transcript) and `language`. With `embeddings=True`, `features` holds the behavioral embedding.
|
|
392
|
+
All results of one utterance share its `id`, `startTime` and `endTime`. To split results by speaker, use the `diarization` label of each utterance. Speaker IDs are generic: the API does not say who is the agent and who is the customer.
|
|
393
|
+
|
|
394
|
+
`neutral` means something different in each task, so always read a label together with its task: `emotion: neutral` means no clear emotion, `strength: neutral` normal energy, `positivity: neutral` neither positive nor negative, and `engagement: neutral` neither engaged nor withdrawn.
|
|
395
|
+
|
|
396
|
+
`finalLabel` gives one answer per task. For behavioral tasks it is the label with the highest probability.
|
|
397
|
+
When you need a score instead of a label, use the probabilities in `prediction`. They are strings, so convert them with `float()`.
|
|
398
|
+
For each task in the table they add up to 1, so each label's probability is a score from 0 to 1 that you can track over time, average over a call, or compare against your own threshold.
|
|
399
|
+
For example, the `angry` probability of `emotion` is a score for anger. `intensity` is not: it does not say which emotion is intense.
|
|
400
|
+
|
|
401
|
+
`positivity` is valence and `strength` is arousal, the two dimensions often used in emotion research. The API gives them as probabilities from 0 to 1. To put them on the usual scale from -1 to 1, subtract the probabilities of the two opposite labels:
|
|
402
|
+
|
|
403
|
+
```python
|
|
404
|
+
for item in result.results:
|
|
405
|
+
if item.task in ("positivity", "strength"):
|
|
406
|
+
scores = {p.label: float(p.posterior) for p in item.prediction}
|
|
407
|
+
if item.task == "positivity":
|
|
408
|
+
print(item.st, "valence", scores["positive"] - scores["negative"])
|
|
409
|
+
else:
|
|
410
|
+
print(item.st, "arousal", scores["strong"] - scores["weak"])
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
All probabilities are model estimates: use them to compare utterances and follow trends, not as exact measurements.
|
|
414
|
+
|
|
415
|
+
See [Definition of behaviors](https://behavioralsignals.readme.io/docs/definition-of-behaviors) for what each signal means.
|
|
416
|
+
|
|
371
417
|
## Deepfake Detection
|
|
372
418
|
|
|
373
419
|
### Audio
|
|
@@ -629,8 +675,8 @@ The server exposes four tools:
|
|
|
629
675
|
| Tool | What it does |
|
|
630
676
|
|---|---|
|
|
631
677
|
| `analyze_behavior` | Uploads an audio file or S3 presigned URL for behavioral analysis and returns the results |
|
|
632
|
-
| `detect_deepfake` | Uploads an audio or video file, or an S3 presigned URL, for deepfake detection and returns the results |
|
|
633
|
-
| `get_result` | Returns the results of a process, in pages, optionally filtered to specific tasks |
|
|
678
|
+
| `detect_deepfake` | Uploads an audio or video file, or an S3 presigned URL, for deepfake detection (video is experimental) and returns the results |
|
|
679
|
+
| `get_result` | Returns the results of a process, in pages, optionally filtered to specific tasks. Each row has the final label and the probability of every label |
|
|
634
680
|
| `list_processes` | Lists processes, newest first, including the failure reason when available |
|
|
635
681
|
|
|
636
682
|
Notes:
|
|
@@ -111,13 +111,25 @@ async def test_formats_labelled_rows_and_drops_the_rest(api):
|
|
|
111
111
|
assert not is_error
|
|
112
112
|
assert text == (
|
|
113
113
|
"pid 7 completed. Rows 1-3 of 3. Tasks: asr, emotion, intensity\n"
|
|
114
|
-
"start\tend\ttask\tlabel\
|
|
114
|
+
"start\tend\ttask\tlabel\tprobabilities\n"
|
|
115
115
|
"0.5\t1.5\tasr\tHello there.\t\n"
|
|
116
|
-
"0.5\t1.5\temotion\tsad\
|
|
116
|
+
"0.5\t1.5\temotion\tsad\tsad=0.71\n"
|
|
117
117
|
"0.5\t1.5\tintensity\t0.62\t"
|
|
118
118
|
)
|
|
119
119
|
|
|
120
120
|
|
|
121
|
+
async def test_rows_show_the_probability_of_every_label(api):
|
|
122
|
+
prediction = [
|
|
123
|
+
ModelPredictions(label="sad", posterior="0.5"),
|
|
124
|
+
ModelPredictions(label="neutral", posterior="0.4"),
|
|
125
|
+
]
|
|
126
|
+
api.responses["wait_for_result"] = ResultResponse(
|
|
127
|
+
pid=7, results=[ResultItem(task="emotion", finalLabel="neutral", prediction=prediction)]
|
|
128
|
+
)
|
|
129
|
+
_, text = await call("get_result", pid=7, analysis="behavioral")
|
|
130
|
+
assert text.endswith("\temotion\tneutral\tsad=0.50, neutral=0.40")
|
|
131
|
+
|
|
132
|
+
|
|
121
133
|
async def test_tabs_and_newlines_in_a_cell_become_spaces(api):
|
|
122
134
|
api.responses["wait_for_result"] = ResultResponse(
|
|
123
135
|
pid=7, results=[item("asr", "Hi\tthere.\nBye.")]
|
|
@@ -134,9 +146,9 @@ async def test_video_rows_have_a_track_column(api):
|
|
|
134
146
|
)
|
|
135
147
|
_, text = await call("get_result", pid=7, analysis="deepfake_video")
|
|
136
148
|
assert text.splitlines()[1:] == [
|
|
137
|
-
"track\tstart\tend\ttask\tlabel\
|
|
138
|
-
"audio\t0.5\t1.5\tdeepfake\tbonafide\
|
|
139
|
-
"video\t0.5\t1.5\tvisual_deepfake\tspoofed\
|
|
149
|
+
"track\tstart\tend\ttask\tlabel\tprobabilities",
|
|
150
|
+
"audio\t0.5\t1.5\tdeepfake\tbonafide\tbonafide=0.90",
|
|
151
|
+
"video\t0.5\t1.5\tvisual_deepfake\tspoofed\tspoofed=0.80",
|
|
140
152
|
]
|
|
141
153
|
|
|
142
154
|
|
|
@@ -59,6 +59,12 @@ def test_print_results_prints_one_line_per_result(capsys):
|
|
|
59
59
|
)
|
|
60
60
|
|
|
61
61
|
|
|
62
|
+
def test_print_results_shows_the_probability_of_the_final_label(capsys):
|
|
63
|
+
prediction = [{"label": "sad", "posterior": "0.5"}, {"label": "neutral", "posterior": "0.4"}]
|
|
64
|
+
print_results([_item("emotion", prediction, "neutral")])
|
|
65
|
+
assert capsys.readouterr().out == "0.487 3.001 emotion neutral (40.0%)\n"
|
|
66
|
+
|
|
67
|
+
|
|
62
68
|
def test_print_results_accepts_none(capsys):
|
|
63
69
|
print_results(None)
|
|
64
70
|
assert capsys.readouterr().out == ""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/__init__.py
RENAMED
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/api_pb2.py
RENAMED
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/api_pb2.pyi
RENAMED
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals/generated/api_pb2_grpc.py
RENAMED
|
File without changes
|
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/requires.txt
RENAMED
|
File without changes
|
{behavioralsignals-0.7.1 → behavioralsignals-0.7.2}/src/behavioralsignals.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|