vision-agents-plugins-tencent 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vision_agents_plugins_tencent-0.6.0/.gitignore +102 -0
- vision_agents_plugins_tencent-0.6.0/PKG-INFO +96 -0
- vision_agents_plugins_tencent-0.6.0/README.md +80 -0
- vision_agents_plugins_tencent-0.6.0/pyproject.toml +42 -0
- vision_agents_plugins_tencent-0.6.0/vision_agents/plugins/tencent/__init__.py +5 -0
- vision_agents_plugins_tencent-0.6.0/vision_agents/plugins/tencent/bindings.py +125 -0
- vision_agents_plugins_tencent-0.6.0/vision_agents/plugins/tencent/delegate.py +242 -0
- vision_agents_plugins_tencent-0.6.0/vision_agents/plugins/tencent/scene.py +43 -0
- vision_agents_plugins_tencent-0.6.0/vision_agents/plugins/tencent/tencent_edge.py +369 -0
- vision_agents_plugins_tencent-0.6.0/vision_agents/plugins/tencent/tracks.py +332 -0
- vision_agents_plugins_tencent-0.6.0/vision_agents/plugins/tencent/video_utils.py +30 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.cursor/*
|
|
7
|
+
# Distribution / packaging
|
|
8
|
+
.Python
|
|
9
|
+
build/
|
|
10
|
+
dist/
|
|
11
|
+
downloads/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
eggs/
|
|
14
|
+
.eggs/
|
|
15
|
+
lib64/
|
|
16
|
+
parts/
|
|
17
|
+
sdist/
|
|
18
|
+
var/
|
|
19
|
+
wheels/
|
|
20
|
+
share/python-wheels/
|
|
21
|
+
pip-wheel-metadata/
|
|
22
|
+
MANIFEST
|
|
23
|
+
*.egg-info/
|
|
24
|
+
*.egg
|
|
25
|
+
|
|
26
|
+
# Installer logs
|
|
27
|
+
pip-log.txt
|
|
28
|
+
pip-delete-this-directory.txt
|
|
29
|
+
|
|
30
|
+
# Unit test / coverage reports
|
|
31
|
+
htmlcov/
|
|
32
|
+
.tox/
|
|
33
|
+
.nox/
|
|
34
|
+
.coverage
|
|
35
|
+
.coverage.*
|
|
36
|
+
.cache
|
|
37
|
+
coverage.xml
|
|
38
|
+
nosetests.xml
|
|
39
|
+
*.cover
|
|
40
|
+
*.py,cover
|
|
41
|
+
.hypothesis/
|
|
42
|
+
.pytest_cache/
|
|
43
|
+
|
|
44
|
+
# Type checker / lint caches
|
|
45
|
+
.mypy_cache/
|
|
46
|
+
.dmypy.json
|
|
47
|
+
dmypy.json
|
|
48
|
+
.pytype/
|
|
49
|
+
.pyre/
|
|
50
|
+
.ruff_cache/
|
|
51
|
+
|
|
52
|
+
# Environments
|
|
53
|
+
.venv
|
|
54
|
+
env/
|
|
55
|
+
venv/
|
|
56
|
+
ENV/
|
|
57
|
+
env.bak/
|
|
58
|
+
venv.bak/
|
|
59
|
+
.env
|
|
60
|
+
.env.local
|
|
61
|
+
.env.*.local
|
|
62
|
+
.env.bak
|
|
63
|
+
pyvenv.cfg
|
|
64
|
+
.python-version
|
|
65
|
+
|
|
66
|
+
# Editors / IDEs
|
|
67
|
+
.vscode/
|
|
68
|
+
.idea/
|
|
69
|
+
|
|
70
|
+
# Jupyter Notebook
|
|
71
|
+
.ipynb_checkpoints/
|
|
72
|
+
|
|
73
|
+
# OS / Misc
|
|
74
|
+
.DS_Store
|
|
75
|
+
*.log
|
|
76
|
+
|
|
77
|
+
# Tooling & repo-specific
|
|
78
|
+
pyrightconfig.json
|
|
79
|
+
shell.nix
|
|
80
|
+
bin/*
|
|
81
|
+
lib/*
|
|
82
|
+
stream-py/
|
|
83
|
+
|
|
84
|
+
# Example lock files (regenerated by uv sync)
|
|
85
|
+
examples/*/uv.lock
|
|
86
|
+
plugins/*/example/uv.lock
|
|
87
|
+
|
|
88
|
+
# Artifacts / assets
|
|
89
|
+
*.pt
|
|
90
|
+
*.kef
|
|
91
|
+
*.onnx
|
|
92
|
+
profile.html
|
|
93
|
+
|
|
94
|
+
/opencode.json
|
|
95
|
+
.ralph-tui/
|
|
96
|
+
.claude/*
|
|
97
|
+
!.claude/skills/
|
|
98
|
+
|
|
99
|
+
.uv-cache/
|
|
100
|
+
|
|
101
|
+
# pytest json report
|
|
102
|
+
.report.json
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vision-agents-plugins-tencent
|
|
3
|
+
Version: 0.6.0
|
|
4
|
+
Summary: Tencent TRTC edge transport for Vision Agents
|
|
5
|
+
Project-URL: Documentation, https://visionagents.ai/
|
|
6
|
+
Project-URL: Website, https://visionagents.ai/
|
|
7
|
+
Project-URL: Source, https://github.com/GetStream/Vision-Agents
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Keywords: edge,rtc,tencent,trtc,vision agents,voice agents
|
|
10
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Requires-Dist: liteav>=13.2; sys_platform == 'linux'
|
|
13
|
+
Requires-Dist: tls-sig-api-v2
|
|
14
|
+
Requires-Dist: vision-agents
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# Vision Agents Plugin: Tencent TRTC Edge
|
|
18
|
+
|
|
19
|
+
Tencent TRTC (Real-Time Communication) edge transport for Vision Agents. Lets an agent join a Tencent TRTC room and exchange audio/video with participants using the [Tencent LiteAV SDK](https://cloud.tencent.com/document/product/647) Python bindings.
|
|
20
|
+
|
|
21
|
+
## Quickstart
|
|
22
|
+
|
|
23
|
+
Talk to an agent in 5 minutes. You'll need Docker, an `.env` with `TENCENT_SDK_APP_ID`, `TENCENT_SDK_SECRET_KEY`, `OPENAI_API_KEY`, and `ELEVEN_API_KEY` at the repo root, and a working microphone in a Chromium-based browser. The example pairs Tencent TRTC with OpenAI (LLM) and ElevenLabs (STT + TTS) — chosen because Gemini and Cartesia aren't reachable from mainland China, which is the natural deployment target for this edge.
|
|
24
|
+
|
|
25
|
+
1. **Open Tencent's hosted TRTC Web SDK quick demo:**
|
|
26
|
+
<https://web.sdk.qcloud.com/trtc/webrtc/v5/demo/quick-demo-js/index.html>
|
|
27
|
+
|
|
28
|
+
- Paste your `TENCENT_SDK_APP_ID` into **SDKAppID** and `TENCENT_SDK_SECRET_KEY` into **SDKSecretKey** — the page generates `UserSig` client-side.
|
|
29
|
+
- Leave the auto-generated **UserID** and **RoomID(String)** as is.
|
|
30
|
+
- Click **Enter Room** → demo log should print `🟩 [user_***] enterRoom.`
|
|
31
|
+
- Click **Start Local Video** — this also publishes the mic.
|
|
32
|
+
|
|
33
|
+
2. **Copy the `RoomID(String)` from the demo form** and launch the agent with it:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
cd plugins/tencent
|
|
37
|
+
TENCENT_TEST_ROOM_ID=<paste-room-id-here> docker compose run --rm tencent-agent
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
On first run this builds the image and resolves the workspace; subsequent runs are fast. When the agent joins you'll see `Tencent TRTC OnRemoteUserEnterRoom: <userId>` matching the **UserID** shown in the demo.
|
|
41
|
+
|
|
42
|
+
3. **Talk.** The flow is `browser → Tencent → STT (ElevenLabs) → LLM (OpenAI) → TTS (ElevenLabs) → Tencent → browser`. Confirm each leg in the agent log:
|
|
43
|
+
- `🎤 [Transcript Complete]: …` — STT got your speech.
|
|
44
|
+
- `🤖 [LLM response final]: …` — LLM produced a reply.
|
|
45
|
+
- Reply plays back in the browser.
|
|
46
|
+
|
|
47
|
+
## Usage in your own code
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from vision_agents.core import Agent, User
|
|
51
|
+
from vision_agents.plugins import tencent
|
|
52
|
+
|
|
53
|
+
agent_user = User(id="agent-1", name="Agent")
|
|
54
|
+
edge = tencent.Edge(
|
|
55
|
+
sdk_app_id=YOUR_SDK_APP_ID,
|
|
56
|
+
user_sig=USER_SIG, # or key=KEY to generate per join
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
agent = Agent(
|
|
60
|
+
edge=edge,
|
|
61
|
+
agent_user=agent_user,
|
|
62
|
+
llm=...,
|
|
63
|
+
# stt, tts, etc.
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
call = await edge.create_call(call_id=str(ROOM_ID), room_id=ROOM_ID)
|
|
67
|
+
await agent.join(call)
|
|
68
|
+
await agent.finish()
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Install
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
uv add vision-agents-plugins-tencent
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
On Linux this pulls `liteav` from PyPI. On macOS the `liteav` dependency is skipped via a platform marker, so the package installs but `tencent.Edge()` raises at runtime — use the Docker setup from the Quickstart.
|
|
78
|
+
|
|
79
|
+
## Configuration
|
|
80
|
+
|
|
81
|
+
`tencent.Edge(...)` parameters:
|
|
82
|
+
|
|
83
|
+
- **sdk_app_id** (int): Tencent TRTC SDK App ID. Falls back to `TENCENT_SDK_APP_ID` env var.
|
|
84
|
+
- **user_sig** (str): User signature for the agent user.
|
|
85
|
+
- **key** (str): Optional. If set and `user_sig` is not, the plugin generates `user_sig` via TLSSigAPIv2. Falls back to `TENCENT_SDK_SECRET_KEY` env var.
|
|
86
|
+
- **video_fps** (int): Outgoing video frame rate.
|
|
87
|
+
|
|
88
|
+
Environment variables:
|
|
89
|
+
|
|
90
|
+
- `TENCENT_SDK_APP_ID`, `TENCENT_SDK_SECRET_KEY` — credentials.
|
|
91
|
+
- `TENCENT_TRTC_SCENE` — one of `auto` (default), `videocall`, `call`, `record`.
|
|
92
|
+
- `TENCENT_TEST_ROOM_ID` — interpolated by `docker-compose.yml` into the example runner's `--call-id` flag.
|
|
93
|
+
|
|
94
|
+
## Platform support
|
|
95
|
+
|
|
96
|
+
The underlying [`liteav`](https://pypi.org/project/liteav/) package ships only manylinux wheels — **Linux x86_64 / aarch64 only**. macOS and Windows must run the agent inside a Linux container (see Quickstart). User sigs are needed for room entry; either generate them with [TLSSigAPIv2](https://cloud.tencent.com/document/product/647/34399) or pass `key=` and let the plugin sign per join.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Vision Agents Plugin: Tencent TRTC Edge
|
|
2
|
+
|
|
3
|
+
Tencent TRTC (Real-Time Communication) edge transport for Vision Agents. Lets an agent join a Tencent TRTC room and exchange audio/video with participants using the [Tencent LiteAV SDK](https://cloud.tencent.com/document/product/647) Python bindings.
|
|
4
|
+
|
|
5
|
+
## Quickstart
|
|
6
|
+
|
|
7
|
+
Talk to an agent in 5 minutes. You'll need Docker, an `.env` with `TENCENT_SDK_APP_ID`, `TENCENT_SDK_SECRET_KEY`, `OPENAI_API_KEY`, and `ELEVEN_API_KEY` at the repo root, and a working microphone in a Chromium-based browser. The example pairs Tencent TRTC with OpenAI (LLM) and ElevenLabs (STT + TTS) — chosen because Gemini and Cartesia aren't reachable from mainland China, which is the natural deployment target for this edge.
|
|
8
|
+
|
|
9
|
+
1. **Open Tencent's hosted TRTC Web SDK quick demo:**
|
|
10
|
+
<https://web.sdk.qcloud.com/trtc/webrtc/v5/demo/quick-demo-js/index.html>
|
|
11
|
+
|
|
12
|
+
- Paste your `TENCENT_SDK_APP_ID` into **SDKAppID** and `TENCENT_SDK_SECRET_KEY` into **SDKSecretKey** — the page generates `UserSig` client-side.
|
|
13
|
+
- Leave the auto-generated **UserID** and **RoomID(String)** as is.
|
|
14
|
+
- Click **Enter Room** → demo log should print `🟩 [user_***] enterRoom.`
|
|
15
|
+
- Click **Start Local Video** — this also publishes the mic.
|
|
16
|
+
|
|
17
|
+
2. **Copy the `RoomID(String)` from the demo form** and launch the agent with it:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
cd plugins/tencent
|
|
21
|
+
TENCENT_TEST_ROOM_ID=<paste-room-id-here> docker compose run --rm tencent-agent
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
On first run this builds the image and resolves the workspace; subsequent runs are fast. When the agent joins you'll see `Tencent TRTC OnRemoteUserEnterRoom: <userId>` matching the **UserID** shown in the demo.
|
|
25
|
+
|
|
26
|
+
3. **Talk.** The flow is `browser → Tencent → STT (ElevenLabs) → LLM (OpenAI) → TTS (ElevenLabs) → Tencent → browser`. Confirm each leg in the agent log:
|
|
27
|
+
- `🎤 [Transcript Complete]: …` — STT got your speech.
|
|
28
|
+
- `🤖 [LLM response final]: …` — LLM produced a reply.
|
|
29
|
+
- Reply plays back in the browser.
|
|
30
|
+
|
|
31
|
+
## Usage in your own code
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from vision_agents.core import Agent, User
|
|
35
|
+
from vision_agents.plugins import tencent
|
|
36
|
+
|
|
37
|
+
agent_user = User(id="agent-1", name="Agent")
|
|
38
|
+
edge = tencent.Edge(
|
|
39
|
+
sdk_app_id=YOUR_SDK_APP_ID,
|
|
40
|
+
user_sig=USER_SIG, # or key=KEY to generate per join
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
agent = Agent(
|
|
44
|
+
edge=edge,
|
|
45
|
+
agent_user=agent_user,
|
|
46
|
+
llm=...,
|
|
47
|
+
# stt, tts, etc.
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
call = await edge.create_call(call_id=str(ROOM_ID), room_id=ROOM_ID)
|
|
51
|
+
await agent.join(call)
|
|
52
|
+
await agent.finish()
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
uv add vision-agents-plugins-tencent
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
On Linux this pulls `liteav` from PyPI. On macOS the `liteav` dependency is skipped via a platform marker, so the package installs but `tencent.Edge()` raises at runtime — use the Docker setup from the Quickstart.
|
|
62
|
+
|
|
63
|
+
## Configuration
|
|
64
|
+
|
|
65
|
+
`tencent.Edge(...)` parameters:
|
|
66
|
+
|
|
67
|
+
- **sdk_app_id** (int): Tencent TRTC SDK App ID. Falls back to `TENCENT_SDK_APP_ID` env var.
|
|
68
|
+
- **user_sig** (str): User signature for the agent user.
|
|
69
|
+
- **key** (str): Optional. If set and `user_sig` is not, the plugin generates `user_sig` via TLSSigAPIv2. Falls back to `TENCENT_SDK_SECRET_KEY` env var.
|
|
70
|
+
- **video_fps** (int): Outgoing video frame rate.
|
|
71
|
+
|
|
72
|
+
Environment variables:
|
|
73
|
+
|
|
74
|
+
- `TENCENT_SDK_APP_ID`, `TENCENT_SDK_SECRET_KEY` — credentials.
|
|
75
|
+
- `TENCENT_TRTC_SCENE` — one of `auto` (default), `videocall`, `call`, `record`.
|
|
76
|
+
- `TENCENT_TEST_ROOM_ID` — interpolated by `docker-compose.yml` into the example runner's `--call-id` flag.
|
|
77
|
+
|
|
78
|
+
## Platform support
|
|
79
|
+
|
|
80
|
+
The underlying [`liteav`](https://pypi.org/project/liteav/) package ships only manylinux wheels — **Linux x86_64 / aarch64 only**. macOS and Windows must run the agent inside a Linux container (see Quickstart). User sigs are needed for room entry; either generate them with [TLSSigAPIv2](https://cloud.tencent.com/document/product/647/34399) or pass `key=` and let the plugin sign per join.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "vision-agents-plugins-tencent"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Tencent TRTC edge transport for Vision Agents"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
keywords = ["tencent", "trtc", "rtc", "edge", "vision agents", "voice agents"]
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
license = "MIT"
|
|
13
|
+
classifiers = ["Operating System :: POSIX :: Linux"]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"vision-agents",
|
|
16
|
+
"liteav>=13.2; sys_platform == 'linux'",
|
|
17
|
+
"tls-sig-api-v2",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Documentation = "https://visionagents.ai/"
|
|
22
|
+
Website = "https://visionagents.ai/"
|
|
23
|
+
Source = "https://github.com/GetStream/Vision-Agents"
|
|
24
|
+
|
|
25
|
+
[tool.hatch.version]
|
|
26
|
+
source = "vcs"
|
|
27
|
+
raw-options = { root = "..", search_parent_directories = true, fallback_version = "0.0.0" }
|
|
28
|
+
|
|
29
|
+
[tool.hatch.build.targets.wheel]
|
|
30
|
+
packages = [".", "vision_agents"]
|
|
31
|
+
|
|
32
|
+
[tool.hatch.build.targets.sdist]
|
|
33
|
+
include = ["/vision_agents"]
|
|
34
|
+
|
|
35
|
+
[tool.uv.sources]
|
|
36
|
+
vision-agents = { workspace = true }
|
|
37
|
+
|
|
38
|
+
[dependency-groups]
|
|
39
|
+
dev = [
|
|
40
|
+
"pytest>=8.4.1",
|
|
41
|
+
"pytest-asyncio>=1.0.0",
|
|
42
|
+
]
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Conditional `liteav` imports for the Tencent plugin.
|
|
2
|
+
|
|
3
|
+
`liteav` ships only manylinux wheels on PyPI, so on macOS/Windows the
|
|
4
|
+
import will fail. This module centralises the conditional import noise:
|
|
5
|
+
all other modules in the plugin re-import the names they need from
|
|
6
|
+
here. ``require_liteav()`` raises a friendly RuntimeError at call time
|
|
7
|
+
if liteav couldn't be loaded.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
from typing import Optional
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"AUDIO_CODEC_TYPE_PCM",
|
|
18
|
+
"AUDIO_OBTAIN_METHOD_CALLBACK",
|
|
19
|
+
"AudioEncodeParams",
|
|
20
|
+
"AudioFrame",
|
|
21
|
+
"CreateTRTCCloud",
|
|
22
|
+
"DestroyTRTCCloud",
|
|
23
|
+
"EnterRoomParams",
|
|
24
|
+
"LITEAV_IMPORT_ERROR",
|
|
25
|
+
"PixelFrame",
|
|
26
|
+
"STREAM_TYPE_VIDEO_HIGH",
|
|
27
|
+
"TRTCCloudDelegate",
|
|
28
|
+
"TRTC_ROLE_ANCHOR",
|
|
29
|
+
"TRTC_SCENE_CALL",
|
|
30
|
+
"TRTC_SCENE_RECORD",
|
|
31
|
+
"TRTC_SCENE_VIDEOCALL",
|
|
32
|
+
"TrtcString",
|
|
33
|
+
"VIDEO_PIXEL_FORMAT_YUV420p",
|
|
34
|
+
"VIDEO_ROTATION_0",
|
|
35
|
+
"VideoEncodeParams",
|
|
36
|
+
"require_liteav",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
LITEAV_IMPORT_ERROR: Optional[ImportError] = None
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
# liteav's typed stubs don't expose these top-level module constants
|
|
43
|
+
# and convenience constructors even though they exist at runtime, so
|
|
44
|
+
# we silence the attr-defined / call-arg complaints in this block.
|
|
45
|
+
from liteav import ( # type: ignore[attr-defined]
|
|
46
|
+
AUDIO_CODEC_TYPE_PCM,
|
|
47
|
+
AUDIO_OBTAIN_METHOD_CALLBACK,
|
|
48
|
+
STREAM_TYPE_VIDEO_HIGH,
|
|
49
|
+
VIDEO_PIXEL_FORMAT_YUV420p,
|
|
50
|
+
VIDEO_ROTATION_0,
|
|
51
|
+
AudioEncodeParams,
|
|
52
|
+
AudioFrame,
|
|
53
|
+
CreateTRTCCloud,
|
|
54
|
+
DestroyTRTCCloud,
|
|
55
|
+
EnterRoomParams,
|
|
56
|
+
PixelFrame,
|
|
57
|
+
TrtcString,
|
|
58
|
+
TRTCCloudDelegate,
|
|
59
|
+
TRTC_ROLE_ANCHOR,
|
|
60
|
+
TRTC_SCENE_RECORD,
|
|
61
|
+
VideoEncodeParams,
|
|
62
|
+
)
|
|
63
|
+
except ImportError as e:
|
|
64
|
+
LITEAV_IMPORT_ERROR = e
|
|
65
|
+
# All names need module-level fallbacks so downstream `from bindings
|
|
66
|
+
# import ...` succeeds on non-linux. TencentEdge.__init__ calls
|
|
67
|
+
# require_liteav() which raises before any of these get touched.
|
|
68
|
+
AUDIO_CODEC_TYPE_PCM = None # type: ignore[assignment]
|
|
69
|
+
AUDIO_OBTAIN_METHOD_CALLBACK = None # type: ignore[assignment]
|
|
70
|
+
STREAM_TYPE_VIDEO_HIGH = None # type: ignore[assignment]
|
|
71
|
+
VIDEO_PIXEL_FORMAT_YUV420p = None # type: ignore[assignment]
|
|
72
|
+
VIDEO_ROTATION_0 = None # type: ignore[assignment]
|
|
73
|
+
AudioEncodeParams = None # type: ignore[misc, assignment]
|
|
74
|
+
AudioFrame = None # type: ignore[misc, assignment]
|
|
75
|
+
CreateTRTCCloud = None
|
|
76
|
+
DestroyTRTCCloud = None
|
|
77
|
+
EnterRoomParams = None # type: ignore[misc, assignment]
|
|
78
|
+
PixelFrame = None # type: ignore[misc, assignment]
|
|
79
|
+
TrtcString = None # type: ignore[misc, assignment]
|
|
80
|
+
TRTCCloudDelegate = None # type: ignore[misc, assignment]
|
|
81
|
+
TRTC_ROLE_ANCHOR = None # type: ignore[assignment]
|
|
82
|
+
TRTC_SCENE_RECORD = None # type: ignore[assignment]
|
|
83
|
+
VideoEncodeParams = None # type: ignore[misc, assignment]
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
from liteav import TRTC_SCENE_CALL, TRTC_SCENE_VIDEOCALL # type: ignore[attr-defined]
|
|
87
|
+
except ImportError:
|
|
88
|
+
TRTC_SCENE_VIDEOCALL = None # type: ignore[assignment]
|
|
89
|
+
TRTC_SCENE_CALL = None # type: ignore[assignment]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# liteav's C++ side writes verbose `[I][...]` / `[E][...]` lines straight
|
|
93
|
+
# to stdout. Calling DisableConsoleLog + SetLogLevel(kNone) silences the
|
|
94
|
+
# periodic noise (thread_watchdog, audio_track_health_monitor, etc.) but
|
|
95
|
+
# the SDK still writes a chunk of connection-setup logs from
|
|
96
|
+
# CreateTRTCCloud → EnterRoom that we can't suppress through its public
|
|
97
|
+
# API. Set TENCENT_LITEAV_DEBUG=1 to bring those periodic logs back for
|
|
98
|
+
# diagnostics. Note on the level enum (non-standard): 0=kAll, 1=kInfo,
|
|
99
|
+
# 2=kWarning, 3=kError, 4=kFatal, 5=kNone (>=6 is UN_DEF, no-op).
|
|
100
|
+
_TRUTHY = {"1", "true", "yes", "on"}
|
|
101
|
+
_LITEAV_LOG_LEVEL_NONE = 5
|
|
102
|
+
|
|
103
|
+
if (
|
|
104
|
+
LITEAV_IMPORT_ERROR is None
|
|
105
|
+
and os.environ.get("TENCENT_LITEAV_DEBUG", "").strip().lower() not in _TRUTHY
|
|
106
|
+
):
|
|
107
|
+
try:
|
|
108
|
+
import liteav as _liteav
|
|
109
|
+
|
|
110
|
+
_liteav.DisableConsoleLog() # type: ignore[attr-defined]
|
|
111
|
+
_liteav.SetLogLevel(_LITEAV_LOG_LEVEL_NONE) # type: ignore[attr-defined]
|
|
112
|
+
except AttributeError as e:
|
|
113
|
+
# Older liteav builds may not expose these helpers.
|
|
114
|
+
logger.debug("Could not silence liteav console log: %s", e)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def require_liteav() -> None:
|
|
118
|
+
"""Raise a friendly error if liteav couldn't be imported on this platform."""
|
|
119
|
+
if LITEAV_IMPORT_ERROR is not None:
|
|
120
|
+
raise RuntimeError(
|
|
121
|
+
"Tencent TRTC edge requires the `liteav` package, which ships only "
|
|
122
|
+
"manylinux wheels and cannot be imported on this platform. Install "
|
|
123
|
+
"on Linux (x86_64 or aarch64) with `pip install liteav` or run the "
|
|
124
|
+
"agent inside a Linux container (see plugins/tencent/README.md)."
|
|
125
|
+
) from LITEAV_IMPORT_ERROR
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""TRTCCloudDelegate subclass that forwards SDK callbacks to TencentEdge.
|
|
2
|
+
|
|
3
|
+
The class is defined only when liteav was importable; on non-linux
|
|
4
|
+
platforms ``TencentDelegate`` resolves to ``None`` and TencentEdge
|
|
5
|
+
fails earlier via ``require_liteav()``.
|
|
6
|
+
|
|
7
|
+
All callbacks are wrapped in try/except because any unhandled Python
|
|
8
|
+
exception in a SWIG director callback triggers
|
|
9
|
+
Swig::DirectorMethodException on the C++ side, which calls
|
|
10
|
+
std::terminate and kills the process.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import logging
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
from vision_agents.plugins.tencent.bindings import (
|
|
18
|
+
STREAM_TYPE_VIDEO_HIGH,
|
|
19
|
+
AudioEncodeParams,
|
|
20
|
+
TRTCCloudDelegate,
|
|
21
|
+
VideoEncodeParams,
|
|
22
|
+
)
|
|
23
|
+
from vision_agents.plugins.tencent.tracks import CHANNELS, SAMPLE_RATE
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from vision_agents.plugins.tencent.tencent_edge import (
|
|
27
|
+
TencentConnection,
|
|
28
|
+
TencentEdge,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _extract_pcm(frame: Any) -> bytes:
|
|
35
|
+
if frame.size <= 0:
|
|
36
|
+
return b""
|
|
37
|
+
return frame.data
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
TencentDelegate: Any = None
|
|
41
|
+
if TRTCCloudDelegate is not None:
|
|
42
|
+
|
|
43
|
+
class _TencentDelegateCls(TRTCCloudDelegate):
|
|
44
|
+
"""Forwards TRTC SDK callbacks to TencentEdge and TencentConnection."""
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
edge: "TencentEdge",
|
|
49
|
+
connection: "TencentConnection",
|
|
50
|
+
loop: asyncio.AbstractEventLoop,
|
|
51
|
+
):
|
|
52
|
+
# Intentionally not super().__init__(): TRTCCloudDelegate is a SWIG
|
|
53
|
+
# director — its __init__ binds the C++ side via `self.this` and
|
|
54
|
+
# registers Python callback dispatch. Tencent's bundled
|
|
55
|
+
# python_x86_64 sample uses exactly this explicit form, and that
|
|
56
|
+
# is the only invocation contract liteav documents. We follow it
|
|
57
|
+
# verbatim rather than relying on cooperative MRO that the C++
|
|
58
|
+
# binding never promised to honour.
|
|
59
|
+
TRTCCloudDelegate.__init__(self)
|
|
60
|
+
self._edge = edge
|
|
61
|
+
self._connection = connection
|
|
62
|
+
self._agent_user_id = edge._agent_user_id or ""
|
|
63
|
+
|
|
64
|
+
def OnError(self, error: int) -> None: # type: ignore[override]
|
|
65
|
+
try:
|
|
66
|
+
logger.error("Tencent TRTC OnError: %s", error)
|
|
67
|
+
self._edge._emit_call_ended()
|
|
68
|
+
except BaseException:
|
|
69
|
+
logger.exception("OnError callback failed")
|
|
70
|
+
|
|
71
|
+
def OnEnterRoom(self) -> None:
|
|
72
|
+
try:
|
|
73
|
+
logger.info("Tencent TRTC OnEnterRoom")
|
|
74
|
+
cloud = self._edge._cloud
|
|
75
|
+
if cloud is None:
|
|
76
|
+
return
|
|
77
|
+
param = AudioEncodeParams()
|
|
78
|
+
param.sample_rate = SAMPLE_RATE
|
|
79
|
+
param.channels = CHANNELS
|
|
80
|
+
param.bitrate_bps = 54000
|
|
81
|
+
cloud.CreateLocalAudioChannel(param)
|
|
82
|
+
|
|
83
|
+
if self._edge._outgoing_video_track is not None:
|
|
84
|
+
cloud.CreateLocalVideoChannel(STREAM_TYPE_VIDEO_HIGH)
|
|
85
|
+
except BaseException:
|
|
86
|
+
logger.exception("OnEnterRoom callback failed")
|
|
87
|
+
|
|
88
|
+
def OnExitRoom(self) -> None:
|
|
89
|
+
try:
|
|
90
|
+
logger.info("Tencent TRTC OnExitRoom")
|
|
91
|
+
self._edge._emit_call_ended()
|
|
92
|
+
except BaseException:
|
|
93
|
+
logger.exception("OnExitRoom callback failed")
|
|
94
|
+
|
|
95
|
+
def OnLocalAudioChannelCreated(self) -> None:
|
|
96
|
+
try:
|
|
97
|
+
if self._edge._audio_track and self._edge._cloud:
|
|
98
|
+
self._edge._audio_track.set_cloud(self._edge._cloud)
|
|
99
|
+
except BaseException:
|
|
100
|
+
logger.exception("OnLocalAudioChannelCreated callback failed")
|
|
101
|
+
|
|
102
|
+
def OnConnectionStateChanged(self, old_state: int, new_state: int) -> None: # type: ignore[override]
|
|
103
|
+
logger.debug(
|
|
104
|
+
"Tencent TRTC connection state: %s -> %s", old_state, new_state
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def OnLocalAudioChannelDestroyed(self) -> None:
|
|
108
|
+
pass
|
|
109
|
+
|
|
110
|
+
def OnLocalVideoChannelCreated(self, stream_type: int) -> None: # type: ignore[override]
|
|
111
|
+
try:
|
|
112
|
+
logger.info(
|
|
113
|
+
"Tencent TRTC OnLocalVideoChannelCreated: stream_type=%s",
|
|
114
|
+
stream_type,
|
|
115
|
+
)
|
|
116
|
+
edge = self._edge
|
|
117
|
+
if edge._outgoing_video_track and edge._cloud and edge._loop:
|
|
118
|
+
vp = VideoEncodeParams()
|
|
119
|
+
vp.frame_rate = edge._video_fps
|
|
120
|
+
vp.bitrate_bps = 1_000_000
|
|
121
|
+
vp.gop_in_seconds = 3
|
|
122
|
+
edge._cloud.SetVideoEncodeParam(STREAM_TYPE_VIDEO_HIGH, vp)
|
|
123
|
+
edge._outgoing_video_track.set_cloud(edge._cloud, edge._loop)
|
|
124
|
+
except BaseException:
|
|
125
|
+
logger.exception("OnLocalVideoChannelCreated callback failed")
|
|
126
|
+
|
|
127
|
+
def OnLocalVideoChannelDestroyed(self, stream_type: int) -> None: # type: ignore[override]
|
|
128
|
+
pass
|
|
129
|
+
|
|
130
|
+
def OnRequestChangeVideoEncodeBitrate( # type: ignore[override]
|
|
131
|
+
self, stream_type: int, bitrate_bps: int
|
|
132
|
+
) -> None:
|
|
133
|
+
pass
|
|
134
|
+
|
|
135
|
+
def OnRequestKeyFrame(self, stream_type: int) -> None: # type: ignore[override]
|
|
136
|
+
logger.debug("Tencent TRTC OnRequestKeyFrame: stream_type=%s", stream_type)
|
|
137
|
+
|
|
138
|
+
def OnRemoteAudioAvailable(self, user_id: str, available: bool) -> None:
|
|
139
|
+
try:
|
|
140
|
+
if user_id and user_id != self._agent_user_id:
|
|
141
|
+
if available:
|
|
142
|
+
self._edge._emit_track_added(user_id)
|
|
143
|
+
else:
|
|
144
|
+
self._edge._emit_track_removed(user_id)
|
|
145
|
+
except BaseException:
|
|
146
|
+
logger.exception("OnRemoteAudioAvailable callback failed")
|
|
147
|
+
|
|
148
|
+
def OnRemoteVideoAvailable( # type: ignore[override]
|
|
149
|
+
self, user_id: str, available: bool, stream_type: int
|
|
150
|
+
) -> None:
|
|
151
|
+
try:
|
|
152
|
+
if not user_id or user_id == self._agent_user_id:
|
|
153
|
+
return
|
|
154
|
+
if available:
|
|
155
|
+
logger.info(
|
|
156
|
+
"Tencent TRTC video available from %s (stream_type=%s)",
|
|
157
|
+
user_id,
|
|
158
|
+
stream_type,
|
|
159
|
+
)
|
|
160
|
+
self._edge._emit_video_track_added(user_id)
|
|
161
|
+
else:
|
|
162
|
+
logger.info("Tencent TRTC video unavailable from %s", user_id)
|
|
163
|
+
self._edge._emit_video_track_removed(user_id)
|
|
164
|
+
except BaseException:
|
|
165
|
+
logger.exception("OnRemoteVideoAvailable callback failed")
|
|
166
|
+
|
|
167
|
+
def OnRemoteVideoFrameReceived( # type: ignore[override]
|
|
168
|
+
self, user_id: str, stream_type: int, frame: Any
|
|
169
|
+
) -> None:
|
|
170
|
+
pass
|
|
171
|
+
|
|
172
|
+
def OnRemotePixelFrameReceived( # type: ignore[override]
|
|
173
|
+
self, user_id: str, stream_type: int, frame: Any
|
|
174
|
+
) -> None:
|
|
175
|
+
try:
|
|
176
|
+
if not user_id or user_id == self._agent_user_id:
|
|
177
|
+
return
|
|
178
|
+
width = frame.width
|
|
179
|
+
height = frame.height
|
|
180
|
+
if width <= 0 or height <= 0 or frame.size <= 0:
|
|
181
|
+
return
|
|
182
|
+
self._edge._push_video_frame(
|
|
183
|
+
user_id, frame.data, width, height, frame.pts
|
|
184
|
+
)
|
|
185
|
+
except BaseException:
|
|
186
|
+
logger.exception("OnRemotePixelFrameReceived callback failed")
|
|
187
|
+
|
|
188
|
+
def OnSeiMessageReceived( # type: ignore[override]
|
|
189
|
+
self, user_id: str, stream_type: int, message_type: int, message: Any
|
|
190
|
+
) -> None:
|
|
191
|
+
pass
|
|
192
|
+
|
|
193
|
+
def OnReceiveCustomCmdMsg(
|
|
194
|
+
self, user_id: str, cmd_id: int, seq: int, message: Any
|
|
195
|
+
) -> None:
|
|
196
|
+
pass
|
|
197
|
+
|
|
198
|
+
def OnMissCustomCmdMsg(
|
|
199
|
+
self, user_id: str, cmd_id: int, error_code: int, missed: int
|
|
200
|
+
) -> None:
|
|
201
|
+
pass
|
|
202
|
+
|
|
203
|
+
def OnNetworkQuality(self, local_quality: Any, remote_qualities: Any) -> None:
|
|
204
|
+
pass
|
|
205
|
+
|
|
206
|
+
def OnRemoteUserEnterRoom(self, info: Any) -> None:
|
|
207
|
+
try:
|
|
208
|
+
user_id = info.user_id.GetValue() if info and info.user_id else ""
|
|
209
|
+
if user_id and user_id != self._agent_user_id:
|
|
210
|
+
self._connection._on_remote_entered()
|
|
211
|
+
logger.info("Tencent TRTC OnRemoteUserEnterRoom: %s", user_id)
|
|
212
|
+
except BaseException:
|
|
213
|
+
logger.exception("OnRemoteUserEnterRoom callback failed")
|
|
214
|
+
|
|
215
|
+
def OnRemoteUserExitRoom(self, info: Any, reason: int) -> None:
|
|
216
|
+
try:
|
|
217
|
+
user_id = info.user_id.GetValue() if info and info.user_id else ""
|
|
218
|
+
if user_id and user_id != self._agent_user_id:
|
|
219
|
+
self._connection._on_remote_left()
|
|
220
|
+
logger.info(
|
|
221
|
+
"Tencent TRTC OnRemoteUserExitRoom: %s reason=%s", user_id, reason
|
|
222
|
+
)
|
|
223
|
+
except BaseException:
|
|
224
|
+
logger.exception("OnRemoteUserExitRoom callback failed")
|
|
225
|
+
|
|
226
|
+
def OnRemoteAudioReceived(self, user_id: str, frame: Any) -> None:
|
|
227
|
+
try:
|
|
228
|
+
if not user_id or user_id == self._agent_user_id:
|
|
229
|
+
return
|
|
230
|
+
data = _extract_pcm(frame)
|
|
231
|
+
if data:
|
|
232
|
+
self._edge._emit_audio_received(user_id, data)
|
|
233
|
+
except BaseException:
|
|
234
|
+
logger.exception("OnRemoteAudioReceived failed")
|
|
235
|
+
|
|
236
|
+
def OnRemoteMixedAudioReceived(self, frame: Any) -> None:
|
|
237
|
+
# Skip mixed audio — per-user callbacks already deliver individual
|
|
238
|
+
# streams and processing both doubles GIL / buffer pressure with
|
|
239
|
+
# no benefit for single-speaker scenarios.
|
|
240
|
+
pass
|
|
241
|
+
|
|
242
|
+
TencentDelegate = _TencentDelegateCls
|