fusion-runtime 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fusion_runtime-0.1.2/fusion_runtime.egg-info → fusion_runtime-0.1.3}/PKG-INFO +35 -2
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/README.md +34 -1
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/agent.py +4 -3
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/download.py +76 -9
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/_checks.py +14 -7
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/app.py +2 -0
- fusion_runtime-0.1.3/fusion_runtime/cli/bench.py +280 -0
- fusion_runtime-0.1.3/fusion_runtime/cli/bench_audio/hello.wav +0 -0
- fusion_runtime-0.1.3/fusion_runtime/cli/bench_audio/order_1042.wav +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/keys.py +4 -6
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/models.py +34 -8
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/up.py +133 -24
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/config.py +26 -2
- fusion_runtime-0.1.3/fusion_runtime/engine/audio_format.py +84 -0
- fusion_runtime-0.1.3/fusion_runtime/engine/audio_watch.py +73 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/barge_in.py +13 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/orchestrator.py +101 -24
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/llm_server.py +148 -18
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/resolver.py +3 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/onnx/kokoro.py +19 -6
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/onnx/tts.py +4 -1
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/openai_http/llm.py +44 -1
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/limits.py +16 -6
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/server.py +117 -23
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/metrics.py +6 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/trace.py +10 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/web/fusion-runtime.js +1 -1
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/web/index.html +18 -3
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3/fusion_runtime.egg-info}/PKG-INFO +35 -2
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/SOURCES.txt +12 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/pyproject.toml +2 -2
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_agent.py +1 -1
- fusion_runtime-0.1.3/tests/test_audio_format.py +91 -0
- fusion_runtime-0.1.3/tests/test_audio_watch.py +114 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_banner.py +27 -0
- fusion_runtime-0.1.3/tests/test_bench.py +83 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_cli.py +40 -14
- fusion_runtime-0.1.3/tests/test_degraded.py +96 -0
- fusion_runtime-0.1.3/tests/test_digital_silence.py +61 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_doctor.py +7 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_http_llm.py +70 -3
- fusion_runtime-0.1.3/tests/test_interruption_switches.py +81 -0
- fusion_runtime-0.1.3/tests/test_knowledge_example.py +48 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_llm_server.py +130 -1
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_runtimes.py +9 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_security.py +27 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_server_telemetry.py +13 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_telemetry.py +14 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_tools.py +41 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/LICENSE +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/NOTICE +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/audio/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/audio/duplex_audio.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/audio/echo_canceller.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/entries.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/gguf.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/models.toml +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/__main__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/_common.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/_talk_client.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/banner.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/doctor.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/talk.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/version.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/common.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/llm.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/stt.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/tts.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/turn.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/conversation.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/scheduler.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/streaming.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/text.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/env.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/registry.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/ctranslate2/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/ctranslate2/stt.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/llama_cpp/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/llama_cpp/llm.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/onnx/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/openai_http/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/auth.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/keys.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/origins.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/tokens.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/errors.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/events.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/hub.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/loop_monitor.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/redact.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/sinks.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/testing/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/testing/conformance.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/testing/fakes.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/tools.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/turns/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/turns/silence.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/base.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/silero.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/turn.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/web/__init__.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/dependency_links.txt +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/entry_points.txt +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/requires.txt +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/top_level.txt +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/setup.cfg +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_barge_in.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_catalog.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_client_barge_in.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_conformance.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_contract.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_conversation.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_core.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_duplex_audio.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_echo_canceller.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_engine_streaming.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_env_file.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_event_loop_blocking.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_greeting.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_languages.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_llama_cpp_stream.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_model_paths.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_registry.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_resolver.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_scheduler.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_self_echo.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_turn_taking.py +0 -0
- {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_web.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fusion-runtime
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Self-hosted voice agent runtime: speech-to-text, LLM and text-to-speech on your own GPU, with tools
|
|
5
5
|
Author-email: Samarth Urs <hello@fusion-runtime.dev>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -81,6 +81,9 @@ Requires Python 3.11–3.13.
|
|
|
81
81
|
pip install fusion-runtime
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
+
The first install can sit quiet for a few minutes: where there's no ready-made llama.cpp for your
|
|
85
|
+
machine, pip builds it. That's normal.
|
|
86
|
+
|
|
84
87
|
An agent is one file. This is the whole thing:
|
|
85
88
|
|
|
86
89
|
```python
|
|
@@ -177,6 +180,32 @@ calling on), llama-server (`--jinja`) or a hosted API. The in-process
|
|
|
177
180
|
llama.cpp runtime can't, and `frun up` says so at startup. A runnable version is
|
|
178
181
|
[`examples/tools_agent.py`](examples/tools_agent.py).
|
|
179
182
|
|
|
183
|
+
### Answering from your own documents
|
|
184
|
+
|
|
185
|
+
Retrieval is a tool like any other: a function that takes the caller's question and returns a few
|
|
186
|
+
short passages, which the model answers from.
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
@tool(timeout_s=3)
|
|
190
|
+
def search_help(question: str) -> dict:
|
|
191
|
+
"""Search the shop's help articles: returns, refunds, delivery, cancelling orders.
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
question: What the caller wants to know, in their words.
|
|
195
|
+
"""
|
|
196
|
+
return {"passages": my_search(question, limit=3)}
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
`my_search` is yours: a vector database, Elasticsearch, your help centre's API.
|
|
200
|
+
[`examples/knowledge_agent.py`](examples/knowledge_agent.py) is a runnable version with a small
|
|
201
|
+
keyword search over the Markdown files in [`examples/knowledge/`](examples/knowledge/), in plain
|
|
202
|
+
Python with nothing to install.
|
|
203
|
+
|
|
204
|
+
On a call, two things matter more than in a chat window. A lookup costs a second model round
|
|
205
|
+
before the caller hears anything, so if your knowledge fits in a page or two, put it in the prompt
|
|
206
|
+
instead: that's faster than any search. And passages go into the prompt, so keep them short (a few
|
|
207
|
+
sentences each, a few per search): long ones slow the model's first word.
|
|
208
|
+
|
|
180
209
|
## On your own site
|
|
181
210
|
|
|
182
211
|
```html
|
|
@@ -241,7 +270,7 @@ console), so you can reproduce them rather than trusting ours. Barge-in fired on
|
|
|
241
270
|
## Several callers at once
|
|
242
271
|
|
|
243
272
|
Measured on the same 3090, real WebSocket sessions, three turns each
|
|
244
|
-
(
|
|
273
|
+
(`frun bench`). Response is the server's own
|
|
245
274
|
figure: the turn ends, audio comes back.
|
|
246
275
|
|
|
247
276
|
| Callers | In-process llama.cpp | vLLM | SGLang |
|
|
@@ -287,9 +316,13 @@ fusion where it is:
|
|
|
287
316
|
```bash
|
|
288
317
|
python3 -m venv ~/vllm-env && ~/vllm-env/bin/pip install vllm
|
|
289
318
|
export FUSION_LLM_ENGINE_ENV=~/vllm-env
|
|
319
|
+
frun models pull agent.py # the LLM too (5.6 GB here), where vLLM looks for it
|
|
290
320
|
frun up agent.py
|
|
291
321
|
```
|
|
292
322
|
|
|
323
|
+
Without the pull, vLLM downloads the model during its first start, which takes 5–15 minutes and
|
|
324
|
+
looks like nothing is happening.
|
|
325
|
+
|
|
293
326
|
`frun up` starts the server before loading speech, so it takes its share of the GPU first (60%
|
|
294
327
|
by default, which fits a 4-bit 7B model next to Whisper and Kokoro on a 24 GB card), restarts it
|
|
295
328
|
if it exits, and stops it with Ctrl+C. GPU memory, context length, tool parsing, extra engine
|
|
@@ -27,6 +27,9 @@ Requires Python 3.11–3.13.
|
|
|
27
27
|
pip install fusion-runtime
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
+
The first install can sit quiet for a few minutes: where there's no ready-made llama.cpp for your
|
|
31
|
+
machine, pip builds it. That's normal.
|
|
32
|
+
|
|
30
33
|
An agent is one file. This is the whole thing:
|
|
31
34
|
|
|
32
35
|
```python
|
|
@@ -123,6 +126,32 @@ calling on), llama-server (`--jinja`) or a hosted API. The in-process
|
|
|
123
126
|
llama.cpp runtime can't, and `frun up` says so at startup. A runnable version is
|
|
124
127
|
[`examples/tools_agent.py`](examples/tools_agent.py).
|
|
125
128
|
|
|
129
|
+
### Answering from your own documents
|
|
130
|
+
|
|
131
|
+
Retrieval is a tool like any other: a function that takes the caller's question and returns a few
|
|
132
|
+
short passages, which the model answers from.
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
@tool(timeout_s=3)
|
|
136
|
+
def search_help(question: str) -> dict:
|
|
137
|
+
"""Search the shop's help articles: returns, refunds, delivery, cancelling orders.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
question: What the caller wants to know, in their words.
|
|
141
|
+
"""
|
|
142
|
+
return {"passages": my_search(question, limit=3)}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
`my_search` is yours: a vector database, Elasticsearch, your help centre's API.
|
|
146
|
+
[`examples/knowledge_agent.py`](examples/knowledge_agent.py) is a runnable version with a small
|
|
147
|
+
keyword search over the Markdown files in [`examples/knowledge/`](examples/knowledge/), in plain
|
|
148
|
+
Python with nothing to install.
|
|
149
|
+
|
|
150
|
+
On a call, two things matter more than in a chat window. A lookup costs a second model round
|
|
151
|
+
before the caller hears anything, so if your knowledge fits in a page or two, put it in the prompt
|
|
152
|
+
instead: that's faster than any search. And passages go into the prompt, so keep them short (a few
|
|
153
|
+
sentences each, a few per search): long ones slow the model's first word.
|
|
154
|
+
|
|
126
155
|
## On your own site
|
|
127
156
|
|
|
128
157
|
```html
|
|
@@ -187,7 +216,7 @@ console), so you can reproduce them rather than trusting ours. Barge-in fired on
|
|
|
187
216
|
## Several callers at once
|
|
188
217
|
|
|
189
218
|
Measured on the same 3090, real WebSocket sessions, three turns each
|
|
190
|
-
(
|
|
219
|
+
(`frun bench`). Response is the server's own
|
|
191
220
|
figure: the turn ends, audio comes back.
|
|
192
221
|
|
|
193
222
|
| Callers | In-process llama.cpp | vLLM | SGLang |
|
|
@@ -233,9 +262,13 @@ fusion where it is:
|
|
|
233
262
|
```bash
|
|
234
263
|
python3 -m venv ~/vllm-env && ~/vllm-env/bin/pip install vllm
|
|
235
264
|
export FUSION_LLM_ENGINE_ENV=~/vllm-env
|
|
265
|
+
frun models pull agent.py # the LLM too (5.6 GB here), where vLLM looks for it
|
|
236
266
|
frun up agent.py
|
|
237
267
|
```
|
|
238
268
|
|
|
269
|
+
Without the pull, vLLM downloads the model during its first start, which takes 5–15 minutes and
|
|
270
|
+
looks like nothing is happening.
|
|
271
|
+
|
|
239
272
|
`frun up` starts the server before loading speech, so it takes its share of the GPU first (60%
|
|
240
273
|
by default, which fits a 4-bit 7B model next to Whisper and Kokoro on a 24 GB card), restarts it
|
|
241
274
|
if it exits, and stops it with Ctrl+C. GPU memory, context length, tool parsing, extra engine
|
|
@@ -220,7 +220,8 @@ class Agent:
|
|
|
220
220
|
vad: Optional[VAD] = None
|
|
221
221
|
tools: Sequence[Any] = () # functions the model can call; plain functions become tools
|
|
222
222
|
greeting: Optional[str] = None # said when a caller connects, before they speak; None = wait for them
|
|
223
|
-
|
|
223
|
+
greeting_interruptible: bool = True # False: always heard in full ("this call may be recorded")
|
|
224
|
+
profile: str = "development" # the defaults everything above is applied to: development (laptops), production (GPU)
|
|
224
225
|
source: Optional[Path] = None # the file it was loaded from, when it came from one
|
|
225
226
|
|
|
226
227
|
def __post_init__(self) -> None:
|
|
@@ -248,10 +249,10 @@ class Agent:
|
|
|
248
249
|
|
|
249
250
|
def config(self, environ: Optional[Mapping[str, str]] = None):
|
|
250
251
|
"""This agent as a PipelineConfig: profile defaults, the agent on top, then the environment."""
|
|
251
|
-
from fusion_runtime.config import PROFILES, with_env_overrides
|
|
252
|
+
from fusion_runtime.config import PROFILES, profile_choices, with_env_overrides
|
|
252
253
|
|
|
253
254
|
if self.profile not in PROFILES:
|
|
254
|
-
raise AgentError(f"unknown profile {self.profile!r}; choose one of: {
|
|
255
|
+
raise AgentError(f"unknown profile {self.profile!r}; choose one of: {profile_choices()}")
|
|
255
256
|
config = PROFILES[self.profile]
|
|
256
257
|
parts: Dict[str, Any] = {}
|
|
257
258
|
|
|
@@ -45,7 +45,7 @@ def bytes_to_download(entry: ModelEntry, root: Path, force: bool = False) -> int
|
|
|
45
45
|
return sum(entry.files[n] for n in names)
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
def check_disk_space(needed: int, root: Path) -> None:
|
|
48
|
+
def check_disk_space(needed: int, root: Path, setting: str = "FUSION_MODEL_DIR") -> None:
|
|
49
49
|
probe = root
|
|
50
50
|
while not probe.exists(): # model dir may not exist yet
|
|
51
51
|
probe = probe.parent
|
|
@@ -54,7 +54,7 @@ def check_disk_space(needed: int, root: Path) -> None:
|
|
|
54
54
|
raise NotEnoughDiskSpace(
|
|
55
55
|
f"Need {format_size(needed)} (plus {format_size(DISK_HEADROOM_BYTES)} headroom) "
|
|
56
56
|
f"but only {format_size(free)} is free at {probe}. "
|
|
57
|
-
"Free up space or set
|
|
57
|
+
f"Free up space or set {setting} to a bigger disk."
|
|
58
58
|
)
|
|
59
59
|
|
|
60
60
|
|
|
@@ -189,16 +189,83 @@ def pull_hf(ref: str, root: Path, log: Callable[[str], None] = print, force: boo
|
|
|
189
189
|
except DownloadError:
|
|
190
190
|
raise
|
|
191
191
|
except Exception as e:
|
|
192
|
-
|
|
193
|
-
if name in ("GatedRepoError", "RepositoryNotFoundError") or "401" in str(e) or "403" in str(e):
|
|
194
|
-
raise ModelAccessDenied(
|
|
195
|
-
f"{repo} is gated or private. Accept its terms on huggingface.co/{repo}, then put an access "
|
|
196
|
-
f"token in ${HF_TOKEN_ENV} (export {HF_TOKEN_ENV}=hf_...)"
|
|
197
|
-
) from e
|
|
198
|
-
raise DownloadError(f"{repo}: download failed: {name}: {e}") from e
|
|
192
|
+
raise _hub_error(repo, e) from e
|
|
199
193
|
return target
|
|
200
194
|
|
|
201
195
|
|
|
196
|
+
def _hub_error(repo: str, e: Exception) -> DownloadError:
|
|
197
|
+
name = type(e).__name__
|
|
198
|
+
if name in ("GatedRepoError", "RepositoryNotFoundError") or "401" in str(e) or "403" in str(e):
|
|
199
|
+
return ModelAccessDenied(
|
|
200
|
+
f"{repo} is gated or private. Accept its terms on huggingface.co/{repo}, then put an access "
|
|
201
|
+
f"token in ${HF_TOKEN_ENV} (export {HF_TOKEN_ENV}=hf_...)")
|
|
202
|
+
return DownloadError(f"{repo}: download failed: {name}: {e}")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
# ---- models vLLM and SGLang load -------------------------------------------------------------
|
|
206
|
+
#
|
|
207
|
+
# A model on vLLM or SGLang is loaded by the engine, from Hugging Face's own cache (HF_HOME), not
|
|
208
|
+
# from fusion's model directory. Left to the engine, a first `frun up` spends 5-15 minutes
|
|
209
|
+
# downloading inside its start-up, looking hung. `frun models pull agent.py` fetches it into that
|
|
210
|
+
# same cache beforehand, with progress, so the engine finds it there.
|
|
211
|
+
|
|
212
|
+
# Weights in other formats the engines don't load (a repo may ship several), and training leftovers
|
|
213
|
+
ENGINE_UNWANTED = HF_UNWANTED + ["*.gguf", "*.onnx", "*.onnx_data", "onnx/**", "*.pt"]
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def engine_files(repo: str, revision: Optional[str] = None, token: Optional[str] = None) -> list:
|
|
217
|
+
"""[(name, size)] of what vLLM or SGLang needs from a repo: weights, configs, tokenizer, chat template."""
|
|
218
|
+
from huggingface_hub import HfApi
|
|
219
|
+
|
|
220
|
+
info = HfApi().repo_info(repo, revision=revision, files_metadata=True, token=token)
|
|
221
|
+
files = [(f.rfilename, f.size or 0) for f in (info.siblings or [])
|
|
222
|
+
if not any(Path(f.rfilename).match(p) for p in ENGINE_UNWANTED)]
|
|
223
|
+
if any(name.endswith(".safetensors") for name, _ in files):
|
|
224
|
+
files = [f for f in files if not f[0].endswith(".bin")] # the same weights again, in the older format
|
|
225
|
+
return files
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def in_engine_cache(repo: str, filename: str = "config.json", revision: Optional[str] = None) -> bool:
|
|
229
|
+
"""Whether Hugging Face's cache holds this file of the repo. Looks on disk only, never online."""
|
|
230
|
+
from huggingface_hub import try_to_load_from_cache
|
|
231
|
+
|
|
232
|
+
return isinstance(try_to_load_from_cache(repo, filename, revision=revision), str)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def engine_cache_dir() -> Path:
|
|
236
|
+
from huggingface_hub import constants
|
|
237
|
+
|
|
238
|
+
return Path(constants.HF_HUB_CACHE)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def pull_for_engine(ref: str, log: Callable[[str], None] = print, force: bool = False) -> int:
|
|
242
|
+
"""Download a vLLM/SGLang model into Hugging Face's cache. Returns the bytes fetched (0: all there)."""
|
|
243
|
+
repo, revision, filename = hf_reference(ref if ref.startswith(HF_PREFIX) else HF_PREFIX + ref)
|
|
244
|
+
if filename:
|
|
245
|
+
raise DownloadError(f"vLLM and SGLang load a whole repo; name it as hf:{repo}, without {filename!r}")
|
|
246
|
+
from huggingface_hub import snapshot_download
|
|
247
|
+
|
|
248
|
+
token = os.getenv(HF_TOKEN_ENV)
|
|
249
|
+
try:
|
|
250
|
+
files = engine_files(repo, revision, token)
|
|
251
|
+
if not files:
|
|
252
|
+
raise DownloadError(f"{repo} has no files vLLM or SGLang can load")
|
|
253
|
+
missing = files if force else [f for f in files if not in_engine_cache(repo, f[0], revision)]
|
|
254
|
+
if not missing:
|
|
255
|
+
return 0
|
|
256
|
+
needed = sum(size for _, size in missing)
|
|
257
|
+
cache = engine_cache_dir()
|
|
258
|
+
check_disk_space(needed, cache, setting="HF_HOME")
|
|
259
|
+
log(f" {repo}{'@' + revision if revision else ''} ({format_size(needed)}) → {cache}, where the engine looks")
|
|
260
|
+
snapshot_download(repo_id=repo, revision=revision, allow_patterns=[name for name, _ in files],
|
|
261
|
+
token=token, force_download=force)
|
|
262
|
+
except DownloadError:
|
|
263
|
+
raise
|
|
264
|
+
except Exception as e:
|
|
265
|
+
raise _hub_error(repo, e) from e
|
|
266
|
+
return needed
|
|
267
|
+
|
|
268
|
+
|
|
202
269
|
def _files_to_fetch(repo: str, revision: Optional[str], filename: Optional[str], token) -> list:
|
|
203
270
|
"""The named file plus the small files every copy needs, or the whole repo when it holds one model."""
|
|
204
271
|
files = hf_files(repo, revision, token)
|
|
@@ -235,7 +235,7 @@ def check_acceleration() -> List[CheckResult]:
|
|
|
235
235
|
if gpu_offload:
|
|
236
236
|
results.append(CheckResult(OK, f"llama.cpp can use the GPU ({backend})"))
|
|
237
237
|
if DEVELOPMENT_CONFIG.llm.n_gpu_layers == 0:
|
|
238
|
-
results.append(CheckResult(INFO, "The development profile
|
|
238
|
+
results.append(CheckResult(INFO, "The development profile is for laptops: it runs the LLM on the CPU (n_gpu_layers=0); production uses the GPU"))
|
|
239
239
|
elif cuda_devices:
|
|
240
240
|
# The GPU image builds llama.cpp for the CPU on purpose: on an NVIDIA card the LLM
|
|
241
241
|
# belongs on a model server, which serves about three times the callers.
|
|
@@ -259,20 +259,27 @@ def check_models() -> List[CheckResult]:
|
|
|
259
259
|
from fusion_runtime.config import model_dir, model_dir_source
|
|
260
260
|
|
|
261
261
|
results = [CheckResult(INFO, f"Model directory: {short_path(model_dir())} ({model_dir_source()})")]
|
|
262
|
+
# Each profile's missing models count only on the machine that profile is for: on a GPU server,
|
|
263
|
+
# the laptop profile's CPU models aren't a problem, and the reverse.
|
|
264
|
+
gpu = bool(_cuda_device_count())
|
|
262
265
|
dev_missing = missing_models(Profile.development, apply_env=False)
|
|
263
266
|
if dev_missing:
|
|
264
|
-
results.append(CheckResult(
|
|
265
|
-
|
|
267
|
+
results.append(CheckResult(
|
|
268
|
+
INFO if gpu else FAIL,
|
|
269
|
+
f"development profile (laptops) is missing {', '.join(dev_missing)}"
|
|
270
|
+
+ (" (only for --config development)" if gpu else ""),
|
|
271
|
+
"frun models pull",
|
|
272
|
+
))
|
|
266
273
|
else:
|
|
267
|
-
results.append(CheckResult(OK, "development profile: all models installed"))
|
|
274
|
+
results.append(CheckResult(OK, "development profile (laptops): all models installed"))
|
|
268
275
|
|
|
269
276
|
prod_missing = missing_models(Profile.production, apply_env=False)
|
|
270
277
|
if not prod_missing:
|
|
271
|
-
results.append(CheckResult(OK, "production profile: all models installed"))
|
|
278
|
+
results.append(CheckResult(OK, "production profile (NVIDIA GPU): all models installed"))
|
|
272
279
|
else:
|
|
273
280
|
results.append(CheckResult(
|
|
274
|
-
WARN if
|
|
275
|
-
f"production profile is missing {', '.join(prod_missing)}",
|
|
281
|
+
WARN if gpu else INFO,
|
|
282
|
+
f"production profile (NVIDIA GPU) is missing {', '.join(prod_missing)}",
|
|
276
283
|
"frun models pull --config production",
|
|
277
284
|
))
|
|
278
285
|
return results
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""The Typer app every `frun` command registers on."""
|
|
2
2
|
import typer
|
|
3
3
|
|
|
4
|
+
from fusion_runtime.cli.bench import bench
|
|
4
5
|
from fusion_runtime.cli.doctor import doctor
|
|
5
6
|
from fusion_runtime.cli.keys import key_app, keys_app, token
|
|
6
7
|
from fusion_runtime.cli.models import models_app
|
|
@@ -49,6 +50,7 @@ def _root(
|
|
|
49
50
|
|
|
50
51
|
app.command()(up)
|
|
51
52
|
app.command()(talk)
|
|
53
|
+
app.command()(bench)
|
|
52
54
|
app.add_typer(key_app, name="key")
|
|
53
55
|
app.add_typer(keys_app, name="keys")
|
|
54
56
|
app.command()(token)
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""`frun bench`: how a running server behaves with several callers at once.
|
|
2
|
+
|
|
3
|
+
Opens N real WebSocket sessions against a running server, streams the same recording down each in
|
|
4
|
+
real time, and reports how long each caller waited for audio to come back, and where that time went.
|
|
5
|
+
|
|
6
|
+
frun up agent.py --host 0.0.0.0 # in one terminal
|
|
7
|
+
frun bench --callers 1,4,8,12 # in another
|
|
8
|
+
|
|
9
|
+
The number that matters is how the median moves from one caller to several: an in-process llama.cpp
|
|
10
|
+
model decodes one reply at a time, so callers queue; on vLLM or SGLang they don't, until speech does.
|
|
11
|
+
`--target-ms` turns that into a capacity: how many callers stayed under it. That's what a quote for a
|
|
12
|
+
customer's agent is based on, measured rather than promised.
|
|
13
|
+
|
|
14
|
+
Audio is streamed in real time on purpose: sending a recording at once would measure something the
|
|
15
|
+
runtime never does, and turn detection watches silence in wall-clock time.
|
|
16
|
+
"""
|
|
17
|
+
import asyncio
|
|
18
|
+
import json
|
|
19
|
+
import statistics
|
|
20
|
+
import time
|
|
21
|
+
import wave
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import List, Optional
|
|
24
|
+
|
|
25
|
+
import typer
|
|
26
|
+
|
|
27
|
+
AUDIO_DIR = Path(__file__).resolve().parent / "bench_audio"
|
|
28
|
+
RECORDINGS = {"hello": "hello.wav", "order": "order_1042.wav"} # order: asks about order 1042, for tool agents
|
|
29
|
+
CHUNK_MS = 20
|
|
30
|
+
# Where each reply's time went, from the server's own turn summary: which stage starts to queue as
|
|
31
|
+
# callers are added is the finding, not just that it's slower.
|
|
32
|
+
STAGES = {"stt_transcribe_ms": "stt", "llm_queue_ms": "llm queue", "llm_first_token_ms": "llm first token",
|
|
33
|
+
"tool_ms": "tools", "tts_first_chunk_ms": "tts first audio"}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load_audio(path: Path) -> bytes:
|
|
37
|
+
with wave.open(str(path), "rb") as wav:
|
|
38
|
+
if wav.getframerate() != 16000 or wav.getnchannels() != 1 or wav.getsampwidth() != 2:
|
|
39
|
+
raise typer.BadParameter(f"{path} must be a 16 kHz mono 16-bit WAV")
|
|
40
|
+
return wav.readframes(wav.getnframes())
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def audio_path(name: str) -> Path:
|
|
44
|
+
if name in RECORDINGS:
|
|
45
|
+
return AUDIO_DIR / RECORDINGS[name]
|
|
46
|
+
path = Path(name).expanduser()
|
|
47
|
+
if not path.is_file():
|
|
48
|
+
raise typer.BadParameter(f"no recording {name!r}: use {', '.join(RECORDINGS)} or a path to a WAV")
|
|
49
|
+
return path
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def resolve_key(url: str, given: Optional[str]) -> str:
|
|
53
|
+
"""The key to present, the way `frun talk` works it out.
|
|
54
|
+
|
|
55
|
+
FUSION_API_KEY if it's set; otherwise, for a server on this machine only, the first key that
|
|
56
|
+
server accepts. Never falls back for a remote URL: that would send your server's keys elsewhere.
|
|
57
|
+
"""
|
|
58
|
+
if given:
|
|
59
|
+
return given
|
|
60
|
+
from fusion_runtime.env import load_env_file
|
|
61
|
+
from fusion_runtime.security.keys import client_key
|
|
62
|
+
|
|
63
|
+
load_env_file()
|
|
64
|
+
http_url = url.replace("ws://", "http://").replace("wss://", "https://")
|
|
65
|
+
return client_key(http_url) or ""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
async def one_session(index: int, url: str, key: str, audio: bytes, turns: int, results: list) -> None:
|
|
69
|
+
"""One caller: connect, then take `turns` turns, timing each reply."""
|
|
70
|
+
import websockets
|
|
71
|
+
|
|
72
|
+
headers = {"Authorization": f"Bearer {key}"} if key else {}
|
|
73
|
+
try: # websockets renamed this parameter; support both without pinning a version
|
|
74
|
+
ws = await websockets.connect(url, additional_headers=headers)
|
|
75
|
+
except TypeError:
|
|
76
|
+
ws = await websockets.connect(url, extra_headers=headers)
|
|
77
|
+
|
|
78
|
+
step = 16000 * 2 // 1000 * CHUNK_MS
|
|
79
|
+
try:
|
|
80
|
+
first = json.loads(await ws.recv()) # the config message, or why the call was refused
|
|
81
|
+
if first.get("type") == "error":
|
|
82
|
+
results.append({"caller": index, "turn": -1,
|
|
83
|
+
"error": first.get("message") + (f" ({first['fix']})" if first.get("fix") else "")})
|
|
84
|
+
return
|
|
85
|
+
# A real microphone never goes digitally silent, and fusion tells a caller sending exact
|
|
86
|
+
# zeros that their mic is muted, so the quiet between turns is faint room noise.
|
|
87
|
+
quiet = b"\x01\x00" * (step // 2)
|
|
88
|
+
|
|
89
|
+
# The first turn allocates caches and isn't representative: taken and thrown away.
|
|
90
|
+
for turn in range(-1, turns):
|
|
91
|
+
for i in range(0, len(audio), step):
|
|
92
|
+
await ws.send(audio[i:i + step])
|
|
93
|
+
await asyncio.sleep(CHUNK_MS / 1000)
|
|
94
|
+
stopped_talking_at = time.perf_counter()
|
|
95
|
+
|
|
96
|
+
first_audio_at = None
|
|
97
|
+
summary: dict = {}
|
|
98
|
+
deadline = stopped_talking_at + 60
|
|
99
|
+
|
|
100
|
+
async def keep_the_mic_open(deadline=deadline):
|
|
101
|
+
# The turn detector ends a turn by hearing silence; stopping the stream would leave
|
|
102
|
+
# the server waiting on a timeout and put that wait into every measurement.
|
|
103
|
+
while time.perf_counter() < deadline:
|
|
104
|
+
await ws.send(quiet)
|
|
105
|
+
await asyncio.sleep(CHUNK_MS / 1000)
|
|
106
|
+
|
|
107
|
+
mic = asyncio.create_task(keep_the_mic_open())
|
|
108
|
+
try:
|
|
109
|
+
while time.perf_counter() < deadline:
|
|
110
|
+
try:
|
|
111
|
+
message = await asyncio.wait_for(ws.recv(), timeout=max(0.1, deadline - time.perf_counter()))
|
|
112
|
+
except asyncio.TimeoutError:
|
|
113
|
+
break
|
|
114
|
+
if isinstance(message, bytes):
|
|
115
|
+
if first_audio_at is None:
|
|
116
|
+
first_audio_at = time.perf_counter()
|
|
117
|
+
continue
|
|
118
|
+
event = json.loads(message)
|
|
119
|
+
if event.get("type") == "error":
|
|
120
|
+
results.append({"caller": index, "turn": turn, "error": event.get("message")})
|
|
121
|
+
return
|
|
122
|
+
if event.get("type") == "turn.trace":
|
|
123
|
+
summary = event.get("summary") or {}
|
|
124
|
+
break # the turn is over; the server said so
|
|
125
|
+
finally:
|
|
126
|
+
mic.cancel()
|
|
127
|
+
|
|
128
|
+
if turn < 0:
|
|
129
|
+
continue # the warm-up turn
|
|
130
|
+
if first_audio_at is None:
|
|
131
|
+
results.append({"caller": index, "turn": turn, "error": "no audio came back"})
|
|
132
|
+
continue
|
|
133
|
+
results.append({
|
|
134
|
+
"caller": index,
|
|
135
|
+
"turn": turn,
|
|
136
|
+
"wall_ms": round((first_audio_at - stopped_talking_at) * 1000),
|
|
137
|
+
"server_ms": round(summary["response_ms"]) if summary.get("response_ms") else None,
|
|
138
|
+
**{stage: summary.get(stage) for stage in STAGES},
|
|
139
|
+
"tool_calls": summary.get("tool_calls") or 0,
|
|
140
|
+
})
|
|
141
|
+
finally:
|
|
142
|
+
await ws.close()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
async def run_round(callers: int, url: str, key: str, audio: bytes, turns: int) -> list:
|
|
146
|
+
results: list = []
|
|
147
|
+
await asyncio.gather(*[_counted(i, url, key, audio, turns, results) for i in range(callers)])
|
|
148
|
+
return results
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
async def _counted(index: int, url: str, key: str, audio: bytes, turns: int, results: list) -> None:
|
|
152
|
+
"""One caller; a call the server drops is a failed turn in the round, not the end of the run."""
|
|
153
|
+
from websockets.exceptions import ConnectionClosed
|
|
154
|
+
|
|
155
|
+
try:
|
|
156
|
+
await one_session(index, url, key, audio, turns, results)
|
|
157
|
+
except ConnectionClosed as e:
|
|
158
|
+
why = getattr(e.rcvd, "reason", "") or f"code {getattr(e.rcvd, 'code', '?')}"
|
|
159
|
+
results.append({"caller": index, "error": f"the server closed the call ({why}); see its log"})
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def summarize(callers: int, results: list, seconds: float) -> dict:
|
|
163
|
+
"""One round as numbers: what `report` prints and `--json` saves."""
|
|
164
|
+
ok = [r for r in results if "wall_ms" in r]
|
|
165
|
+
errors = [r for r in results if "error" in r]
|
|
166
|
+
row: dict = {"callers": callers, "turns": len(ok), "seconds": round(seconds, 1), "failed": len(errors),
|
|
167
|
+
"error": errors[0].get("error") if errors else None}
|
|
168
|
+
if ok:
|
|
169
|
+
walls = sorted(r["wall_ms"] for r in ok)
|
|
170
|
+
row.update(median_ms=round(statistics.median(walls)), worst_ms=walls[-1])
|
|
171
|
+
server = [r["server_ms"] for r in ok if r["server_ms"]]
|
|
172
|
+
row["server_median_ms"] = round(statistics.median(server)) if server else None
|
|
173
|
+
row["stages_median_ms"] = {label: round(statistics.median(values)) for stage, label in STAGES.items()
|
|
174
|
+
if (values := [r[stage] for r in ok if r.get(stage) is not None])}
|
|
175
|
+
row["tool_calls"] = sum(r["tool_calls"] for r in ok)
|
|
176
|
+
return row
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def report(row: dict, baseline: Optional[float]) -> str:
|
|
180
|
+
if "median_ms" not in row:
|
|
181
|
+
return f" {row['callers']:>2} caller(s): nothing completed. {row['error'] or ''}".rstrip()
|
|
182
|
+
line = (f" {row['callers']:>2} caller(s): median {row['median_ms']:>5} ms worst {row['worst_ms']:>5} ms "
|
|
183
|
+
f"{row['turns']} turns in {row['seconds']}s")
|
|
184
|
+
if row["server_median_ms"]:
|
|
185
|
+
line += f" (server said {row['server_median_ms']} ms)"
|
|
186
|
+
if baseline:
|
|
187
|
+
line += f" {row['median_ms'] / baseline:.1f}x of one caller"
|
|
188
|
+
lines = [line]
|
|
189
|
+
if row["stages_median_ms"]:
|
|
190
|
+
lines.append(" median ms: " + ", ".join(f"{k} {v}" for k, v in row["stages_median_ms"].items()))
|
|
191
|
+
if row["tool_calls"]:
|
|
192
|
+
lines.append(f" {row['tool_calls']} tool call(s) in {row['turns']} turns")
|
|
193
|
+
if row["failed"]:
|
|
194
|
+
lines.append(f" {row['failed']} turn(s) failed: {row['error']}")
|
|
195
|
+
return "\n".join(lines)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def bottleneck(rows: List[dict]) -> str:
|
|
199
|
+
"""Which stage grew most from the fewest callers to the most: where the next caller waits."""
|
|
200
|
+
measured = [r for r in sorted(rows, key=lambda r: r["callers"]) if r.get("stages_median_ms")]
|
|
201
|
+
if len(measured) < 2 or measured[0]["callers"] == measured[-1]["callers"]:
|
|
202
|
+
return "Run more than one round (--callers 1,4,8) to see which stage queues as callers are added."
|
|
203
|
+
first, last = measured[0]["stages_median_ms"], measured[-1]["stages_median_ms"]
|
|
204
|
+
growth = {stage: last[stage] - first.get(stage, 0) for stage in last}
|
|
205
|
+
stage = max(growth, key=growth.get)
|
|
206
|
+
span = f"from {measured[0]['callers']} to {measured[-1]['callers']} caller(s)"
|
|
207
|
+
if growth[stage] < 100:
|
|
208
|
+
return f"No stage grew by more than 100 ms {span}: nothing is queueing yet."
|
|
209
|
+
llm = {k: v for k, v in last.items() if k.startswith("llm")}
|
|
210
|
+
if stage.startswith("llm"):
|
|
211
|
+
return (f"The LLM is where callers wait: {stage} went from {first.get(stage, 0)} to {last[stage]} ms {span}.\n"
|
|
212
|
+
"An in-process model answers one caller at a time; put it on vLLM or SGLang (frun up starts them).")
|
|
213
|
+
held = ", ".join(f"{k} {v} ms" for k, v in llm.items())
|
|
214
|
+
return (f"Speech is where callers wait: {stage} went from {first.get(stage, 0)} to {last[stage]} ms {span}"
|
|
215
|
+
+ (f", while the LLM held ({held})" if held else "") + ".\n"
|
|
216
|
+
"Whisper and Kokoro handle one caller at a time on each server; more callers need more servers.")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def capacity(rows: List[dict], target_ms: float) -> str:
|
|
220
|
+
"""The most callers whose median stayed under the target, with every round below it clean too."""
|
|
221
|
+
best = None
|
|
222
|
+
for row in sorted(rows, key=lambda r: r["callers"]):
|
|
223
|
+
if row.get("median_ms") is None or row["failed"] or row["median_ms"] > target_ms:
|
|
224
|
+
break
|
|
225
|
+
best = row
|
|
226
|
+
if best is None:
|
|
227
|
+
return f"No round stayed under {target_ms:.0f} ms (median) without failures."
|
|
228
|
+
return (f"Up to {best['callers']} caller(s) stayed under {target_ms:.0f} ms (median {best['median_ms']} ms, "
|
|
229
|
+
f"worst {best['worst_ms']} ms) on this setup.")
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
async def _bench(url: str, key: str, audio: bytes, callers: List[int], turns: int, label: str,
|
|
233
|
+
target_ms: Optional[float], json_out: Optional[Path]) -> None:
|
|
234
|
+
ws_url = url.rstrip("/") + "/v1/voice/ws"
|
|
235
|
+
typer.echo(f"{label or ws_url} — {len(audio) / 2 / 16000:.2f}s of caller audio, {turns} turn(s) each\n")
|
|
236
|
+
rows, baseline = [], None
|
|
237
|
+
for count in callers:
|
|
238
|
+
started = time.perf_counter()
|
|
239
|
+
results = await run_round(count, ws_url, key, audio, turns)
|
|
240
|
+
row = summarize(count, results, time.perf_counter() - started)
|
|
241
|
+
rows.append(row)
|
|
242
|
+
typer.echo(report(row, baseline))
|
|
243
|
+
if count == 1 and row.get("median_ms"):
|
|
244
|
+
baseline = row["median_ms"]
|
|
245
|
+
await asyncio.sleep(2) # let sessions close before the next round claims slots
|
|
246
|
+
typer.echo("")
|
|
247
|
+
typer.echo(bottleneck(rows))
|
|
248
|
+
typer.echo(capacity(rows, target_ms) if target_ms else "--target-ms 1500 turns the rounds into a capacity.")
|
|
249
|
+
if json_out:
|
|
250
|
+
json_out.write_text(json.dumps({"label": label or ws_url, "turns_per_caller": turns,
|
|
251
|
+
"target_ms": target_ms, "rounds": rows}, indent=2) + "\n")
|
|
252
|
+
typer.echo(f"Saved: {json_out}")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def bench(
|
|
256
|
+
url: str = typer.Option("ws://127.0.0.1:8000", "--url", help="The running server."),
|
|
257
|
+
key: str = typer.Option(None, "--key", help="An accepted key. For a server on this machine it's found for you."),
|
|
258
|
+
callers: str = typer.Option("1,2,4", "--callers", help="How many callers at once, per round, comma separated."),
|
|
259
|
+
turns: int = typer.Option(3, "--turns", min=1, help="Turns per caller, after one warm-up turn."),
|
|
260
|
+
audio: str = typer.Option("hello", "--audio",
|
|
261
|
+
help="What callers say: hello, order (asks about order 1042, for tool agents), "
|
|
262
|
+
"or a 16 kHz mono 16-bit WAV."),
|
|
263
|
+
target_ms: float = typer.Option(None, "--target-ms", min=1,
|
|
264
|
+
help="Report how many callers stayed under this median wait (e.g. 1500)."),
|
|
265
|
+
json_out: Path = typer.Option(None, "--json", help="Also save every round's numbers to this file."),
|
|
266
|
+
label: str = typer.Option("", "--label", help="A heading for the run, e.g. 'RTX 4090, vLLM'."),
|
|
267
|
+
) -> None:
|
|
268
|
+
"""Measure a running server with several callers at once: wait per caller, and where it went."""
|
|
269
|
+
try:
|
|
270
|
+
counts = [int(n) for n in callers.split(",") if n.strip()]
|
|
271
|
+
except ValueError:
|
|
272
|
+
raise typer.BadParameter(f"--callers takes numbers separated by commas, like 1,4,8; got {callers!r}") from None
|
|
273
|
+
if not counts or min(counts) < 1:
|
|
274
|
+
raise typer.BadParameter("--callers needs at least one number, each 1 or more")
|
|
275
|
+
sound = load_audio(audio_path(audio))
|
|
276
|
+
try:
|
|
277
|
+
asyncio.run(_bench(url, resolve_key(url, key), sound, counts, turns, label, target_ms, json_out))
|
|
278
|
+
except OSError as e:
|
|
279
|
+
typer.echo(f"Error: can't reach {url} ({e}). Is the server running? Start it with: frun up", err=True)
|
|
280
|
+
raise typer.Exit(1)
|
|
Binary file
|
|
Binary file
|
|
@@ -90,6 +90,9 @@ def token(
|
|
|
90
90
|
from fusion_runtime.config import API_KEY_ENV
|
|
91
91
|
from fusion_runtime.security import client_key
|
|
92
92
|
|
|
93
|
+
if public_url and not public_url.startswith(("http://", "https://")): # before a token is spent on it
|
|
94
|
+
typer.echo(f"Error: --public-url must start with https:// (or http://), got {public_url!r}", err=True)
|
|
95
|
+
raise typer.Exit(1)
|
|
93
96
|
key = key or client_key(url)
|
|
94
97
|
if not key:
|
|
95
98
|
typer.echo(f"Error: no key given. Pass --key, or set {API_KEY_ENV} (a .env file works).\n"
|
|
@@ -109,11 +112,6 @@ def token(
|
|
|
109
112
|
raise typer.Exit(1)
|
|
110
113
|
response.raise_for_status()
|
|
111
114
|
body = response.json()
|
|
112
|
-
shown = base
|
|
113
|
-
if public_url:
|
|
114
|
-
if not public_url.startswith(("http://", "https://")):
|
|
115
|
-
typer.echo(f"Error: --public-url must start with https:// (or http://), got {public_url!r}", err=True)
|
|
116
|
-
raise typer.Exit(1)
|
|
117
|
-
shown = public_url.rstrip("/")
|
|
115
|
+
shown = public_url.rstrip("/") if public_url else base
|
|
118
116
|
typer.echo(f"{shown}/?token={body['token']}")
|
|
119
117
|
typer.echo(f"\nOpen that in a browser within {body['expires_in']} seconds. It works once.")
|