fusion-runtime 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. {fusion_runtime-0.1.2/fusion_runtime.egg-info → fusion_runtime-0.1.3}/PKG-INFO +35 -2
  2. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/README.md +34 -1
  3. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/agent.py +4 -3
  4. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/download.py +76 -9
  5. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/_checks.py +14 -7
  6. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/app.py +2 -0
  7. fusion_runtime-0.1.3/fusion_runtime/cli/bench.py +280 -0
  8. fusion_runtime-0.1.3/fusion_runtime/cli/bench_audio/hello.wav +0 -0
  9. fusion_runtime-0.1.3/fusion_runtime/cli/bench_audio/order_1042.wav +0 -0
  10. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/keys.py +4 -6
  11. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/models.py +34 -8
  12. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/up.py +133 -24
  13. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/config.py +26 -2
  14. fusion_runtime-0.1.3/fusion_runtime/engine/audio_format.py +84 -0
  15. fusion_runtime-0.1.3/fusion_runtime/engine/audio_watch.py +73 -0
  16. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/barge_in.py +13 -0
  17. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/orchestrator.py +101 -24
  18. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/llm_server.py +148 -18
  19. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/resolver.py +3 -0
  20. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/onnx/kokoro.py +19 -6
  21. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/onnx/tts.py +4 -1
  22. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/openai_http/llm.py +44 -1
  23. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/limits.py +16 -6
  24. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/server.py +117 -23
  25. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/metrics.py +6 -0
  26. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/trace.py +10 -0
  27. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/web/fusion-runtime.js +1 -1
  28. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/web/index.html +18 -3
  29. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3/fusion_runtime.egg-info}/PKG-INFO +35 -2
  30. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/SOURCES.txt +12 -0
  31. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/pyproject.toml +2 -2
  32. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_agent.py +1 -1
  33. fusion_runtime-0.1.3/tests/test_audio_format.py +91 -0
  34. fusion_runtime-0.1.3/tests/test_audio_watch.py +114 -0
  35. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_banner.py +27 -0
  36. fusion_runtime-0.1.3/tests/test_bench.py +83 -0
  37. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_cli.py +40 -14
  38. fusion_runtime-0.1.3/tests/test_degraded.py +96 -0
  39. fusion_runtime-0.1.3/tests/test_digital_silence.py +61 -0
  40. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_doctor.py +7 -0
  41. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_http_llm.py +70 -3
  42. fusion_runtime-0.1.3/tests/test_interruption_switches.py +81 -0
  43. fusion_runtime-0.1.3/tests/test_knowledge_example.py +48 -0
  44. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_llm_server.py +130 -1
  45. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_runtimes.py +9 -0
  46. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_security.py +27 -0
  47. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_server_telemetry.py +13 -0
  48. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_telemetry.py +14 -0
  49. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_tools.py +41 -0
  50. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/LICENSE +0 -0
  51. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/NOTICE +0 -0
  52. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/__init__.py +0 -0
  53. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/audio/__init__.py +0 -0
  54. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/audio/duplex_audio.py +0 -0
  55. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/audio/echo_canceller.py +0 -0
  56. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/__init__.py +0 -0
  57. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/entries.py +0 -0
  58. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/gguf.py +0 -0
  59. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/catalog/models.toml +0 -0
  60. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/__init__.py +0 -0
  61. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/__main__.py +0 -0
  62. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/_common.py +0 -0
  63. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/_talk_client.py +0 -0
  64. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/banner.py +0 -0
  65. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/doctor.py +0 -0
  66. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/talk.py +0 -0
  67. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/cli/version.py +0 -0
  68. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/__init__.py +0 -0
  69. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/common.py +0 -0
  70. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/llm.py +0 -0
  71. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/stt.py +0 -0
  72. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/tts.py +0 -0
  73. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/contract/turn.py +0 -0
  74. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/__init__.py +0 -0
  75. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/conversation.py +0 -0
  76. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/scheduler.py +0 -0
  77. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/streaming.py +0 -0
  78. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/engine/text.py +0 -0
  79. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/env.py +0 -0
  80. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/registry.py +0 -0
  81. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/__init__.py +0 -0
  82. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/ctranslate2/__init__.py +0 -0
  83. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/ctranslate2/stt.py +0 -0
  84. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/llama_cpp/__init__.py +0 -0
  85. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/llama_cpp/llm.py +0 -0
  86. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/onnx/__init__.py +0 -0
  87. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/runtimes/openai_http/__init__.py +0 -0
  88. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/__init__.py +0 -0
  89. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/auth.py +0 -0
  90. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/keys.py +0 -0
  91. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/origins.py +0 -0
  92. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/security/tokens.py +0 -0
  93. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/__init__.py +0 -0
  94. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/errors.py +0 -0
  95. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/events.py +0 -0
  96. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/hub.py +0 -0
  97. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/loop_monitor.py +0 -0
  98. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/redact.py +0 -0
  99. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/telemetry/sinks.py +0 -0
  100. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/testing/__init__.py +0 -0
  101. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/testing/conformance.py +0 -0
  102. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/testing/fakes.py +0 -0
  103. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/tools.py +0 -0
  104. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/turns/__init__.py +0 -0
  105. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/turns/silence.py +0 -0
  106. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/__init__.py +0 -0
  107. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/base.py +0 -0
  108. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/silero.py +0 -0
  109. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/vad/turn.py +0 -0
  110. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime/web/__init__.py +0 -0
  111. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/dependency_links.txt +0 -0
  112. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/entry_points.txt +0 -0
  113. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/requires.txt +0 -0
  114. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/fusion_runtime.egg-info/top_level.txt +0 -0
  115. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/setup.cfg +0 -0
  116. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_barge_in.py +0 -0
  117. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_catalog.py +0 -0
  118. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_client_barge_in.py +0 -0
  119. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_conformance.py +0 -0
  120. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_contract.py +0 -0
  121. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_conversation.py +0 -0
  122. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_core.py +0 -0
  123. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_duplex_audio.py +0 -0
  124. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_echo_canceller.py +0 -0
  125. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_engine_streaming.py +0 -0
  126. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_env_file.py +0 -0
  127. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_event_loop_blocking.py +0 -0
  128. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_greeting.py +0 -0
  129. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_languages.py +0 -0
  130. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_llama_cpp_stream.py +0 -0
  131. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_model_paths.py +0 -0
  132. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_registry.py +0 -0
  133. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_resolver.py +0 -0
  134. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_scheduler.py +0 -0
  135. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_self_echo.py +0 -0
  136. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_turn_taking.py +0 -0
  137. {fusion_runtime-0.1.2 → fusion_runtime-0.1.3}/tests/test_web.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fusion-runtime
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Self-hosted voice agent runtime: speech-to-text, LLM and text-to-speech on your own GPU, with tools
5
5
  Author-email: Samarth Urs <hello@fusion-runtime.dev>
6
6
  License-Expression: Apache-2.0
@@ -81,6 +81,9 @@ Requires Python 3.11–3.13.
81
81
  pip install fusion-runtime
82
82
  ```
83
83
 
84
+ The first install can sit quiet for a few minutes: where there's no ready-made llama.cpp for your
85
+ machine, pip builds it. That's normal.
86
+
84
87
  An agent is one file. This is the whole thing:
85
88
 
86
89
  ```python
@@ -177,6 +180,32 @@ calling on), llama-server (`--jinja`) or a hosted API. The in-process
177
180
  llama.cpp runtime can't, and `frun up` says so at startup. A runnable version is
178
181
  [`examples/tools_agent.py`](examples/tools_agent.py).
179
182
 
183
+ ### Answering from your own documents
184
+
185
+ Retrieval is a tool like any other: a function that takes the caller's question and returns a few
186
+ short passages, which the model answers from.
187
+
188
+ ```python
189
+ @tool(timeout_s=3)
190
+ def search_help(question: str) -> dict:
191
+ """Search the shop's help articles: returns, refunds, delivery, cancelling orders.
192
+
193
+ Args:
194
+ question: What the caller wants to know, in their words.
195
+ """
196
+ return {"passages": my_search(question, limit=3)}
197
+ ```
198
+
199
+ `my_search` is yours: a vector database, Elasticsearch, your help centre's API.
200
+ [`examples/knowledge_agent.py`](examples/knowledge_agent.py) is a runnable version with a small
201
+ keyword search over the Markdown files in [`examples/knowledge/`](examples/knowledge/), in plain
202
+ Python with nothing to install.
203
+
204
+ On a call, two things matter more than in a chat window. A lookup costs a second model round
205
+ before the caller hears anything, so if your knowledge fits in a page or two, put it in the prompt
206
+ instead: that's faster than any search. And passages go into the prompt, so keep them short (a few
207
+ sentences each, a few per search): long ones slow the model's first word.
208
+
180
209
  ## On your own site
181
210
 
182
211
  ```html
@@ -241,7 +270,7 @@ console), so you can reproduce them rather than trusting ours. Barge-in fired on
241
270
  ## Several callers at once
242
271
 
243
272
  Measured on the same 3090, real WebSocket sessions, three turns each
244
- ([`scripts/concurrency_check.py`](scripts/concurrency_check.py)). Response is the server's own
273
+ (`frun bench`). Response is the server's own
245
274
  figure: the turn ends, audio comes back.
246
275
 
247
276
  | Callers | In-process llama.cpp | vLLM | SGLang |
@@ -287,9 +316,13 @@ fusion where it is:
287
316
  ```bash
288
317
  python3 -m venv ~/vllm-env && ~/vllm-env/bin/pip install vllm
289
318
  export FUSION_LLM_ENGINE_ENV=~/vllm-env
319
+ frun models pull agent.py # the LLM too (5.6 GB here), where vLLM looks for it
290
320
  frun up agent.py
291
321
  ```
292
322
 
323
+ Without the pull, vLLM downloads the model during its first start, which takes 5–15 minutes and
324
+ looks like nothing is happening.
325
+
293
326
  `frun up` starts the server before loading speech, so it takes its share of the GPU first (60%
294
327
  by default, which fits a 4-bit 7B model next to Whisper and Kokoro on a 24 GB card), restarts it
295
328
  if it exits, and stops it with Ctrl+C. GPU memory, context length, tool parsing, extra engine
@@ -27,6 +27,9 @@ Requires Python 3.11–3.13.
27
27
  pip install fusion-runtime
28
28
  ```
29
29
 
30
+ The first install can sit quiet for a few minutes: where there's no ready-made llama.cpp for your
31
+ machine, pip builds it. That's normal.
32
+
30
33
  An agent is one file. This is the whole thing:
31
34
 
32
35
  ```python
@@ -123,6 +126,32 @@ calling on), llama-server (`--jinja`) or a hosted API. The in-process
123
126
  llama.cpp runtime can't, and `frun up` says so at startup. A runnable version is
124
127
  [`examples/tools_agent.py`](examples/tools_agent.py).
125
128
 
129
+ ### Answering from your own documents
130
+
131
+ Retrieval is a tool like any other: a function that takes the caller's question and returns a few
132
+ short passages, which the model answers from.
133
+
134
+ ```python
135
+ @tool(timeout_s=3)
136
+ def search_help(question: str) -> dict:
137
+ """Search the shop's help articles: returns, refunds, delivery, cancelling orders.
138
+
139
+ Args:
140
+ question: What the caller wants to know, in their words.
141
+ """
142
+ return {"passages": my_search(question, limit=3)}
143
+ ```
144
+
145
+ `my_search` is yours: a vector database, Elasticsearch, your help centre's API.
146
+ [`examples/knowledge_agent.py`](examples/knowledge_agent.py) is a runnable version with a small
147
+ keyword search over the Markdown files in [`examples/knowledge/`](examples/knowledge/), in plain
148
+ Python with nothing to install.
149
+
150
+ On a call, two things matter more than in a chat window. A lookup costs a second model round
151
+ before the caller hears anything, so if your knowledge fits in a page or two, put it in the prompt
152
+ instead: that's faster than any search. And passages go into the prompt, so keep them short (a few
153
+ sentences each, a few per search): long ones slow the model's first word.
154
+
126
155
  ## On your own site
127
156
 
128
157
  ```html
@@ -187,7 +216,7 @@ console), so you can reproduce them rather than trusting ours. Barge-in fired on
187
216
  ## Several callers at once
188
217
 
189
218
  Measured on the same 3090, real WebSocket sessions, three turns each
190
- ([`scripts/concurrency_check.py`](scripts/concurrency_check.py)). Response is the server's own
219
+ (`frun bench`). Response is the server's own
191
220
  figure: the turn ends, audio comes back.
192
221
 
193
222
  | Callers | In-process llama.cpp | vLLM | SGLang |
@@ -233,9 +262,13 @@ fusion where it is:
233
262
  ```bash
234
263
  python3 -m venv ~/vllm-env && ~/vllm-env/bin/pip install vllm
235
264
  export FUSION_LLM_ENGINE_ENV=~/vllm-env
265
+ frun models pull agent.py # the LLM too (5.6 GB here), where vLLM looks for it
236
266
  frun up agent.py
237
267
  ```
238
268
 
269
+ Without the pull, vLLM downloads the model during its first start, which takes 5–15 minutes and
270
+ looks like nothing is happening.
271
+
239
272
  `frun up` starts the server before loading speech, so it takes its share of the GPU first (60%
240
273
  by default, which fits a 4-bit 7B model next to Whisper and Kokoro on a 24 GB card), restarts it
241
274
  if it exits, and stops it with Ctrl+C. GPU memory, context length, tool parsing, extra engine
@@ -220,7 +220,8 @@ class Agent:
220
220
  vad: Optional[VAD] = None
221
221
  tools: Sequence[Any] = () # functions the model can call; plain functions become tools
222
222
  greeting: Optional[str] = None # said when a caller connects, before they speak; None = wait for them
223
- profile: str = "development" # the defaults everything above is applied to
223
+ greeting_interruptible: bool = True # False: always heard in full ("this call may be recorded")
224
+ profile: str = "development" # the defaults everything above is applied to: development (laptops), production (GPU)
224
225
  source: Optional[Path] = None # the file it was loaded from, when it came from one
225
226
 
226
227
  def __post_init__(self) -> None:
@@ -248,10 +249,10 @@ class Agent:
248
249
 
249
250
  def config(self, environ: Optional[Mapping[str, str]] = None):
250
251
  """This agent as a PipelineConfig: profile defaults, the agent on top, then the environment."""
251
- from fusion_runtime.config import PROFILES, with_env_overrides
252
+ from fusion_runtime.config import PROFILES, profile_choices, with_env_overrides
252
253
 
253
254
  if self.profile not in PROFILES:
254
- raise AgentError(f"unknown profile {self.profile!r}; choose one of: {', '.join(PROFILES)}")
255
+ raise AgentError(f"unknown profile {self.profile!r}; choose one of: {profile_choices()}")
255
256
  config = PROFILES[self.profile]
256
257
  parts: Dict[str, Any] = {}
257
258
 
@@ -45,7 +45,7 @@ def bytes_to_download(entry: ModelEntry, root: Path, force: bool = False) -> int
45
45
  return sum(entry.files[n] for n in names)
46
46
 
47
47
 
48
- def check_disk_space(needed: int, root: Path) -> None:
48
+ def check_disk_space(needed: int, root: Path, setting: str = "FUSION_MODEL_DIR") -> None:
49
49
  probe = root
50
50
  while not probe.exists(): # model dir may not exist yet
51
51
  probe = probe.parent
@@ -54,7 +54,7 @@ def check_disk_space(needed: int, root: Path) -> None:
54
54
  raise NotEnoughDiskSpace(
55
55
  f"Need {format_size(needed)} (plus {format_size(DISK_HEADROOM_BYTES)} headroom) "
56
56
  f"but only {format_size(free)} is free at {probe}. "
57
- "Free up space or set FUSION_MODEL_DIR to a bigger disk."
57
+ f"Free up space or set {setting} to a bigger disk."
58
58
  )
59
59
 
60
60
 
@@ -189,16 +189,83 @@ def pull_hf(ref: str, root: Path, log: Callable[[str], None] = print, force: boo
189
189
  except DownloadError:
190
190
  raise
191
191
  except Exception as e:
192
- name = type(e).__name__
193
- if name in ("GatedRepoError", "RepositoryNotFoundError") or "401" in str(e) or "403" in str(e):
194
- raise ModelAccessDenied(
195
- f"{repo} is gated or private. Accept its terms on huggingface.co/{repo}, then put an access "
196
- f"token in ${HF_TOKEN_ENV} (export {HF_TOKEN_ENV}=hf_...)"
197
- ) from e
198
- raise DownloadError(f"{repo}: download failed: {name}: {e}") from e
192
+ raise _hub_error(repo, e) from e
199
193
  return target
200
194
 
201
195
 
196
+ def _hub_error(repo: str, e: Exception) -> DownloadError:
197
+ name = type(e).__name__
198
+ if name in ("GatedRepoError", "RepositoryNotFoundError") or "401" in str(e) or "403" in str(e):
199
+ return ModelAccessDenied(
200
+ f"{repo} is gated or private. Accept its terms on huggingface.co/{repo}, then put an access "
201
+ f"token in ${HF_TOKEN_ENV} (export {HF_TOKEN_ENV}=hf_...)")
202
+ return DownloadError(f"{repo}: download failed: {name}: {e}")
203
+
204
+
205
+ # ---- models vLLM and SGLang load -------------------------------------------------------------
206
+ #
207
+ # A model on vLLM or SGLang is loaded by the engine, from Hugging Face's own cache (HF_HOME), not
208
+ # from fusion's model directory. Left to the engine, a first `frun up` spends 5-15 minutes
209
+ # downloading inside its start-up, looking hung. `frun models pull agent.py` fetches it into that
210
+ # same cache beforehand, with progress, so the engine finds it there.
211
+
212
+ # Weights in other formats the engines don't load (a repo may ship several), and training leftovers
213
+ ENGINE_UNWANTED = HF_UNWANTED + ["*.gguf", "*.onnx", "*.onnx_data", "onnx/**", "*.pt"]
214
+
215
+
216
+ def engine_files(repo: str, revision: Optional[str] = None, token: Optional[str] = None) -> list:
217
+ """[(name, size)] of what vLLM or SGLang needs from a repo: weights, configs, tokenizer, chat template."""
218
+ from huggingface_hub import HfApi
219
+
220
+ info = HfApi().repo_info(repo, revision=revision, files_metadata=True, token=token)
221
+ files = [(f.rfilename, f.size or 0) for f in (info.siblings or [])
222
+ if not any(Path(f.rfilename).match(p) for p in ENGINE_UNWANTED)]
223
+ if any(name.endswith(".safetensors") for name, _ in files):
224
+ files = [f for f in files if not f[0].endswith(".bin")] # the same weights again, in the older format
225
+ return files
226
+
227
+
228
+ def in_engine_cache(repo: str, filename: str = "config.json", revision: Optional[str] = None) -> bool:
229
+ """Whether Hugging Face's cache holds this file of the repo. Looks on disk only, never online."""
230
+ from huggingface_hub import try_to_load_from_cache
231
+
232
+ return isinstance(try_to_load_from_cache(repo, filename, revision=revision), str)
233
+
234
+
235
+ def engine_cache_dir() -> Path:
236
+ from huggingface_hub import constants
237
+
238
+ return Path(constants.HF_HUB_CACHE)
239
+
240
+
241
+ def pull_for_engine(ref: str, log: Callable[[str], None] = print, force: bool = False) -> int:
242
+ """Download a vLLM/SGLang model into Hugging Face's cache. Returns the bytes fetched (0: all there)."""
243
+ repo, revision, filename = hf_reference(ref if ref.startswith(HF_PREFIX) else HF_PREFIX + ref)
244
+ if filename:
245
+ raise DownloadError(f"vLLM and SGLang load a whole repo; name it as hf:{repo}, without {filename!r}")
246
+ from huggingface_hub import snapshot_download
247
+
248
+ token = os.getenv(HF_TOKEN_ENV)
249
+ try:
250
+ files = engine_files(repo, revision, token)
251
+ if not files:
252
+ raise DownloadError(f"{repo} has no files vLLM or SGLang can load")
253
+ missing = files if force else [f for f in files if not in_engine_cache(repo, f[0], revision)]
254
+ if not missing:
255
+ return 0
256
+ needed = sum(size for _, size in missing)
257
+ cache = engine_cache_dir()
258
+ check_disk_space(needed, cache, setting="HF_HOME")
259
+ log(f" {repo}{'@' + revision if revision else ''} ({format_size(needed)}) → {cache}, where the engine looks")
260
+ snapshot_download(repo_id=repo, revision=revision, allow_patterns=[name for name, _ in files],
261
+ token=token, force_download=force)
262
+ except DownloadError:
263
+ raise
264
+ except Exception as e:
265
+ raise _hub_error(repo, e) from e
266
+ return needed
267
+
268
+
202
269
  def _files_to_fetch(repo: str, revision: Optional[str], filename: Optional[str], token) -> list:
203
270
  """The named file plus the small files every copy needs, or the whole repo when it holds one model."""
204
271
  files = hf_files(repo, revision, token)
@@ -235,7 +235,7 @@ def check_acceleration() -> List[CheckResult]:
235
235
  if gpu_offload:
236
236
  results.append(CheckResult(OK, f"llama.cpp can use the GPU ({backend})"))
237
237
  if DEVELOPMENT_CONFIG.llm.n_gpu_layers == 0:
238
- results.append(CheckResult(INFO, "The development profile still runs the LLM on CPU (n_gpu_layers=0)"))
238
+ results.append(CheckResult(INFO, "The development profile is for laptops: it runs the LLM on the CPU (n_gpu_layers=0); production uses the GPU"))
239
239
  elif cuda_devices:
240
240
  # The GPU image builds llama.cpp for the CPU on purpose: on an NVIDIA card the LLM
241
241
  # belongs on a model server, which serves about three times the callers.
@@ -259,20 +259,27 @@ def check_models() -> List[CheckResult]:
259
259
  from fusion_runtime.config import model_dir, model_dir_source
260
260
 
261
261
  results = [CheckResult(INFO, f"Model directory: {short_path(model_dir())} ({model_dir_source()})")]
262
+ # Each profile's missing models count only on the machine that profile is for: on a GPU server,
263
+ # the laptop profile's CPU models aren't a problem, and the reverse.
264
+ gpu = bool(_cuda_device_count())
262
265
  dev_missing = missing_models(Profile.development, apply_env=False)
263
266
  if dev_missing:
264
- results.append(CheckResult(FAIL, f"development profile is missing {', '.join(dev_missing)}",
265
- "frun models pull"))
267
+ results.append(CheckResult(
268
+ INFO if gpu else FAIL,
269
+ f"development profile (laptops) is missing {', '.join(dev_missing)}"
270
+ + (" (only for --config development)" if gpu else ""),
271
+ "frun models pull",
272
+ ))
266
273
  else:
267
- results.append(CheckResult(OK, "development profile: all models installed"))
274
+ results.append(CheckResult(OK, "development profile (laptops): all models installed"))
268
275
 
269
276
  prod_missing = missing_models(Profile.production, apply_env=False)
270
277
  if not prod_missing:
271
- results.append(CheckResult(OK, "production profile: all models installed"))
278
+ results.append(CheckResult(OK, "production profile (NVIDIA GPU): all models installed"))
272
279
  else:
273
280
  results.append(CheckResult(
274
- WARN if _cuda_device_count() else INFO,
275
- f"production profile is missing {', '.join(prod_missing)}",
281
+ WARN if gpu else INFO,
282
+ f"production profile (NVIDIA GPU) is missing {', '.join(prod_missing)}",
276
283
  "frun models pull --config production",
277
284
  ))
278
285
  return results
@@ -1,6 +1,7 @@
1
1
  """The Typer app every `frun` command registers on."""
2
2
  import typer
3
3
 
4
+ from fusion_runtime.cli.bench import bench
4
5
  from fusion_runtime.cli.doctor import doctor
5
6
  from fusion_runtime.cli.keys import key_app, keys_app, token
6
7
  from fusion_runtime.cli.models import models_app
@@ -49,6 +50,7 @@ def _root(
49
50
 
50
51
  app.command()(up)
51
52
  app.command()(talk)
53
+ app.command()(bench)
52
54
  app.add_typer(key_app, name="key")
53
55
  app.add_typer(keys_app, name="keys")
54
56
  app.command()(token)
@@ -0,0 +1,280 @@
1
+ """`frun bench`: how a running server behaves with several callers at once.
2
+
3
+ Opens N real WebSocket sessions against a running server, streams the same recording down each in
4
+ real time, and reports how long each caller waited for audio to come back, and where that time went.
5
+
6
+ frun up agent.py --host 0.0.0.0 # in one terminal
7
+ frun bench --callers 1,4,8,12 # in another
8
+
9
+ The number that matters is how the median moves from one caller to several: an in-process llama.cpp
10
+ model decodes one reply at a time, so callers queue; on vLLM or SGLang they don't, until speech does.
11
+ `--target-ms` turns that into a capacity: how many callers stayed under it. That's what a quote for a
12
+ customer's agent is based on, measured rather than promised.
13
+
14
+ Audio is streamed in real time on purpose: sending a recording at once would measure something the
15
+ runtime never does, and turn detection watches silence in wall-clock time.
16
+ """
17
+ import asyncio
18
+ import json
19
+ import statistics
20
+ import time
21
+ import wave
22
+ from pathlib import Path
23
+ from typing import List, Optional
24
+
25
+ import typer
26
+
27
+ AUDIO_DIR = Path(__file__).resolve().parent / "bench_audio"
28
+ RECORDINGS = {"hello": "hello.wav", "order": "order_1042.wav"} # order: asks about order 1042, for tool agents
29
+ CHUNK_MS = 20
30
+ # Where each reply's time went, from the server's own turn summary: which stage starts to queue as
31
+ # callers are added is the finding, not just that it's slower.
32
+ STAGES = {"stt_transcribe_ms": "stt", "llm_queue_ms": "llm queue", "llm_first_token_ms": "llm first token",
33
+ "tool_ms": "tools", "tts_first_chunk_ms": "tts first audio"}
34
+
35
+
36
+ def load_audio(path: Path) -> bytes:
37
+ with wave.open(str(path), "rb") as wav:
38
+ if wav.getframerate() != 16000 or wav.getnchannels() != 1 or wav.getsampwidth() != 2:
39
+ raise typer.BadParameter(f"{path} must be a 16 kHz mono 16-bit WAV")
40
+ return wav.readframes(wav.getnframes())
41
+
42
+
43
+ def audio_path(name: str) -> Path:
44
+ if name in RECORDINGS:
45
+ return AUDIO_DIR / RECORDINGS[name]
46
+ path = Path(name).expanduser()
47
+ if not path.is_file():
48
+ raise typer.BadParameter(f"no recording {name!r}: use {', '.join(RECORDINGS)} or a path to a WAV")
49
+ return path
50
+
51
+
52
+ def resolve_key(url: str, given: Optional[str]) -> str:
53
+ """The key to present, the way `frun talk` works it out.
54
+
55
+ FUSION_API_KEY if it's set; otherwise, for a server on this machine only, the first key that
56
+ server accepts. Never falls back for a remote URL: that would send your server's keys elsewhere.
57
+ """
58
+ if given:
59
+ return given
60
+ from fusion_runtime.env import load_env_file
61
+ from fusion_runtime.security.keys import client_key
62
+
63
+ load_env_file()
64
+ http_url = url.replace("ws://", "http://").replace("wss://", "https://")
65
+ return client_key(http_url) or ""
66
+
67
+
68
+ async def one_session(index: int, url: str, key: str, audio: bytes, turns: int, results: list) -> None:
69
+ """One caller: connect, then take `turns` turns, timing each reply."""
70
+ import websockets
71
+
72
+ headers = {"Authorization": f"Bearer {key}"} if key else {}
73
+ try: # websockets renamed this parameter; support both without pinning a version
74
+ ws = await websockets.connect(url, additional_headers=headers)
75
+ except TypeError:
76
+ ws = await websockets.connect(url, extra_headers=headers)
77
+
78
+ step = 16000 * 2 // 1000 * CHUNK_MS
79
+ try:
80
+ first = json.loads(await ws.recv()) # the config message, or why the call was refused
81
+ if first.get("type") == "error":
82
+ results.append({"caller": index, "turn": -1,
83
+ "error": first.get("message") + (f" ({first['fix']})" if first.get("fix") else "")})
84
+ return
85
+ # A real microphone never goes digitally silent, and fusion tells a caller sending exact
86
+ # zeros that their mic is muted, so the quiet between turns is faint room noise.
87
+ quiet = b"\x01\x00" * (step // 2)
88
+
89
+ # The first turn allocates caches and isn't representative: taken and thrown away.
90
+ for turn in range(-1, turns):
91
+ for i in range(0, len(audio), step):
92
+ await ws.send(audio[i:i + step])
93
+ await asyncio.sleep(CHUNK_MS / 1000)
94
+ stopped_talking_at = time.perf_counter()
95
+
96
+ first_audio_at = None
97
+ summary: dict = {}
98
+ deadline = stopped_talking_at + 60
99
+
100
+ async def keep_the_mic_open(deadline=deadline):
101
+ # The turn detector ends a turn by hearing silence; stopping the stream would leave
102
+ # the server waiting on a timeout and put that wait into every measurement.
103
+ while time.perf_counter() < deadline:
104
+ await ws.send(quiet)
105
+ await asyncio.sleep(CHUNK_MS / 1000)
106
+
107
+ mic = asyncio.create_task(keep_the_mic_open())
108
+ try:
109
+ while time.perf_counter() < deadline:
110
+ try:
111
+ message = await asyncio.wait_for(ws.recv(), timeout=max(0.1, deadline - time.perf_counter()))
112
+ except asyncio.TimeoutError:
113
+ break
114
+ if isinstance(message, bytes):
115
+ if first_audio_at is None:
116
+ first_audio_at = time.perf_counter()
117
+ continue
118
+ event = json.loads(message)
119
+ if event.get("type") == "error":
120
+ results.append({"caller": index, "turn": turn, "error": event.get("message")})
121
+ return
122
+ if event.get("type") == "turn.trace":
123
+ summary = event.get("summary") or {}
124
+ break # the turn is over; the server said so
125
+ finally:
126
+ mic.cancel()
127
+
128
+ if turn < 0:
129
+ continue # the warm-up turn
130
+ if first_audio_at is None:
131
+ results.append({"caller": index, "turn": turn, "error": "no audio came back"})
132
+ continue
133
+ results.append({
134
+ "caller": index,
135
+ "turn": turn,
136
+ "wall_ms": round((first_audio_at - stopped_talking_at) * 1000),
137
+ "server_ms": round(summary["response_ms"]) if summary.get("response_ms") else None,
138
+ **{stage: summary.get(stage) for stage in STAGES},
139
+ "tool_calls": summary.get("tool_calls") or 0,
140
+ })
141
+ finally:
142
+ await ws.close()
143
+
144
+
145
+ async def run_round(callers: int, url: str, key: str, audio: bytes, turns: int) -> list:
146
+ results: list = []
147
+ await asyncio.gather(*[_counted(i, url, key, audio, turns, results) for i in range(callers)])
148
+ return results
149
+
150
+
151
+ async def _counted(index: int, url: str, key: str, audio: bytes, turns: int, results: list) -> None:
152
+ """One caller; a call the server drops is a failed turn in the round, not the end of the run."""
153
+ from websockets.exceptions import ConnectionClosed
154
+
155
+ try:
156
+ await one_session(index, url, key, audio, turns, results)
157
+ except ConnectionClosed as e:
158
+ why = getattr(e.rcvd, "reason", "") or f"code {getattr(e.rcvd, 'code', '?')}"
159
+ results.append({"caller": index, "error": f"the server closed the call ({why}); see its log"})
160
+
161
+
162
+ def summarize(callers: int, results: list, seconds: float) -> dict:
163
+ """One round as numbers: what `report` prints and `--json` saves."""
164
+ ok = [r for r in results if "wall_ms" in r]
165
+ errors = [r for r in results if "error" in r]
166
+ row: dict = {"callers": callers, "turns": len(ok), "seconds": round(seconds, 1), "failed": len(errors),
167
+ "error": errors[0].get("error") if errors else None}
168
+ if ok:
169
+ walls = sorted(r["wall_ms"] for r in ok)
170
+ row.update(median_ms=round(statistics.median(walls)), worst_ms=walls[-1])
171
+ server = [r["server_ms"] for r in ok if r["server_ms"]]
172
+ row["server_median_ms"] = round(statistics.median(server)) if server else None
173
+ row["stages_median_ms"] = {label: round(statistics.median(values)) for stage, label in STAGES.items()
174
+ if (values := [r[stage] for r in ok if r.get(stage) is not None])}
175
+ row["tool_calls"] = sum(r["tool_calls"] for r in ok)
176
+ return row
177
+
178
+
179
+ def report(row: dict, baseline: Optional[float]) -> str:
180
+ if "median_ms" not in row:
181
+ return f" {row['callers']:>2} caller(s): nothing completed. {row['error'] or ''}".rstrip()
182
+ line = (f" {row['callers']:>2} caller(s): median {row['median_ms']:>5} ms worst {row['worst_ms']:>5} ms "
183
+ f"{row['turns']} turns in {row['seconds']}s")
184
+ if row["server_median_ms"]:
185
+ line += f" (server said {row['server_median_ms']} ms)"
186
+ if baseline:
187
+ line += f" {row['median_ms'] / baseline:.1f}x of one caller"
188
+ lines = [line]
189
+ if row["stages_median_ms"]:
190
+ lines.append(" median ms: " + ", ".join(f"{k} {v}" for k, v in row["stages_median_ms"].items()))
191
+ if row["tool_calls"]:
192
+ lines.append(f" {row['tool_calls']} tool call(s) in {row['turns']} turns")
193
+ if row["failed"]:
194
+ lines.append(f" {row['failed']} turn(s) failed: {row['error']}")
195
+ return "\n".join(lines)
196
+
197
+
198
+ def bottleneck(rows: List[dict]) -> str:
199
+ """Which stage grew most from the fewest callers to the most: where the next caller waits."""
200
+ measured = [r for r in sorted(rows, key=lambda r: r["callers"]) if r.get("stages_median_ms")]
201
+ if len(measured) < 2 or measured[0]["callers"] == measured[-1]["callers"]:
202
+ return "Run more than one round (--callers 1,4,8) to see which stage queues as callers are added."
203
+ first, last = measured[0]["stages_median_ms"], measured[-1]["stages_median_ms"]
204
+ growth = {stage: last[stage] - first.get(stage, 0) for stage in last}
205
+ stage = max(growth, key=growth.get)
206
+ span = f"from {measured[0]['callers']} to {measured[-1]['callers']} caller(s)"
207
+ if growth[stage] < 100:
208
+ return f"No stage grew by more than 100 ms {span}: nothing is queueing yet."
209
+ llm = {k: v for k, v in last.items() if k.startswith("llm")}
210
+ if stage.startswith("llm"):
211
+ return (f"The LLM is where callers wait: {stage} went from {first.get(stage, 0)} to {last[stage]} ms {span}.\n"
212
+ "An in-process model answers one caller at a time; put it on vLLM or SGLang (frun up starts them).")
213
+ held = ", ".join(f"{k} {v} ms" for k, v in llm.items())
214
+ return (f"Speech is where callers wait: {stage} went from {first.get(stage, 0)} to {last[stage]} ms {span}"
215
+ + (f", while the LLM held ({held})" if held else "") + ".\n"
216
+ "Whisper and Kokoro handle one caller at a time on each server; more callers need more servers.")
217
+
218
+
219
+ def capacity(rows: List[dict], target_ms: float) -> str:
220
+ """The most callers whose median stayed under the target, with every round below it clean too."""
221
+ best = None
222
+ for row in sorted(rows, key=lambda r: r["callers"]):
223
+ if row.get("median_ms") is None or row["failed"] or row["median_ms"] > target_ms:
224
+ break
225
+ best = row
226
+ if best is None:
227
+ return f"No round stayed under {target_ms:.0f} ms (median) without failures."
228
+ return (f"Up to {best['callers']} caller(s) stayed under {target_ms:.0f} ms (median {best['median_ms']} ms, "
229
+ f"worst {best['worst_ms']} ms) on this setup.")
230
+
231
+
232
+ async def _bench(url: str, key: str, audio: bytes, callers: List[int], turns: int, label: str,
233
+ target_ms: Optional[float], json_out: Optional[Path]) -> None:
234
+ ws_url = url.rstrip("/") + "/v1/voice/ws"
235
+ typer.echo(f"{label or ws_url} — {len(audio) / 2 / 16000:.2f}s of caller audio, {turns} turn(s) each\n")
236
+ rows, baseline = [], None
237
+ for count in callers:
238
+ started = time.perf_counter()
239
+ results = await run_round(count, ws_url, key, audio, turns)
240
+ row = summarize(count, results, time.perf_counter() - started)
241
+ rows.append(row)
242
+ typer.echo(report(row, baseline))
243
+ if count == 1 and row.get("median_ms"):
244
+ baseline = row["median_ms"]
245
+ await asyncio.sleep(2) # let sessions close before the next round claims slots
246
+ typer.echo("")
247
+ typer.echo(bottleneck(rows))
248
+ typer.echo(capacity(rows, target_ms) if target_ms else "--target-ms 1500 turns the rounds into a capacity.")
249
+ if json_out:
250
+ json_out.write_text(json.dumps({"label": label or ws_url, "turns_per_caller": turns,
251
+ "target_ms": target_ms, "rounds": rows}, indent=2) + "\n")
252
+ typer.echo(f"Saved: {json_out}")
253
+
254
+
255
+ def bench(
256
+ url: str = typer.Option("ws://127.0.0.1:8000", "--url", help="The running server."),
257
+ key: str = typer.Option(None, "--key", help="An accepted key. For a server on this machine it's found for you."),
258
+ callers: str = typer.Option("1,2,4", "--callers", help="How many callers at once, per round, comma separated."),
259
+ turns: int = typer.Option(3, "--turns", min=1, help="Turns per caller, after one warm-up turn."),
260
+ audio: str = typer.Option("hello", "--audio",
261
+ help="What callers say: hello, order (asks about order 1042, for tool agents), "
262
+ "or a 16 kHz mono 16-bit WAV."),
263
+ target_ms: float = typer.Option(None, "--target-ms", min=1,
264
+ help="Report how many callers stayed under this median wait (e.g. 1500)."),
265
+ json_out: Path = typer.Option(None, "--json", help="Also save every round's numbers to this file."),
266
+ label: str = typer.Option("", "--label", help="A heading for the run, e.g. 'RTX 4090, vLLM'."),
267
+ ) -> None:
268
+ """Measure a running server with several callers at once: wait per caller, and where it went."""
269
+ try:
270
+ counts = [int(n) for n in callers.split(",") if n.strip()]
271
+ except ValueError:
272
+ raise typer.BadParameter(f"--callers takes numbers separated by commas, like 1,4,8; got {callers!r}") from None
273
+ if not counts or min(counts) < 1:
274
+ raise typer.BadParameter("--callers needs at least one number, each 1 or more")
275
+ sound = load_audio(audio_path(audio))
276
+ try:
277
+ asyncio.run(_bench(url, resolve_key(url, key), sound, counts, turns, label, target_ms, json_out))
278
+ except OSError as e:
279
+ typer.echo(f"Error: can't reach {url} ({e}). Is the server running? Start it with: frun up", err=True)
280
+ raise typer.Exit(1)
@@ -90,6 +90,9 @@ def token(
90
90
  from fusion_runtime.config import API_KEY_ENV
91
91
  from fusion_runtime.security import client_key
92
92
 
93
+ if public_url and not public_url.startswith(("http://", "https://")): # before a token is spent on it
94
+ typer.echo(f"Error: --public-url must start with https:// (or http://), got {public_url!r}", err=True)
95
+ raise typer.Exit(1)
93
96
  key = key or client_key(url)
94
97
  if not key:
95
98
  typer.echo(f"Error: no key given. Pass --key, or set {API_KEY_ENV} (a .env file works).\n"
@@ -109,11 +112,6 @@ def token(
109
112
  raise typer.Exit(1)
110
113
  response.raise_for_status()
111
114
  body = response.json()
112
- shown = base
113
- if public_url:
114
- if not public_url.startswith(("http://", "https://")):
115
- typer.echo(f"Error: --public-url must start with https:// (or http://), got {public_url!r}", err=True)
116
- raise typer.Exit(1)
117
- shown = public_url.rstrip("/")
115
+ shown = public_url.rstrip("/") if public_url else base
118
116
  typer.echo(f"{shown}/?token={body['token']}")
119
117
  typer.echo(f"\nOpen that in a browser within {body['expires_in']} seconds. It works once.")