monkeybot 2.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- monkeybot/__init__.py +3 -0
- monkeybot/cli/__init__.py +3 -0
- monkeybot/cli/__main__.py +8 -0
- monkeybot/cli/audio_io.py +8 -0
- monkeybot/cli/gateway_manager.py +17 -0
- monkeybot/cli/main.py +22 -0
- monkeybot/cli/push_to_talk.py +12 -0
- monkeybot/cli/realtime_client.py +13 -0
- monkeybot/core/__init__.py +19 -0
- monkeybot/core/attachments/__init__.py +22 -0
- monkeybot/core/attachments/catalog.py +62 -0
- monkeybot/core/attachments/config.py +52 -0
- monkeybot/core/attachments/freeze.py +158 -0
- monkeybot/core/attachments/resolve.py +70 -0
- monkeybot/core/attachments/store.py +180 -0
- monkeybot/core/attachments/text.py +72 -0
- monkeybot/core/attachments/tools.py +54 -0
- monkeybot/core/bootstrap.py +242 -0
- monkeybot/core/config/__init__.py +71 -0
- monkeybot/core/config/realtime_config.py +150 -0
- monkeybot/core/config/runtime_env.py +262 -0
- monkeybot/core/config/settings.py +341 -0
- monkeybot/core/config/validation.py +249 -0
- monkeybot/core/config/yaml_loader.py +45 -0
- monkeybot/core/context/__init__.py +781 -0
- monkeybot/core/context/campaign_context.py +8 -0
- monkeybot/core/context/common.py +14 -0
- monkeybot/core/context/curator.py +255 -0
- monkeybot/core/context/epoch.py +226 -0
- monkeybot/core/context/memory_prompt.py +222 -0
- monkeybot/core/context/tool_output_policy.py +270 -0
- monkeybot/core/context/tool_result_ingress.py +290 -0
- monkeybot/core/context/tool_shapers.py +361 -0
- monkeybot/core/hooks/__init__.py +261 -0
- monkeybot/core/llm/__init__.py +4 -0
- monkeybot/core/llm/provider.py +296 -0
- monkeybot/core/llm/realtime_provider.py +203 -0
- monkeybot/core/llm/usage.py +57 -0
- monkeybot/core/logging_utils.py +24 -0
- monkeybot/core/mcp/__init__.py +1 -0
- monkeybot/core/mcp/mcp_client.py +1215 -0
- monkeybot/core/mcp/ports_mcp.py +109 -0
- monkeybot/core/memory/__init__.py +24 -0
- monkeybot/core/memory/hook.py +413 -0
- monkeybot/core/memory/index_format.py +104 -0
- monkeybot/core/memory/integrity.py +180 -0
- monkeybot/core/memory/organizer.py +270 -0
- monkeybot/core/memory/storage_ops.py +139 -0
- monkeybot/core/memory/subsystem.py +91 -0
- monkeybot/core/messages/__init__.py +16 -0
- monkeybot/core/messages/convert_provider.py +41 -0
- monkeybot/core/messages/tool_integrity.py +262 -0
- monkeybot/core/messages/transform_context.py +84 -0
- monkeybot/core/path_safety.py +11 -0
- monkeybot/core/persistence/__init__.py +17 -0
- monkeybot/core/persistence/backends.py +236 -0
- monkeybot/core/persistence/db.py +28 -0
- monkeybot/core/persistence/durable_runs.py +286 -0
- monkeybot/core/persistence/firestore.py +658 -0
- monkeybot/core/persistence/firestore_scheduled_loops.py +336 -0
- monkeybot/core/persistence/history.py +156 -0
- monkeybot/core/persistence/postgres.py +895 -0
- monkeybot/core/persistence/runs.py +76 -0
- monkeybot/core/persistence/scheduled_loops.py +435 -0
- monkeybot/core/persistence/session_turn_locks.py +94 -0
- monkeybot/core/persistence/sqlite.py +218 -0
- monkeybot/core/persistence/sqlite_backend.py +74 -0
- monkeybot/core/persistence/thread_summary.py +61 -0
- monkeybot/core/persistence/transcript.py +194 -0
- monkeybot/core/persistence/usage.py +149 -0
- monkeybot/core/prompts/__init__.py +1 -0
- monkeybot/core/prompts/harness_prompt.py +197 -0
- monkeybot/core/prompts/prompt.py +215 -0
- monkeybot/core/runtime/__init__.py +1 -0
- monkeybot/core/runtime/context_budget.py +267 -0
- monkeybot/core/runtime/events.py +819 -0
- monkeybot/core/runtime/input_admission.py +154 -0
- monkeybot/core/runtime/loop.py +2374 -0
- monkeybot/core/runtime/provider_stream_mapper.py +159 -0
- monkeybot/core/runtime/realtime_loop.py +654 -0
- monkeybot/core/runtime/utterance_buffer.py +179 -0
- monkeybot/core/subagents/__init__.py +1 -0
- monkeybot/core/subagents/subagent_proto.py +331 -0
- monkeybot/core/subagents/subagent_worker.py +441 -0
- monkeybot/core/subagents/worker_pool.py +403 -0
- monkeybot/core/testing/__init__.py +1 -0
- monkeybot/core/testing/mocks_provider.py +86 -0
- monkeybot/core/testing/mocks_realtime_provider.py +137 -0
- monkeybot/core/tools/__init__.py +1 -0
- monkeybot/core/tools/core_tool_executor.py +1548 -0
- monkeybot/core/tools/inspector.py +226 -0
- monkeybot/core/tools/loop_inspector.py +45 -0
- monkeybot/core/tools/patch.py +480 -0
- monkeybot/core/tools/permission.py +284 -0
- monkeybot/core/tools/sandbox_executor.py +255 -0
- monkeybot/core/tools/spill_inventory.py +35 -0
- monkeybot/core/tools/terminal.py +381 -0
- monkeybot/core/tools/text_normalize.py +25 -0
- monkeybot/core/tools/types.py +33 -0
- monkeybot/core/tools/workspace_service.py +710 -0
- monkeybot/core/tools/workspace_tools.py +116 -0
- monkeybot/core/types/__init__.py +1 -0
- monkeybot/core/types/content_blocks.py +644 -0
- monkeybot/core/types/interfaces.py +156 -0
- monkeybot/core/types/types_tools.py +29 -0
- monkeybot/core/workspace/__init__.py +8 -0
- monkeybot/core/workspace/factory.py +45 -0
- monkeybot/core/workspace/gcs.py +130 -0
- monkeybot/core/workspace/local.py +162 -0
- monkeybot/core/workspace/protocol.py +45 -0
- monkeybot/core/workspace/s3.py +151 -0
- monkeybot/core/workspace_layout.py +27 -0
- monkeybot/gateway/__init__.py +1 -0
- monkeybot/gateway/bootstrap.py +18 -0
- monkeybot/gateway/main.py +47 -0
- monkeybot/gateway/realtime/__init__.py +31 -0
- monkeybot/gateway/realtime/app.py +321 -0
- monkeybot/gateway/realtime/deps.py +52 -0
- monkeybot/gateway/realtime/errors.py +81 -0
- monkeybot/gateway/realtime/guardrails.py +88 -0
- monkeybot/gateway/realtime/manager.py +77 -0
- monkeybot/gateway/realtime/metrics.py +144 -0
- monkeybot/gateway/realtime/routes.py +864 -0
- monkeybot/gateway/realtime/session.py +232 -0
- monkeybot/gateway/realtime/wire.py +412 -0
- monkeybot/gateway/realtime_main.py +49 -0
- monkeybot/gateway/sse/__init__.py +1 -0
- monkeybot/gateway/sse/app.py +733 -0
- monkeybot/gateway/sse/loop_port.py +31 -0
- monkeybot/gateway/sse/models.py +177 -0
- monkeybot/gateway/sse/reply_body.py +91 -0
- monkeybot/gateway/sse/routes.py +1101 -0
- monkeybot/gateway/sse/scheduler_routes.py +200 -0
- monkeybot/gateway/sse/scheduler_wiring.py +96 -0
- monkeybot/gateway/sse/session_bus.py +226 -0
- monkeybot/gateway/sse/sse.py +46 -0
- monkeybot/gateway/sse/workspace_layout.py +7 -0
- monkeybot/observability/__init__.py +220 -0
- monkeybot/observability/_state.py +10 -0
- monkeybot/observability/instrumentation.py +153 -0
- monkeybot/observability/propagation.py +65 -0
- monkeybot/observability/spans.py +455 -0
- monkeybot/providers/__init__.py +19 -0
- monkeybot/providers/_openai_compat.py +450 -0
- monkeybot/providers/_utils.py +473 -0
- monkeybot/providers/bedrock.py +145 -0
- monkeybot/providers/claude.py +125 -0
- monkeybot/providers/gemini.py +677 -0
- monkeybot/providers/gemini_live.py +398 -0
- monkeybot/providers/huggingface.py +129 -0
- monkeybot/providers/nvidia.py +104 -0
- monkeybot/providers/ollama.py +152 -0
- monkeybot/providers/openai.py +127 -0
- monkeybot/providers/pricing.py +60 -0
- monkeybot/providers/sampling.py +44 -0
- monkeybot/providers/vertex_claude.py +148 -0
- monkeybot/scaffold/__init__.py +33 -0
- monkeybot/scheduler/__init__.py +13 -0
- monkeybot/scheduler/__main__.py +4 -0
- monkeybot/scheduler/engine.py +333 -0
- monkeybot/scheduler/http_invoker.py +61 -0
- monkeybot/scheduler/interval.py +77 -0
- monkeybot/scheduler/tick_result.py +34 -0
- monkeybot/scheduler/worker.py +87 -0
- monkeybot/subagents/__init__.py +1 -0
- monkeybot/subagents/worker/__init__.py +1 -0
- monkeybot/subagents/worker/__main__.py +22 -0
- monkeybot/web_search/__init__.py +82 -0
- monkeybot/web_search/backends/__init__.py +5 -0
- monkeybot/web_search/backends/duckduckgo.py +32 -0
- monkeybot/web_search/backends/firecrawl.py +43 -0
- monkeybot/web_search/backends/tavily.py +45 -0
- monkeybot/web_search/protocol.py +25 -0
- monkeybot/web_search/tool.py +56 -0
- monkeybot-2.1.1.dist-info/METADATA +318 -0
- monkeybot-2.1.1.dist-info/RECORD +178 -0
- monkeybot-2.1.1.dist-info/WHEEL +4 -0
- monkeybot-2.1.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""Per-connection realtime session state machine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
from monkeybot.core.llm.realtime_provider import (
|
|
11
|
+
AudioFormat,
|
|
12
|
+
RealtimeAudioDelta,
|
|
13
|
+
RealtimeDone,
|
|
14
|
+
RealtimeEvent,
|
|
15
|
+
RealtimeInterrupted,
|
|
16
|
+
RealtimePartialTranscript,
|
|
17
|
+
RealtimeProvider,
|
|
18
|
+
RealtimeSession,
|
|
19
|
+
RealtimeTextDelta,
|
|
20
|
+
RealtimeToolCall,
|
|
21
|
+
RealtimeTurnBoundary,
|
|
22
|
+
RealtimeUsage,
|
|
23
|
+
)
|
|
24
|
+
from monkeybot.core.logging_utils import kv
|
|
25
|
+
from monkeybot.core.runtime.utterance_buffer import UtteranceBuffer
|
|
26
|
+
|
|
27
|
+
from .metrics import RealtimeMetrics
|
|
28
|
+
from .wire import audio_duration_sec
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger("monkeybot.gateway.realtime.session")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class SessionStateError(Exception):
|
|
34
|
+
"""Invalid state transition or operation on a realtime session."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
RealtimeSessionState = Literal[
|
|
38
|
+
"listening", "thinking", "speaking", "interrupted", "tool_running", "closing"
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class RealtimeConnectionState:
|
|
44
|
+
"""Mutable state for one realtime WebSocket connection."""
|
|
45
|
+
|
|
46
|
+
session_id: str
|
|
47
|
+
request_id: str
|
|
48
|
+
provider: RealtimeProvider
|
|
49
|
+
realtime_session: RealtimeSession
|
|
50
|
+
buffer: UtteranceBuffer = field(default_factory=UtteranceBuffer)
|
|
51
|
+
state: RealtimeSessionState = "listening"
|
|
52
|
+
opened_at: float = 0.0
|
|
53
|
+
last_activity_at: float = 0.0
|
|
54
|
+
# Pending tool/subagent payloads to inject when the session reaches idle.
|
|
55
|
+
# Each item is either tool-result tuples for send_tool_results, or a context
|
|
56
|
+
# string for send_context (e.g. hook inject_text).
|
|
57
|
+
idle_delivery_queue: asyncio.Queue[Any] = field(default_factory=asyncio.Queue)
|
|
58
|
+
metrics: RealtimeMetrics = field(init=False)
|
|
59
|
+
pending_responses: dict[str, asyncio.Future[Any]] = field(default_factory=dict)
|
|
60
|
+
_closed: bool = False
|
|
61
|
+
|
|
62
|
+
def __post_init__(self) -> None:
|
|
63
|
+
self.metrics = RealtimeMetrics(
|
|
64
|
+
session_id=self.session_id,
|
|
65
|
+
request_id=self.request_id,
|
|
66
|
+
opened_at=self.opened_at,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
def register_pending(self, key: str) -> asyncio.Future[Any]:
|
|
70
|
+
loop = asyncio.get_running_loop()
|
|
71
|
+
fut: asyncio.Future[Any] = loop.create_future()
|
|
72
|
+
self.pending_responses[key] = fut
|
|
73
|
+
return fut
|
|
74
|
+
|
|
75
|
+
def resolve_pending(self, key: str, payload: dict[str, Any]) -> bool:
|
|
76
|
+
fut = self.pending_responses.pop(key, None)
|
|
77
|
+
if fut is None or fut.done():
|
|
78
|
+
return False
|
|
79
|
+
fut.set_result(payload)
|
|
80
|
+
return True
|
|
81
|
+
|
|
82
|
+
def abandon_pending_timeout(self, key: str) -> None:
|
|
83
|
+
fut = self.pending_responses.pop(key, None)
|
|
84
|
+
if fut is not None and not fut.done():
|
|
85
|
+
fut.set_result({"_timeout": True})
|
|
86
|
+
|
|
87
|
+
def abandon_pending_cancel_all(self) -> None:
|
|
88
|
+
for key in list(self.pending_responses):
|
|
89
|
+
fut = self.pending_responses.pop(key, None)
|
|
90
|
+
if fut is not None and not fut.done():
|
|
91
|
+
fut.cancel()
|
|
92
|
+
|
|
93
|
+
def is_pending_or_terminal(
|
|
94
|
+
self, key: str
|
|
95
|
+
) -> Literal["pending", "terminated", "unknown"]:
|
|
96
|
+
if key in self.pending_responses:
|
|
97
|
+
return "pending"
|
|
98
|
+
return "unknown"
|
|
99
|
+
|
|
100
|
+
def transition(self, new_state: RealtimeSessionState) -> None:
|
|
101
|
+
"""Validate and apply a state transition."""
|
|
102
|
+
valid = (self.state, new_state) in _VALID_TRANSITIONS
|
|
103
|
+
if not valid:
|
|
104
|
+
raise SessionStateError(
|
|
105
|
+
f"Invalid realtime transition {self.state} -> {new_state}"
|
|
106
|
+
)
|
|
107
|
+
old = self.state
|
|
108
|
+
self.state = new_state
|
|
109
|
+
logger.debug(
|
|
110
|
+
"realtime state transition %s",
|
|
111
|
+
kv(
|
|
112
|
+
request_id=self.request_id,
|
|
113
|
+
session_id=self.session_id,
|
|
114
|
+
old=old,
|
|
115
|
+
new=new_state,
|
|
116
|
+
),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def mark_activity(self, now: float) -> None:
|
|
120
|
+
self.last_activity_at = now
|
|
121
|
+
|
|
122
|
+
def apply_provider_event(self, event: RealtimeEvent) -> None:
|
|
123
|
+
"""Update state, buffer, and metrics based on a provider event."""
|
|
124
|
+
if self.state == "closing":
|
|
125
|
+
return
|
|
126
|
+
if isinstance(event, RealtimePartialTranscript):
|
|
127
|
+
if event.is_final:
|
|
128
|
+
# Gemini Live emits input transcription as final-only, often after
|
|
129
|
+
# the client already sent audio_stream_end. Attach it to the
|
|
130
|
+
# just-finished utterance so history commit sees spoken text.
|
|
131
|
+
self.buffer.apply_final_user_transcript(event.text)
|
|
132
|
+
else:
|
|
133
|
+
self.buffer.add_user_text(event.text)
|
|
134
|
+
elif isinstance(event, (RealtimeAudioDelta, RealtimeTextDelta)):
|
|
135
|
+
if isinstance(event, RealtimeAudioDelta):
|
|
136
|
+
self.metrics.mark_model_audio_sec(
|
|
137
|
+
audio_duration_sec(event.chunk, self.realtime_session.output_format)
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
self.buffer.add_assistant_text(event.text)
|
|
141
|
+
self.metrics.mark_first_output()
|
|
142
|
+
if self.state in ("listening", "thinking"):
|
|
143
|
+
self.transition("speaking")
|
|
144
|
+
elif isinstance(event, RealtimeTurnBoundary):
|
|
145
|
+
if event.role == "user":
|
|
146
|
+
if self.state in ("speaking", "thinking"):
|
|
147
|
+
# User interrupted the assistant and then finalized.
|
|
148
|
+
self.transition("interrupted")
|
|
149
|
+
self.transition("listening")
|
|
150
|
+
self.buffer.mark_user_turn_boundary()
|
|
151
|
+
self.metrics.start_turn()
|
|
152
|
+
else:
|
|
153
|
+
# Assistant turn boundary.
|
|
154
|
+
self.buffer.mark_assistant_turn_boundary()
|
|
155
|
+
self.metrics.end_turn()
|
|
156
|
+
# mark_assistant_turn_boundary clears the in-flight buffer; check the
|
|
157
|
+
# finalized turn for tool calls.
|
|
158
|
+
if self.buffer.current_assistant_turn().tool_calls:
|
|
159
|
+
self.transition("tool_running")
|
|
160
|
+
return
|
|
161
|
+
self.transition("listening")
|
|
162
|
+
elif isinstance(event, RealtimeToolCall):
|
|
163
|
+
self.buffer.add_assistant_tool_call(event)
|
|
164
|
+
self.metrics.mark_tool_in_turn()
|
|
165
|
+
elif isinstance(event, RealtimeInterrupted):
|
|
166
|
+
# Provider acknowledged interrupt.
|
|
167
|
+
if self.state in ("speaking", "thinking"):
|
|
168
|
+
self.transition("interrupted")
|
|
169
|
+
elif isinstance(event, RealtimeUsage):
|
|
170
|
+
self.metrics.mark_usage(
|
|
171
|
+
input_tokens=event.input_tokens,
|
|
172
|
+
output_tokens=event.output_tokens,
|
|
173
|
+
)
|
|
174
|
+
elif isinstance(event, RealtimeDone) and self.state in (
|
|
175
|
+
"speaking",
|
|
176
|
+
"thinking",
|
|
177
|
+
"interrupted",
|
|
178
|
+
):
|
|
179
|
+
# Assistant turn ended naturally.
|
|
180
|
+
self.buffer.mark_assistant_turn_boundary()
|
|
181
|
+
self.metrics.end_turn()
|
|
182
|
+
self.transition("listening")
|
|
183
|
+
|
|
184
|
+
def handle_user_interrupt(self) -> None:
|
|
185
|
+
"""Client or user barge-in."""
|
|
186
|
+
if self.state in ("speaking", "thinking"):
|
|
187
|
+
self.metrics.mark_interrupt()
|
|
188
|
+
self.buffer.interrupt()
|
|
189
|
+
self.transition("interrupted")
|
|
190
|
+
# The provider interrupt call is made by the gateway; here we just update state.
|
|
191
|
+
elif self.state == "tool_running":
|
|
192
|
+
# Constraint 4: do not interrupt a tool call once dispatched.
|
|
193
|
+
pass
|
|
194
|
+
|
|
195
|
+
def mark_user_audio(self, duration_sec: float) -> None:
|
|
196
|
+
"""Track user audio duration and activity."""
|
|
197
|
+
self.metrics.mark_user_audio_sec(duration_sec)
|
|
198
|
+
|
|
199
|
+
def is_idle(self) -> bool:
|
|
200
|
+
return self.state == "listening" and not self.buffer.in_user_turn
|
|
201
|
+
|
|
202
|
+
def enqueue_idle_delivery(self, payload: Any) -> None:
|
|
203
|
+
"""Queue a payload for injection when the session is idle."""
|
|
204
|
+
self.idle_delivery_queue.put_nowait(payload)
|
|
205
|
+
|
|
206
|
+
def close(self) -> None:
|
|
207
|
+
self._closed = True
|
|
208
|
+
self.state = "closing"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
_VALID_TRANSITIONS: set[tuple[RealtimeSessionState, RealtimeSessionState]] = {
|
|
212
|
+
("listening", "listening"),
|
|
213
|
+
("listening", "thinking"),
|
|
214
|
+
("listening", "speaking"), # model can start speaking immediately after user input
|
|
215
|
+
("thinking", "speaking"),
|
|
216
|
+
("thinking", "listening"),
|
|
217
|
+
("thinking", "interrupted"),
|
|
218
|
+
("speaking", "interrupted"),
|
|
219
|
+
("interrupted", "listening"),
|
|
220
|
+
("speaking", "listening"),
|
|
221
|
+
("speaking", "tool_running"),
|
|
222
|
+
("thinking", "tool_running"),
|
|
223
|
+
("listening", "tool_running"), # tool-only turns may skip speaking
|
|
224
|
+
("tool_running", "listening"),
|
|
225
|
+
("tool_running", "thinking"),
|
|
226
|
+
# ("tool_running", "interrupted") is explicitly disallowed.
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
# Any source state may transition to closing.
|
|
231
|
+
for _src in ("listening", "thinking", "speaking", "interrupted", "tool_running"):
|
|
232
|
+
_VALID_TRANSITIONS.add((_src, "closing"))
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"""WebSocket frame protocol for MonkeyBot realtime sessions.
|
|
2
|
+
|
|
3
|
+
Frames are either binary audio chunks or JSON control messages. The gateway keeps
|
|
4
|
+
client/gateway audio formats fixed per deployment; the provider may negotiate a
|
|
5
|
+
different format internally, but the client contract is stable.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
from monkeybot.core.llm.realtime_provider import AudioFormat
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class ClientConnectFrame:
|
|
19
|
+
"""First message from the client; requests a realtime session for the given id."""
|
|
20
|
+
|
|
21
|
+
kind: Literal["connect"] = "connect"
|
|
22
|
+
session_id: str = ""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class ClientTextFrame:
|
|
27
|
+
"""Text input from the client."""
|
|
28
|
+
|
|
29
|
+
kind: Literal["text"] = "text"
|
|
30
|
+
text: str = ""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class ClientInterruptFrame:
|
|
35
|
+
"""Client explicitly requests to interrupt the current assistant turn."""
|
|
36
|
+
|
|
37
|
+
kind: Literal["interrupt"] = "interrupt"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class ClientAudioStreamEndFrame:
|
|
42
|
+
"""Client finished a push-to-talk utterance; provider should treat speech as complete."""
|
|
43
|
+
|
|
44
|
+
kind: Literal["audio_stream_end"] = "audio_stream_end"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class ClientCloseFrame:
|
|
49
|
+
"""Client is closing the realtime session gracefully."""
|
|
50
|
+
|
|
51
|
+
kind: Literal["close"] = "close"
|
|
52
|
+
reason: str = "client_close"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class ClientToolConfirmationResponseFrame:
|
|
57
|
+
kind: Literal["tool_confirmation_response"] = "tool_confirmation_response"
|
|
58
|
+
tool_call_id: str = ""
|
|
59
|
+
approved: bool = False
|
|
60
|
+
reason: str = ""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class ClientElicitationResponseFrame:
|
|
65
|
+
kind: Literal["elicitation_response"] = "elicitation_response"
|
|
66
|
+
elicitation_id: str = ""
|
|
67
|
+
user_data: Any = None
|
|
68
|
+
cancelled: bool = False
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
ClientFrame = (
|
|
72
|
+
ClientConnectFrame
|
|
73
|
+
| ClientTextFrame
|
|
74
|
+
| ClientInterruptFrame
|
|
75
|
+
| ClientAudioStreamEndFrame
|
|
76
|
+
| ClientCloseFrame
|
|
77
|
+
| ClientToolConfirmationResponseFrame
|
|
78
|
+
| ClientElicitationResponseFrame
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True)
|
|
83
|
+
class ServerConnectedFrame:
|
|
84
|
+
kind: Literal["connected"] = "connected"
|
|
85
|
+
session_id: str = ""
|
|
86
|
+
input_format: str = "pcm_s16le_24khz_mono"
|
|
87
|
+
output_format: str = "pcm_s16le_24khz_mono"
|
|
88
|
+
chunk_ms: int = 200
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class ServerTextDeltaFrame:
|
|
93
|
+
kind: Literal["text_delta"] = "text_delta"
|
|
94
|
+
delta: str = ""
|
|
95
|
+
is_final: bool = False
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class ServerAudioFrame:
|
|
100
|
+
kind: Literal["audio"] = "audio"
|
|
101
|
+
chunk: bytes = b""
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class ServerInterruptedFrame:
|
|
106
|
+
kind: Literal["interrupted"] = "interrupted"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass(frozen=True)
|
|
110
|
+
class ServerTurnBoundaryFrame:
|
|
111
|
+
kind: Literal["turn_boundary"] = "turn_boundary"
|
|
112
|
+
role: Literal["user", "assistant"] = "user"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@dataclass(frozen=True)
|
|
116
|
+
class ServerToolCallFrame:
|
|
117
|
+
kind: Literal["tool_call"] = "tool_call"
|
|
118
|
+
call_id: str = ""
|
|
119
|
+
name: str = ""
|
|
120
|
+
args: dict[str, Any] = field(default_factory=dict)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@dataclass(frozen=True)
|
|
124
|
+
class ServerToolResultFrame:
|
|
125
|
+
kind: Literal["tool_result"] = "tool_result"
|
|
126
|
+
call_id: str = ""
|
|
127
|
+
name: str = ""
|
|
128
|
+
result: str = ""
|
|
129
|
+
error: str | None = None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@dataclass(frozen=True)
|
|
133
|
+
class ServerUserTranscriptFrame:
|
|
134
|
+
kind: Literal["user_transcript"] = "user_transcript"
|
|
135
|
+
text: str = ""
|
|
136
|
+
is_final: bool = True
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class ServerUsageFrame:
|
|
141
|
+
kind: Literal["usage"] = "usage"
|
|
142
|
+
usage: dict[str, Any] = field(default_factory=dict)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass(frozen=True)
|
|
146
|
+
class ServerToolConfirmationFrame:
|
|
147
|
+
kind: Literal["tool_confirmation"] = "tool_confirmation"
|
|
148
|
+
tool_call_id: str = ""
|
|
149
|
+
tool_name: str = ""
|
|
150
|
+
prompt: str = ""
|
|
151
|
+
arguments: dict[str, Any] = field(default_factory=dict)
|
|
152
|
+
timeout_sec: float | None = None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass(frozen=True)
|
|
156
|
+
class ServerElicitationFrame:
|
|
157
|
+
kind: Literal["elicitation"] = "elicitation"
|
|
158
|
+
elicitation_id: str = ""
|
|
159
|
+
prompt: str = ""
|
|
160
|
+
schema: dict[str, Any] | None = None
|
|
161
|
+
timeout_sec: float | None = None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(frozen=True)
|
|
165
|
+
class ServerErrorFrame:
|
|
166
|
+
kind: Literal["error"] = "error"
|
|
167
|
+
error: str = ""
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@dataclass(frozen=True)
|
|
171
|
+
class ServerSessionEndedFrame:
|
|
172
|
+
kind: Literal["session_ended"] = "session_ended"
|
|
173
|
+
reason: str = "session_end"
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
ServerFrame = (
|
|
177
|
+
ServerConnectedFrame
|
|
178
|
+
| ServerTextDeltaFrame
|
|
179
|
+
| ServerAudioFrame
|
|
180
|
+
| ServerInterruptedFrame
|
|
181
|
+
| ServerTurnBoundaryFrame
|
|
182
|
+
| ServerToolCallFrame
|
|
183
|
+
| ServerToolResultFrame
|
|
184
|
+
| ServerUserTranscriptFrame
|
|
185
|
+
| ServerUsageFrame
|
|
186
|
+
| ServerToolConfirmationFrame
|
|
187
|
+
| ServerElicitationFrame
|
|
188
|
+
| ServerErrorFrame
|
|
189
|
+
| ServerSessionEndedFrame
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _frame_size_for(fmt: AudioFormat) -> int:
|
|
194
|
+
bytes_per_sample = 2 if fmt.encoding.startswith("pcm_s") else 1
|
|
195
|
+
samples = int(fmt.sample_rate_hz * fmt.frame_ms / 1000)
|
|
196
|
+
return samples * bytes_per_sample * fmt.channels
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def audio_duration_sec(chunk: bytes, fmt: AudioFormat) -> float:
|
|
200
|
+
"""Duration in seconds for a PCM (or similar) audio chunk."""
|
|
201
|
+
bytes_per_sample = 2 if fmt.encoding.startswith("pcm_s") else 1
|
|
202
|
+
frame_size = bytes_per_sample * fmt.channels
|
|
203
|
+
if frame_size == 0:
|
|
204
|
+
return 0.0
|
|
205
|
+
samples = len(chunk) // frame_size
|
|
206
|
+
return samples / fmt.sample_rate_hz
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def parse_client_frame(data: str | bytes) -> ClientFrame:
|
|
210
|
+
"""Parse a client JSON control frame."""
|
|
211
|
+
if isinstance(data, bytes):
|
|
212
|
+
raise ProtocolError("Binary frames are audio chunks and must be handled before parsing")
|
|
213
|
+
try:
|
|
214
|
+
payload = json.loads(data)
|
|
215
|
+
except json.JSONDecodeError as exc:
|
|
216
|
+
raise ProtocolError(f"Invalid JSON frame: {exc}") from exc
|
|
217
|
+
if not isinstance(payload, dict):
|
|
218
|
+
raise ProtocolError("Client frame must be a JSON object")
|
|
219
|
+
kind = str(payload.get("kind") or payload.get("type") or "")
|
|
220
|
+
if kind == "connect":
|
|
221
|
+
return ClientConnectFrame(session_id=str(payload.get("session_id", "")))
|
|
222
|
+
if kind == "text":
|
|
223
|
+
return ClientTextFrame(text=str(payload.get("text", "")))
|
|
224
|
+
if kind == "interrupt":
|
|
225
|
+
return ClientInterruptFrame()
|
|
226
|
+
if kind == "audio_stream_end":
|
|
227
|
+
return ClientAudioStreamEndFrame()
|
|
228
|
+
if kind == "close":
|
|
229
|
+
return ClientCloseFrame(reason=str(payload.get("reason", "client_close")))
|
|
230
|
+
if kind == "tool_confirmation_response":
|
|
231
|
+
return ClientToolConfirmationResponseFrame(
|
|
232
|
+
tool_call_id=str(payload.get("tool_call_id", "")),
|
|
233
|
+
approved=bool(payload.get("approved", False)),
|
|
234
|
+
reason=str(payload.get("reason", "")),
|
|
235
|
+
)
|
|
236
|
+
if kind == "elicitation_response":
|
|
237
|
+
return ClientElicitationResponseFrame(
|
|
238
|
+
elicitation_id=str(payload.get("elicitation_id", "")),
|
|
239
|
+
user_data=payload.get("user_data"),
|
|
240
|
+
cancelled=bool(payload.get("cancelled", False)),
|
|
241
|
+
)
|
|
242
|
+
raise ProtocolError(f"Unknown client frame kind: {kind!r}")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def parse_server_frame(data: str) -> ServerFrame:
|
|
246
|
+
"""Parse a server JSON control frame."""
|
|
247
|
+
try:
|
|
248
|
+
payload = json.loads(data)
|
|
249
|
+
except json.JSONDecodeError as exc:
|
|
250
|
+
raise ProtocolError(f"Invalid JSON frame: {exc}") from exc
|
|
251
|
+
if not isinstance(payload, dict):
|
|
252
|
+
raise ProtocolError("Server frame must be a JSON object")
|
|
253
|
+
kind = str(payload.get("kind") or payload.get("type") or "")
|
|
254
|
+
if kind == "connected":
|
|
255
|
+
return ServerConnectedFrame(
|
|
256
|
+
session_id=str(payload.get("session_id", "")),
|
|
257
|
+
input_format=str(payload.get("input_format", "pcm_s16le_24khz_mono")),
|
|
258
|
+
output_format=str(payload.get("output_format", "pcm_s16le_24khz_mono")),
|
|
259
|
+
chunk_ms=int(payload.get("chunk_ms", 200)),
|
|
260
|
+
)
|
|
261
|
+
if kind == "text_delta":
|
|
262
|
+
return ServerTextDeltaFrame(
|
|
263
|
+
delta=str(payload.get("delta", "")),
|
|
264
|
+
is_final=bool(payload.get("is_final", False)),
|
|
265
|
+
)
|
|
266
|
+
if kind == "interrupted":
|
|
267
|
+
return ServerInterruptedFrame()
|
|
268
|
+
if kind == "turn_boundary":
|
|
269
|
+
role = str(payload.get("role", "user"))
|
|
270
|
+
return ServerTurnBoundaryFrame(role=role) # type: ignore[arg-type]
|
|
271
|
+
if kind == "tool_call":
|
|
272
|
+
return ServerToolCallFrame(
|
|
273
|
+
call_id=str(payload.get("call_id", "")),
|
|
274
|
+
name=str(payload.get("name", "")),
|
|
275
|
+
args=dict(payload.get("args") or {}),
|
|
276
|
+
)
|
|
277
|
+
if kind == "tool_result":
|
|
278
|
+
err = payload.get("error")
|
|
279
|
+
return ServerToolResultFrame(
|
|
280
|
+
call_id=str(payload.get("call_id", "")),
|
|
281
|
+
name=str(payload.get("name") or payload.get("tool") or ""),
|
|
282
|
+
result=str(payload.get("result", "")),
|
|
283
|
+
error=None if err is None else str(err),
|
|
284
|
+
)
|
|
285
|
+
if kind == "user_transcript":
|
|
286
|
+
return ServerUserTranscriptFrame(
|
|
287
|
+
text=str(payload.get("text", "")),
|
|
288
|
+
is_final=bool(payload.get("is_final", True)),
|
|
289
|
+
)
|
|
290
|
+
if kind == "usage":
|
|
291
|
+
usage = payload.get("usage")
|
|
292
|
+
return ServerUsageFrame(usage=dict(usage) if isinstance(usage, dict) else dict(payload))
|
|
293
|
+
if kind == "tool_confirmation":
|
|
294
|
+
args = payload.get("arguments") or payload.get("args") or {}
|
|
295
|
+
timeout = payload.get("timeout_sec")
|
|
296
|
+
return ServerToolConfirmationFrame(
|
|
297
|
+
tool_call_id=str(payload.get("tool_call_id", "")),
|
|
298
|
+
tool_name=str(payload.get("tool_name") or payload.get("tool") or ""),
|
|
299
|
+
prompt=str(payload.get("prompt") or payload.get("message") or ""),
|
|
300
|
+
arguments=dict(args) if isinstance(args, dict) else {},
|
|
301
|
+
timeout_sec=float(timeout) if timeout is not None else None,
|
|
302
|
+
)
|
|
303
|
+
if kind == "elicitation":
|
|
304
|
+
schema = payload.get("schema") or payload.get("requestedSchema")
|
|
305
|
+
timeout = payload.get("timeout_sec")
|
|
306
|
+
return ServerElicitationFrame(
|
|
307
|
+
elicitation_id=str(payload.get("elicitation_id", "")),
|
|
308
|
+
prompt=str(payload.get("prompt") or payload.get("message") or ""),
|
|
309
|
+
schema=dict(schema) if isinstance(schema, dict) else None,
|
|
310
|
+
timeout_sec=float(timeout) if timeout is not None else None,
|
|
311
|
+
)
|
|
312
|
+
if kind == "error":
|
|
313
|
+
return ServerErrorFrame(error=str(payload.get("error", "")))
|
|
314
|
+
if kind == "session_ended":
|
|
315
|
+
return ServerSessionEndedFrame(reason=str(payload.get("reason", "session_end")))
|
|
316
|
+
raise ProtocolError(f"Unknown server frame kind: {kind!r}")
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def encode_server_frame(frame: ServerFrame) -> str | bytes:
|
|
320
|
+
"""Encode a server frame to JSON text or binary audio."""
|
|
321
|
+
if isinstance(frame, ServerAudioFrame):
|
|
322
|
+
return frame.chunk
|
|
323
|
+
payload: dict[str, Any] = {"kind": frame.kind}
|
|
324
|
+
if isinstance(frame, ServerConnectedFrame):
|
|
325
|
+
payload.update(
|
|
326
|
+
{
|
|
327
|
+
"session_id": frame.session_id,
|
|
328
|
+
"input_format": frame.input_format,
|
|
329
|
+
"output_format": frame.output_format,
|
|
330
|
+
"chunk_ms": frame.chunk_ms,
|
|
331
|
+
}
|
|
332
|
+
)
|
|
333
|
+
elif isinstance(frame, ServerTextDeltaFrame):
|
|
334
|
+
payload.update({"delta": frame.delta, "is_final": frame.is_final})
|
|
335
|
+
elif isinstance(frame, ServerTurnBoundaryFrame):
|
|
336
|
+
payload["role"] = frame.role
|
|
337
|
+
elif isinstance(frame, ServerToolCallFrame):
|
|
338
|
+
payload.update(
|
|
339
|
+
{"call_id": frame.call_id, "name": frame.name, "args": frame.args or {}}
|
|
340
|
+
)
|
|
341
|
+
elif isinstance(frame, ServerToolResultFrame):
|
|
342
|
+
payload.update(
|
|
343
|
+
{
|
|
344
|
+
"call_id": frame.call_id,
|
|
345
|
+
"name": frame.name,
|
|
346
|
+
"result": frame.result,
|
|
347
|
+
"error": frame.error,
|
|
348
|
+
}
|
|
349
|
+
)
|
|
350
|
+
elif isinstance(frame, ServerUserTranscriptFrame):
|
|
351
|
+
payload.update({"text": frame.text, "is_final": frame.is_final})
|
|
352
|
+
elif isinstance(frame, ServerUsageFrame):
|
|
353
|
+
payload["usage"] = frame.usage
|
|
354
|
+
elif isinstance(frame, ServerToolConfirmationFrame):
|
|
355
|
+
payload.update(
|
|
356
|
+
{
|
|
357
|
+
"tool_call_id": frame.tool_call_id,
|
|
358
|
+
"tool_name": frame.tool_name,
|
|
359
|
+
"prompt": frame.prompt,
|
|
360
|
+
"arguments": frame.arguments,
|
|
361
|
+
"timeout_sec": frame.timeout_sec,
|
|
362
|
+
}
|
|
363
|
+
)
|
|
364
|
+
elif isinstance(frame, ServerElicitationFrame):
|
|
365
|
+
payload.update(
|
|
366
|
+
{
|
|
367
|
+
"elicitation_id": frame.elicitation_id,
|
|
368
|
+
"prompt": frame.prompt,
|
|
369
|
+
"schema": frame.schema,
|
|
370
|
+
"timeout_sec": frame.timeout_sec,
|
|
371
|
+
}
|
|
372
|
+
)
|
|
373
|
+
elif isinstance(frame, ServerErrorFrame):
|
|
374
|
+
payload["error"] = frame.error
|
|
375
|
+
elif isinstance(frame, ServerSessionEndedFrame):
|
|
376
|
+
payload["reason"] = frame.reason
|
|
377
|
+
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
class ProtocolError(Exception):
|
|
381
|
+
"""Malformed realtime frame."""
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
__all__ = [
|
|
385
|
+
"ClientAudioStreamEndFrame",
|
|
386
|
+
"ClientCloseFrame",
|
|
387
|
+
"ClientConnectFrame",
|
|
388
|
+
"ClientElicitationResponseFrame",
|
|
389
|
+
"ClientFrame",
|
|
390
|
+
"ClientInterruptFrame",
|
|
391
|
+
"ClientTextFrame",
|
|
392
|
+
"ClientToolConfirmationResponseFrame",
|
|
393
|
+
"ProtocolError",
|
|
394
|
+
"ServerAudioFrame",
|
|
395
|
+
"ServerConnectedFrame",
|
|
396
|
+
"ServerElicitationFrame",
|
|
397
|
+
"ServerErrorFrame",
|
|
398
|
+
"ServerFrame",
|
|
399
|
+
"ServerInterruptedFrame",
|
|
400
|
+
"ServerSessionEndedFrame",
|
|
401
|
+
"ServerTextDeltaFrame",
|
|
402
|
+
"ServerToolCallFrame",
|
|
403
|
+
"ServerToolConfirmationFrame",
|
|
404
|
+
"ServerToolResultFrame",
|
|
405
|
+
"ServerTurnBoundaryFrame",
|
|
406
|
+
"ServerUsageFrame",
|
|
407
|
+
"ServerUserTranscriptFrame",
|
|
408
|
+
"audio_duration_sec",
|
|
409
|
+
"encode_server_frame",
|
|
410
|
+
"parse_client_frame",
|
|
411
|
+
"parse_server_frame",
|
|
412
|
+
]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Gateway server entrypoint for the realtime WebSocket harness.
|
|
2
|
+
|
|
3
|
+
Run locally with:
|
|
4
|
+
|
|
5
|
+
python -m monkeybot.gateway.realtime_main
|
|
6
|
+
|
|
7
|
+
This serves both the existing SSE routes and the realtime WebSocket endpoint at
|
|
8
|
+
``WS /sessions/{session_id}/realtime``. The existing SSE-only entrypoint remains at
|
|
9
|
+
:mod:`monkeybot.gateway.main`.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
import os
|
|
16
|
+
|
|
17
|
+
from monkeybot.core.logging_utils import normalize_log_level
|
|
18
|
+
from monkeybot.gateway.bootstrap import ensure_gateway_runtime_env
|
|
19
|
+
|
|
20
|
+
ensure_gateway_runtime_env()
|
|
21
|
+
|
|
22
|
+
logging.basicConfig(
|
|
23
|
+
level=normalize_log_level(os.getenv("LOG_LEVEL")),
|
|
24
|
+
format="%(levelname)s:%(name)s:%(message)s",
|
|
25
|
+
)
|
|
26
|
+
# Suppress chatty third-party loggers regardless of LOG_LEVEL.
|
|
27
|
+
for _noisy in ("httpx", "httpcore", "urllib3", "google_genai", "google.auth"):
|
|
28
|
+
logging.getLogger(_noisy).setLevel(logging.WARNING)
|
|
29
|
+
|
|
30
|
+
from monkeybot.gateway.realtime.app import app # noqa: E402
|
|
31
|
+
|
|
32
|
+
__all__ = ["app"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
if __name__ == "__main__":
|
|
36
|
+
import uvicorn
|
|
37
|
+
|
|
38
|
+
port = int(os.getenv("PORT", os.getenv("GATEWAY_PORT", "8000")))
|
|
39
|
+
log_level = os.getenv("LOG_LEVEL", "info").lower()
|
|
40
|
+
timeout_graceful_shutdown = int(os.getenv("GRACEFUL_SHUTDOWN_TIMEOUT_SEC", "5"))
|
|
41
|
+
|
|
42
|
+
uvicorn.run(
|
|
43
|
+
app,
|
|
44
|
+
host="0.0.0.0",
|
|
45
|
+
port=port,
|
|
46
|
+
log_level=log_level,
|
|
47
|
+
access_log=True,
|
|
48
|
+
timeout_graceful_shutdown=timeout_graceful_shutdown,
|
|
49
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""SSE gateway: session event bus, framing, and HTTP routes (v2 API)."""
|