crabcode 0.1.6__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crabcode-0.1.6.dist-info/METADATA +63 -0
- crabcode-0.1.6.dist-info/RECORD +205 -0
- crabcode-0.1.6.dist-info/WHEEL +4 -0
- crabcode-0.1.6.dist-info/entry_points.txt +2 -0
- crabcode_cli/__init__.py +7 -0
- crabcode_cli/__main__.py +4 -0
- crabcode_cli/_native/crabcode-computer-use-host.exe +0 -0
- crabcode_cli/_native/runtime.json +6 -0
- crabcode_cli/app.py +776 -0
- crabcode_cli/banner.py +39 -0
- crabcode_cli/computer_use.py +286 -0
- crabcode_cli/model_manager.py +649 -0
- crabcode_cli/model_picker.py +168 -0
- crabcode_cli/pipe.py +148 -0
- crabcode_cli/render/__init__.py +0 -0
- crabcode_cli/render/messages.py +19 -0
- crabcode_cli/repl.py +4841 -0
- crabcode_cli/session_picker.py +239 -0
- crabcode_core/__init__.py +7 -0
- crabcode_core/_version.py +3 -0
- crabcode_core/_windows_process.py +68 -0
- crabcode_core/agent_manager.py +1731 -0
- crabcode_core/api/__init__.py +20 -0
- crabcode_core/api/anthropic_adapter.py +648 -0
- crabcode_core/api/azure_adapter.py +69 -0
- crabcode_core/api/base.py +233 -0
- crabcode_core/api/codex_adapter.py +1327 -0
- crabcode_core/api/error_logging.py +132 -0
- crabcode_core/api/gemini_adapter.py +26 -0
- crabcode_core/api/model_info.py +257 -0
- crabcode_core/api/network.py +62 -0
- crabcode_core/api/ollama_adapter.py +27 -0
- crabcode_core/api/openai_adapter.py +323 -0
- crabcode_core/api/registry.py +58 -0
- crabcode_core/command_line.py +18 -0
- crabcode_core/compact/__init__.py +17 -0
- crabcode_core/compact/compact.py +514 -0
- crabcode_core/compact/context_tokens.py +165 -0
- crabcode_core/computer_use_mode.py +38 -0
- crabcode_core/computer_use_observation.py +144 -0
- crabcode_core/config/__init__.py +3 -0
- crabcode_core/config/claudemd.py +61 -0
- crabcode_core/config/manager.py +237 -0
- crabcode_core/config/model_catalog.py +256 -0
- crabcode_core/events.py +4896 -0
- crabcode_core/file_lock.py +81 -0
- crabcode_core/filesystem.py +22 -0
- crabcode_core/goal.py +149 -0
- crabcode_core/hooks/__init__.py +5 -0
- crabcode_core/hooks/manager.py +290 -0
- crabcode_core/image_io.py +111 -0
- crabcode_core/io_worker.py +211 -0
- crabcode_core/logging_utils.py +168 -0
- crabcode_core/lsp/__init__.py +6 -0
- crabcode_core/lsp/client.py +786 -0
- crabcode_core/lsp/diagnostics.py +150 -0
- crabcode_core/lsp/manager.py +583 -0
- crabcode_core/mcp/__init__.py +4 -0
- crabcode_core/mcp/client.py +169 -0
- crabcode_core/mcp/config.py +57 -0
- crabcode_core/path_validation.py +34 -0
- crabcode_core/paths.py +24 -0
- crabcode_core/peer/__init__.py +10 -0
- crabcode_core/peer/runtime.py +518 -0
- crabcode_core/permissions/__init__.py +7 -0
- crabcode_core/permissions/ai_reviewer.py +184 -0
- crabcode_core/permissions/manager.py +187 -0
- crabcode_core/plan/__init__.py +5 -0
- crabcode_core/plan/executor.py +270 -0
- crabcode_core/plan/types.py +169 -0
- crabcode_core/prompts/__init__.py +0 -0
- crabcode_core/prompts/blocks.py +10 -0
- crabcode_core/prompts/context.py +229 -0
- crabcode_core/prompts/library.py +92 -0
- crabcode_core/prompts/profile.py +79 -0
- crabcode_core/prompts/system.py +356 -0
- crabcode_core/prompts/templates.py +104 -0
- crabcode_core/query/__init__.py +0 -0
- crabcode_core/query/computer_use_history.py +83 -0
- crabcode_core/query/loop.py +2446 -0
- crabcode_core/query/retry.py +116 -0
- crabcode_core/schedule/__init__.py +13 -0
- crabcode_core/schedule/manager.py +631 -0
- crabcode_core/schedule/models.py +204 -0
- crabcode_core/schedule/store.py +558 -0
- crabcode_core/schedule/timing.py +287 -0
- crabcode_core/session/__init__.py +12 -0
- crabcode_core/session/export.py +141 -0
- crabcode_core/session/meta_db.py +684 -0
- crabcode_core/session/storage.py +2032 -0
- crabcode_core/session/title_gen.py +52 -0
- crabcode_core/skills/__init__.py +6 -0
- crabcode_core/skills/context.py +23 -0
- crabcode_core/skills/loader.py +214 -0
- crabcode_core/skills/matcher.py +203 -0
- crabcode_core/snapshot/__init__.py +21 -0
- crabcode_core/snapshot/snapshot.py +436 -0
- crabcode_core/snapshot/tracker.py +359 -0
- crabcode_core/subprocess_utils.py +260 -0
- crabcode_core/team/__init__.py +27 -0
- crabcode_core/team/bridge.py +66 -0
- crabcode_core/team/inbox.py +237 -0
- crabcode_core/team/manager.py +1660 -0
- crabcode_core/team/message_bus.py +821 -0
- crabcode_core/team/models.py +191 -0
- crabcode_core/team/recovery.py +192 -0
- crabcode_core/text_io.py +52 -0
- crabcode_core/tools/__init__.py +112 -0
- crabcode_core/tools/_input_helpers.py +39 -0
- crabcode_core/tools/_input_schema.py +130 -0
- crabcode_core/tools/agent.py +413 -0
- crabcode_core/tools/apply_patch.py +587 -0
- crabcode_core/tools/ask_user.py +122 -0
- crabcode_core/tools/bash.py +154 -0
- crabcode_core/tools/browser.py +678 -0
- crabcode_core/tools/checklist.py +241 -0
- crabcode_core/tools/checkpoint.py +78 -0
- crabcode_core/tools/computer_use.py +599 -0
- crabcode_core/tools/diff_utils.py +123 -0
- crabcode_core/tools/file_edit.py +240 -0
- crabcode_core/tools/file_read.py +187 -0
- crabcode_core/tools/file_write.py +154 -0
- crabcode_core/tools/glob.py +123 -0
- crabcode_core/tools/goal.py +134 -0
- crabcode_core/tools/grep.py +247 -0
- crabcode_core/tools/image.py +173 -0
- crabcode_core/tools/image_generate.py +115 -0
- crabcode_core/tools/lint.py +388 -0
- crabcode_core/tools/loading.py +187 -0
- crabcode_core/tools/memory.py +456 -0
- crabcode_core/tools/monitor.py +677 -0
- crabcode_core/tools/peer.py +103 -0
- crabcode_core/tools/revert.py +90 -0
- crabcode_core/tools/schedule.py +683 -0
- crabcode_core/tools/skill.py +108 -0
- crabcode_core/tools/switch_mode.py +213 -0
- crabcode_core/tools/team.py +329 -0
- crabcode_core/tools/tool_search.py +64 -0
- crabcode_core/tools/web_search.py +444 -0
- crabcode_core/turn_details.py +64 -0
- crabcode_core/types/__init__.py +95 -0
- crabcode_core/types/config.py +434 -0
- crabcode_core/types/event.py +334 -0
- crabcode_core/types/message.py +263 -0
- crabcode_core/types/tool.py +208 -0
- crabcode_core/usage.py +289 -0
- crabcode_core/utf8_sanitize.py +29 -0
- crabcode_debugger/__init__.py +5 -0
- crabcode_debugger/adapters.py +332 -0
- crabcode_debugger/dap.py +304 -0
- crabcode_debugger/memory.py +1806 -0
- crabcode_debugger/process.py +707 -0
- crabcode_debugger/runtime.py +58 -0
- crabcode_debugger/sessions.py +387 -0
- crabcode_debugger/tool.py +731 -0
- crabcode_gateway/__init__.py +18 -0
- crabcode_gateway/acp/__init__.py +13 -0
- crabcode_gateway/acp/agent.py +1581 -0
- crabcode_gateway/acp/session.py +114 -0
- crabcode_gateway/acp/transport.py +39 -0
- crabcode_gateway/acp/types.py +90 -0
- crabcode_gateway/adapter.py +30 -0
- crabcode_gateway/auth.py +139 -0
- crabcode_gateway/computer_use.py +364 -0
- crabcode_gateway/document_engine.py +637 -0
- crabcode_gateway/document_worker.py +301 -0
- crabcode_gateway/event_bus.py +655 -0
- crabcode_gateway/grpc/__init__.py +0 -0
- crabcode_gateway/grpc/crabcode_pb2.py +70 -0
- crabcode_gateway/grpc/crabcode_pb2_grpc.py +474 -0
- crabcode_gateway/grpc/proto/crabcode.proto +160 -0
- crabcode_gateway/grpc/server.py +765 -0
- crabcode_gateway/middleware.py +344 -0
- crabcode_gateway/protocol.py +17 -0
- crabcode_gateway/routes/__init__.py +0 -0
- crabcode_gateway/routes/agent.py +392 -0
- crabcode_gateway/routes/auth.py +150 -0
- crabcode_gateway/routes/computer_use.py +101 -0
- crabcode_gateway/routes/config.py +1698 -0
- crabcode_gateway/routes/document.py +1810 -0
- crabcode_gateway/routes/event.py +3801 -0
- crabcode_gateway/routes/git_review.py +232 -0
- crabcode_gateway/routes/health.py +14 -0
- crabcode_gateway/routes/peer.py +78 -0
- crabcode_gateway/routes/permission.py +121 -0
- crabcode_gateway/routes/schedule.py +300 -0
- crabcode_gateway/routes/session.py +1496 -0
- crabcode_gateway/routes/snapshot.py +211 -0
- crabcode_gateway/routes/tasks.py +318 -0
- crabcode_gateway/routes/team.py +392 -0
- crabcode_gateway/routes/usage.py +49 -0
- crabcode_gateway/routes/workspace.py +299 -0
- crabcode_gateway/safe_file.py +63 -0
- crabcode_gateway/schemas.py +1820 -0
- crabcode_gateway/server.py +514 -0
- crabcode_gateway/session_registry.py +36 -0
- crabcode_gateway/task_registry.py +708 -0
- crabcode_gateway/turn_details.py +60 -0
- crabcode_search/__init__.py +11 -0
- crabcode_search/background.py +368 -0
- crabcode_search/chunker.py +517 -0
- crabcode_search/embedder.py +497 -0
- crabcode_search/indexer.py +215 -0
- crabcode_search/store.py +222 -0
- crabcode_search/tool.py +268 -0
|
@@ -0,0 +1,1660 @@
|
|
|
1
|
+
"""TeamManager — core runtime for managing agent teams.
|
|
2
|
+
|
|
3
|
+
Owns team lifecycle, teammate state machines, task boards, and message routing.
|
|
4
|
+
Delegates agent spawn/wait/cancel to the existing AgentManager.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import hashlib
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Awaitable, Callable
|
|
14
|
+
|
|
15
|
+
from crabcode_core.logging_utils import get_logger
|
|
16
|
+
from crabcode_core.paths import get_config_home
|
|
17
|
+
from crabcode_core.team.message_bus import TeamMessageBus
|
|
18
|
+
from crabcode_core.team.models import (
|
|
19
|
+
BridgePolicy,
|
|
20
|
+
CrossTeamMessage,
|
|
21
|
+
TaskItem,
|
|
22
|
+
TaskStatus,
|
|
23
|
+
TeamConfig,
|
|
24
|
+
TeamMessage,
|
|
25
|
+
TeamState,
|
|
26
|
+
TeammateInfo,
|
|
27
|
+
TeammateRole,
|
|
28
|
+
TeammateState,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
logger = get_logger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class TeamRuntime:
|
|
36
|
+
"""Runtime state for a single team."""
|
|
37
|
+
|
|
38
|
+
team_id: str
|
|
39
|
+
config: TeamConfig
|
|
40
|
+
message_bus: TeamMessageBus
|
|
41
|
+
task_board: list[TaskItem] = field(default_factory=list)
|
|
42
|
+
teammates: dict[str, TeammateInfo] = field(default_factory=dict)
|
|
43
|
+
task_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
|
44
|
+
# Serializes membership changes (especially spawn + capacity checks).
|
|
45
|
+
membership_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
|
46
|
+
# Spawning an AgentManager run emits lifecycle events that re-enter this
|
|
47
|
+
# manager. Keep the potentially-awaiting spawn outside membership_lock,
|
|
48
|
+
# while this lock still serializes capacity reservations.
|
|
49
|
+
spawn_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
|
50
|
+
pending_spawns: int = 0
|
|
51
|
+
# Serializes manager-owned bus operations with unregister/cleanup without
|
|
52
|
+
# blocking lifecycle callbacks that only need membership_lock.
|
|
53
|
+
bus_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
|
54
|
+
state: TeamState = TeamState.FORMING
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class TeamManager:
|
|
58
|
+
"""Owns all teams for a CoreSession.
|
|
59
|
+
|
|
60
|
+
Uses the existing AgentManager for spawn/wait/cancel operations.
|
|
61
|
+
Manages teammate state machines, message routing, and task boards.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
*,
|
|
67
|
+
agent_manager: Any,
|
|
68
|
+
settings: Any,
|
|
69
|
+
event_sink: Callable[[Any], Awaitable[None]],
|
|
70
|
+
cwd: str = ".",
|
|
71
|
+
session_id: str = "",
|
|
72
|
+
) -> None:
|
|
73
|
+
self._agent_manager = agent_manager
|
|
74
|
+
self._settings = settings
|
|
75
|
+
self._event_sink = event_sink
|
|
76
|
+
self._cwd = cwd
|
|
77
|
+
self._session_id = session_id
|
|
78
|
+
self._teams: dict[str, TeamRuntime] = {}
|
|
79
|
+
# Serializes team creation/removal so a replacement cannot delete the
|
|
80
|
+
# new team's inbox while the old team's cleanup is still running.
|
|
81
|
+
self._teams_lock = asyncio.Lock()
|
|
82
|
+
self._shutting_down: set[str] = set()
|
|
83
|
+
# A manager is reusable while close() is draining cancellation
|
|
84
|
+
# callbacks (some integrations intentionally create a replacement team
|
|
85
|
+
# from such a callback), but becomes terminal once close has completed.
|
|
86
|
+
# This prevents stale ToolContext instances from resurrecting a closed
|
|
87
|
+
# session's team runtime.
|
|
88
|
+
self._closed = False
|
|
89
|
+
# Keep shutdown work independent from the caller's task. Cancelling a
|
|
90
|
+
# tool invocation must not abandon agent cancellation or inbox cleanup.
|
|
91
|
+
self._shutdown_tasks: dict[str, asyncio.Task[None]] = {}
|
|
92
|
+
self._bridges: dict[tuple[str, str], BridgePolicy] = {}
|
|
93
|
+
self._cross_team_messages: list[CrossTeamMessage] = []
|
|
94
|
+
# Messages received while a teammate is running are held here until
|
|
95
|
+
# its current turn settles. AgentManager.send_input(..., interrupt=False)
|
|
96
|
+
# intentionally rejects busy runs, so without this buffer messages were
|
|
97
|
+
# persisted but never shown to the model.
|
|
98
|
+
self._pending_messages: dict[str, list[tuple[str, str]]] = {}
|
|
99
|
+
self._delivery_locks: dict[str, asyncio.Lock] = {}
|
|
100
|
+
self._pending_delivery_tasks: dict[str, asyncio.Task[None]] = {}
|
|
101
|
+
self._teammate_watch_tasks: dict[str, asyncio.Task[None]] = {}
|
|
102
|
+
# Preserve recovery details without widening recover()'s historical
|
|
103
|
+
# public return shape ({team_id, agent_id}). recover_teams() uses this
|
|
104
|
+
# to report whether an interrupted member was busy or cancelling.
|
|
105
|
+
self._last_recovered_states: dict[tuple[str, str], str] = {}
|
|
106
|
+
|
|
107
|
+
# ------------------------------------------------------------------
|
|
108
|
+
# Team lifecycle
|
|
109
|
+
# ------------------------------------------------------------------
|
|
110
|
+
|
|
111
|
+
async def create_team(
|
|
112
|
+
self,
|
|
113
|
+
name: str,
|
|
114
|
+
*,
|
|
115
|
+
max_teammates: int | None = None,
|
|
116
|
+
config_override: TeamConfig | None = None,
|
|
117
|
+
) -> str:
|
|
118
|
+
async with self._teams_lock:
|
|
119
|
+
if self._closed:
|
|
120
|
+
raise RuntimeError("Team manager is closed")
|
|
121
|
+
return await self._create_team(
|
|
122
|
+
name,
|
|
123
|
+
max_teammates=max_teammates,
|
|
124
|
+
config_override=config_override,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
async def _create_team(
|
|
128
|
+
self,
|
|
129
|
+
name: str,
|
|
130
|
+
*,
|
|
131
|
+
max_teammates: int | None = None,
|
|
132
|
+
config_override: TeamConfig | None = None,
|
|
133
|
+
) -> str:
|
|
134
|
+
"""Create a new team and return its team_id."""
|
|
135
|
+
if not isinstance(name, str) or not name.strip():
|
|
136
|
+
raise ValueError("Team name must not be empty")
|
|
137
|
+
# Team IDs are used as directory names by the inbox store. Reject
|
|
138
|
+
# separators here so callers cannot escape the configured inbox root.
|
|
139
|
+
if "\x00" in name or "/" in name or "\\" in name or name in {".", ".."}:
|
|
140
|
+
raise ValueError("Team name cannot contain path separators")
|
|
141
|
+
team_id = name # Use name as ID for simplicity
|
|
142
|
+
|
|
143
|
+
if team_id in self._teams:
|
|
144
|
+
raise ValueError(f"Team '{team_id}' already exists")
|
|
145
|
+
if team_id in self._shutting_down:
|
|
146
|
+
raise ValueError(f"Team '{team_id}' is shutting down")
|
|
147
|
+
|
|
148
|
+
team_settings = getattr(self._settings, "team", None)
|
|
149
|
+
if max_teammates is not None and max_teammates <= 0:
|
|
150
|
+
raise ValueError("max_teammates must be greater than zero")
|
|
151
|
+
if config_override is not None:
|
|
152
|
+
# Keep the public `name` argument authoritative while preserving
|
|
153
|
+
# compatibility with callers that reuse a config object whose
|
|
154
|
+
# display name differs.
|
|
155
|
+
config = (
|
|
156
|
+
config_override
|
|
157
|
+
if config_override.name == name
|
|
158
|
+
else config_override.model_copy(update={"name": name})
|
|
159
|
+
)
|
|
160
|
+
else:
|
|
161
|
+
configured_policy = getattr(team_settings, "bridge_policy", BridgePolicy.DENY)
|
|
162
|
+
try:
|
|
163
|
+
configured_policy = BridgePolicy(configured_policy)
|
|
164
|
+
except (TypeError, ValueError):
|
|
165
|
+
logger.warning("Unknown team bridge policy %r; defaulting to deny", configured_policy)
|
|
166
|
+
configured_policy = BridgePolicy.DENY
|
|
167
|
+
config = TeamConfig(
|
|
168
|
+
name=name,
|
|
169
|
+
max_teammates=(
|
|
170
|
+
max_teammates
|
|
171
|
+
if max_teammates is not None
|
|
172
|
+
else (team_settings.max_teammates if team_settings else 8)
|
|
173
|
+
),
|
|
174
|
+
backpressure_queue_size=team_settings.backpressure_queue_size if team_settings else 100,
|
|
175
|
+
max_message_size_bytes=team_settings.max_message_size_bytes if team_settings else 10_000,
|
|
176
|
+
bridge_policy=configured_policy,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
storage_root = (
|
|
180
|
+
self._scope_inbox_root(Path(config.inbox_dir), custom=True)
|
|
181
|
+
if config.inbox_dir
|
|
182
|
+
else self._resolve_inbox_root()
|
|
183
|
+
)
|
|
184
|
+
inject_fn = self._make_inject_fn()
|
|
185
|
+
wake_fn = self._make_wake_fn()
|
|
186
|
+
|
|
187
|
+
bus = TeamMessageBus(
|
|
188
|
+
team_name=team_id,
|
|
189
|
+
config=config,
|
|
190
|
+
inject_fn=inject_fn,
|
|
191
|
+
wake_fn=wake_fn,
|
|
192
|
+
storage_root=storage_root,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
runtime = TeamRuntime(
|
|
196
|
+
team_id=team_id,
|
|
197
|
+
config=config,
|
|
198
|
+
message_bus=bus,
|
|
199
|
+
state=TeamState.ACTIVE,
|
|
200
|
+
)
|
|
201
|
+
self._teams[team_id] = runtime
|
|
202
|
+
return team_id
|
|
203
|
+
|
|
204
|
+
async def shutdown_team(self, team_id: str) -> bool:
|
|
205
|
+
"""Shutdown a team: cancel all teammates, delete inboxes."""
|
|
206
|
+
# Reserve the name while holding only the map lock. The expensive
|
|
207
|
+
# cleanup happens in a separate task, so agent cancellation callbacks
|
|
208
|
+
# can safely call back into TeamManager (including create_team).
|
|
209
|
+
async with self._teams_lock:
|
|
210
|
+
runtime = self._teams.get(team_id)
|
|
211
|
+
if runtime is None or team_id in self._shutting_down:
|
|
212
|
+
return False
|
|
213
|
+
self._shutting_down.add(team_id)
|
|
214
|
+
cleanup_task = asyncio.create_task(self._finish_shutdown(team_id, runtime))
|
|
215
|
+
self._shutdown_tasks[team_id] = cleanup_task
|
|
216
|
+
|
|
217
|
+
try:
|
|
218
|
+
await asyncio.shield(cleanup_task)
|
|
219
|
+
except asyncio.CancelledError:
|
|
220
|
+
# shield() leaves the cleanup task running. Wait for it before
|
|
221
|
+
# propagating cancellation so callers never observe a half-closed
|
|
222
|
+
# runtime or a prematurely reusable team name.
|
|
223
|
+
try:
|
|
224
|
+
await asyncio.shield(cleanup_task)
|
|
225
|
+
except asyncio.CancelledError:
|
|
226
|
+
# A second cancellation request should not resurrect the team;
|
|
227
|
+
# the task remains tracked and close() will await it.
|
|
228
|
+
pass
|
|
229
|
+
raise
|
|
230
|
+
return True
|
|
231
|
+
|
|
232
|
+
async def _finish_shutdown(self, team_id: str, runtime: TeamRuntime) -> None:
|
|
233
|
+
"""Perform the non-cancellable portion of team shutdown."""
|
|
234
|
+
agent_infos: list[tuple[str, TeammateInfo]] = []
|
|
235
|
+
shutdown_events: list[tuple[str, str, str]] = []
|
|
236
|
+
try:
|
|
237
|
+
# Mark the runtime and members before cancellation. Keep the
|
|
238
|
+
# membership entries until the final phase so send/remove calls
|
|
239
|
+
# fail cleanly and state watchers cannot turn members READY again.
|
|
240
|
+
async with runtime.membership_lock:
|
|
241
|
+
runtime.state = TeamState.SHUTDOWN
|
|
242
|
+
agent_infos = list(runtime.teammates.items())
|
|
243
|
+
for agent_id, info in agent_infos:
|
|
244
|
+
old_state = info.state.value
|
|
245
|
+
info.transition_to(TeammateState.SHUTDOWN, force=True)
|
|
246
|
+
shutdown_events.append((agent_id, old_state, info.role.value))
|
|
247
|
+
|
|
248
|
+
# An add_teammate call may have reserved capacity and be waiting
|
|
249
|
+
# for AgentManager.spawn_agent outside membership_lock. Wait for
|
|
250
|
+
# that transaction to either commit or roll back before deleting
|
|
251
|
+
# the runtime/inboxes, so close() cannot return with an orphan run.
|
|
252
|
+
async with runtime.spawn_lock:
|
|
253
|
+
pass
|
|
254
|
+
|
|
255
|
+
# Stop delivery/watch tasks before signalling agent completion. A
|
|
256
|
+
# waiter can otherwise observe the completion event and start a new
|
|
257
|
+
# send_input run after cancel_agent has returned.
|
|
258
|
+
await asyncio.gather(
|
|
259
|
+
*(self._cancel_agent_tasks(agent_id) for agent_id, _info in agent_infos),
|
|
260
|
+
return_exceptions=True,
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
# Agent cancellation can run tool cleanup that calls back into this
|
|
264
|
+
# manager, so it must remain outside membership_lock.
|
|
265
|
+
for agent_id, _info in agent_infos:
|
|
266
|
+
try:
|
|
267
|
+
await self._agent_manager.cancel_agent(agent_id)
|
|
268
|
+
except Exception:
|
|
269
|
+
logger.warning(
|
|
270
|
+
"Failed to cancel agent %s during team shutdown",
|
|
271
|
+
agent_id,
|
|
272
|
+
exc_info=True,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
# Unregister only after manager-owned bus operations have finished.
|
|
276
|
+
# bus_lock is deliberately distinct from membership_lock so a
|
|
277
|
+
# delivery callback can update teammate state while a send is in
|
|
278
|
+
# flight.
|
|
279
|
+
async with runtime.bus_lock:
|
|
280
|
+
# A delivery callback can have queued work after the first
|
|
281
|
+
# cancellation sweep but before the bus operation committed.
|
|
282
|
+
# Drain those tasks once more while the runtime is fenced,
|
|
283
|
+
# before cancelling the underlying agents.
|
|
284
|
+
await asyncio.gather(
|
|
285
|
+
*(self._cancel_agent_tasks(agent_id) for agent_id, _info in agent_infos),
|
|
286
|
+
return_exceptions=True,
|
|
287
|
+
)
|
|
288
|
+
async with runtime.membership_lock:
|
|
289
|
+
for agent_id in list(runtime.teammates):
|
|
290
|
+
runtime.message_bus.unregister_agent(agent_id)
|
|
291
|
+
runtime.teammates.clear()
|
|
292
|
+
|
|
293
|
+
for agent_id, old_state, role in shutdown_events:
|
|
294
|
+
await self._emit_team_state(
|
|
295
|
+
team_id,
|
|
296
|
+
agent_id,
|
|
297
|
+
old_state,
|
|
298
|
+
TeammateState.SHUTDOWN.value,
|
|
299
|
+
role,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
# Delete inbox files while the name remains reserved, preventing a
|
|
303
|
+
# replacement team from sharing this directory during cleanup.
|
|
304
|
+
await runtime.message_bus.delete_team_inboxes()
|
|
305
|
+
finally:
|
|
306
|
+
async with self._teams_lock:
|
|
307
|
+
if self._teams.get(team_id) is runtime:
|
|
308
|
+
del self._teams[team_id]
|
|
309
|
+
self._shutting_down.discard(team_id)
|
|
310
|
+
current = asyncio.current_task()
|
|
311
|
+
if self._shutdown_tasks.get(team_id) is current:
|
|
312
|
+
self._shutdown_tasks.pop(team_id, None)
|
|
313
|
+
|
|
314
|
+
# ------------------------------------------------------------------
|
|
315
|
+
# Teammate management
|
|
316
|
+
# ------------------------------------------------------------------
|
|
317
|
+
|
|
318
|
+
async def add_teammate(
|
|
319
|
+
self,
|
|
320
|
+
team_id: str,
|
|
321
|
+
*,
|
|
322
|
+
role: TeammateRole = TeammateRole.WORKER,
|
|
323
|
+
prompt: str,
|
|
324
|
+
name: str | None = None,
|
|
325
|
+
model_profile: str | None = None,
|
|
326
|
+
) -> str:
|
|
327
|
+
"""Add a teammate to the team. Spawns a sub-agent via AgentManager."""
|
|
328
|
+
runtime = self._teams.get(team_id)
|
|
329
|
+
if runtime is None:
|
|
330
|
+
raise ValueError(f"Team '{team_id}' not found")
|
|
331
|
+
|
|
332
|
+
if not isinstance(prompt, str) or not prompt.strip():
|
|
333
|
+
raise ValueError("Teammate prompt must not be empty")
|
|
334
|
+
|
|
335
|
+
# Reserve capacity under the membership lock, then spawn outside it.
|
|
336
|
+
# AgentManager emits queued lifecycle events while spawning; those
|
|
337
|
+
# events re-enter handle_agent_event(), which also needs
|
|
338
|
+
# membership_lock. Holding it across the await would deadlock a real
|
|
339
|
+
# session. spawn_lock serializes reservations so concurrent calls still
|
|
340
|
+
# cannot exceed max_teammates.
|
|
341
|
+
agent_id: str | None = None
|
|
342
|
+
reservation_held = False
|
|
343
|
+
try:
|
|
344
|
+
async with runtime.spawn_lock:
|
|
345
|
+
async with runtime.membership_lock:
|
|
346
|
+
if runtime.state != TeamState.ACTIVE or team_id in self._shutting_down:
|
|
347
|
+
raise ValueError(f"Team '{team_id}' is not active")
|
|
348
|
+
if (
|
|
349
|
+
len(runtime.teammates) + runtime.pending_spawns
|
|
350
|
+
>= runtime.config.max_teammates
|
|
351
|
+
):
|
|
352
|
+
raise ValueError(
|
|
353
|
+
f"Team '{team_id}' is full "
|
|
354
|
+
f"({len(runtime.teammates) + runtime.pending_spawns}/"
|
|
355
|
+
f"{runtime.config.max_teammates})"
|
|
356
|
+
)
|
|
357
|
+
runtime.pending_spawns += 1
|
|
358
|
+
reservation_held = True
|
|
359
|
+
spawn_name = name or f"{role.value}-{len(runtime.teammates)}"
|
|
360
|
+
|
|
361
|
+
try:
|
|
362
|
+
# Spawn outside membership_lock so lifecycle callbacks can
|
|
363
|
+
# update the team state without waiting on this operation.
|
|
364
|
+
agent_id = await self._agent_manager.spawn_agent(
|
|
365
|
+
prompt=prompt,
|
|
366
|
+
subagent_type="generalPurpose",
|
|
367
|
+
name=spawn_name,
|
|
368
|
+
model_profile=model_profile,
|
|
369
|
+
depth=1,
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
async with runtime.membership_lock:
|
|
373
|
+
runtime.pending_spawns -= 1
|
|
374
|
+
reservation_held = False
|
|
375
|
+
if (
|
|
376
|
+
runtime.state != TeamState.ACTIVE
|
|
377
|
+
or team_id in self._shutting_down
|
|
378
|
+
or self._teams.get(team_id) is not runtime
|
|
379
|
+
):
|
|
380
|
+
raise ValueError(f"Team '{team_id}' is no longer active")
|
|
381
|
+
|
|
382
|
+
# Register with the message bus and create teammate
|
|
383
|
+
# info. If either step fails, the spawned run is
|
|
384
|
+
# cancelled below so it cannot become an orphan.
|
|
385
|
+
runtime.message_bus.register_agent(agent_id)
|
|
386
|
+
info = TeammateInfo(
|
|
387
|
+
agent_id=agent_id,
|
|
388
|
+
role=role,
|
|
389
|
+
state=TeammateState.BUSY,
|
|
390
|
+
model_profile=model_profile,
|
|
391
|
+
name=name,
|
|
392
|
+
)
|
|
393
|
+
runtime.teammates[agent_id] = info
|
|
394
|
+
self._delivery_locks.setdefault(agent_id, asyncio.Lock())
|
|
395
|
+
except BaseException:
|
|
396
|
+
if reservation_held:
|
|
397
|
+
async with runtime.membership_lock:
|
|
398
|
+
runtime.pending_spawns = max(0, runtime.pending_spawns - 1)
|
|
399
|
+
reservation_held = False
|
|
400
|
+
raise
|
|
401
|
+
|
|
402
|
+
# Emit state event and arm the watcher inside the same rollback
|
|
403
|
+
# boundary. Both operations can be cancelled by a caller; a
|
|
404
|
+
# spawned AgentManager run must never be left behind in that case.
|
|
405
|
+
old_state = TeammateState.IDLE.value
|
|
406
|
+
new_state = TeammateState.BUSY.value
|
|
407
|
+
await self._emit_team_state(team_id, agent_id, old_state, new_state, role.value)
|
|
408
|
+
|
|
409
|
+
# Keep TeamStatus in sync with the underlying AgentManager run.
|
|
410
|
+
# This watcher is also the hand-off point for queued messages.
|
|
411
|
+
self._ensure_teammate_watcher(team_id, agent_id)
|
|
412
|
+
|
|
413
|
+
return agent_id
|
|
414
|
+
except BaseException:
|
|
415
|
+
# Once spawn_agent returned, always attempt rollback. The
|
|
416
|
+
# cancellation may have landed between spawn and bus registration,
|
|
417
|
+
# leaving no teammate entry to key off.
|
|
418
|
+
if agent_id is not None:
|
|
419
|
+
await self._shielded_rollback_teammate(runtime, team_id, agent_id)
|
|
420
|
+
raise
|
|
421
|
+
|
|
422
|
+
async def remove_teammate(self, team_id: str, agent_id: str) -> bool:
|
|
423
|
+
"""Remove a teammate from the team."""
|
|
424
|
+
runtime = self._teams.get(team_id)
|
|
425
|
+
if runtime is None:
|
|
426
|
+
return False
|
|
427
|
+
|
|
428
|
+
async with runtime.bus_lock:
|
|
429
|
+
async with runtime.membership_lock:
|
|
430
|
+
if runtime.state != TeamState.ACTIVE or team_id in self._shutting_down:
|
|
431
|
+
return False
|
|
432
|
+
info = runtime.teammates.pop(agent_id, None)
|
|
433
|
+
if info is None:
|
|
434
|
+
return False
|
|
435
|
+
|
|
436
|
+
old_state = info.state.value
|
|
437
|
+
info.transition_to(TeammateState.SHUTDOWN, force=True)
|
|
438
|
+
runtime.message_bus.unregister_agent(agent_id)
|
|
439
|
+
|
|
440
|
+
# Stop delivery/state waiters before cancel_agent sets its completion
|
|
441
|
+
# event. Otherwise a pending waiter can wake and restart this removed
|
|
442
|
+
# agent in the gap before task cleanup.
|
|
443
|
+
await self._cancel_agent_tasks(agent_id)
|
|
444
|
+
|
|
445
|
+
# Cancellation can invoke tool cleanup that calls back into this
|
|
446
|
+
# manager, so do it after releasing membership_lock.
|
|
447
|
+
try:
|
|
448
|
+
await self._agent_manager.cancel_agent(agent_id)
|
|
449
|
+
except Exception:
|
|
450
|
+
logger.warning("Failed to cancel agent %s during removal", agent_id, exc_info=True)
|
|
451
|
+
|
|
452
|
+
await self._emit_team_state(team_id, agent_id, old_state, TeammateState.SHUTDOWN.value, info.role.value)
|
|
453
|
+
return True
|
|
454
|
+
|
|
455
|
+
# ------------------------------------------------------------------
|
|
456
|
+
# Messaging
|
|
457
|
+
# ------------------------------------------------------------------
|
|
458
|
+
|
|
459
|
+
async def send_message(
|
|
460
|
+
self,
|
|
461
|
+
team_id: str,
|
|
462
|
+
from_agent: str,
|
|
463
|
+
to_agent: str,
|
|
464
|
+
text: str,
|
|
465
|
+
) -> TeamMessage | None:
|
|
466
|
+
"""Send a message from one teammate to another."""
|
|
467
|
+
runtime = self._teams.get(team_id)
|
|
468
|
+
if runtime is None:
|
|
469
|
+
return None
|
|
470
|
+
if not isinstance(text, str) or not text.strip():
|
|
471
|
+
return None
|
|
472
|
+
# Authorize under membership_lock, but do not hold it across the bus
|
|
473
|
+
# operation. Delivery injection calls back into _mark_teammate_busy,
|
|
474
|
+
# which needs the same lock; holding it here deadlocks idle recipients.
|
|
475
|
+
recipient_epoch: int | None = None
|
|
476
|
+
async with runtime.membership_lock:
|
|
477
|
+
if (
|
|
478
|
+
self._teams.get(team_id) is not runtime
|
|
479
|
+
or runtime.state != TeamState.ACTIVE
|
|
480
|
+
or team_id in self._shutting_down
|
|
481
|
+
):
|
|
482
|
+
return None
|
|
483
|
+
if from_agent and from_agent not in runtime.teammates:
|
|
484
|
+
logger.warning("Message from non-member %s in team %s", from_agent, team_id)
|
|
485
|
+
return None
|
|
486
|
+
if not to_agent or to_agent not in runtime.teammates:
|
|
487
|
+
return None
|
|
488
|
+
# Capture the registration generation. If this member is removed
|
|
489
|
+
# and replaced with the same id while the bus lock is contended,
|
|
490
|
+
# the replacement must not inherit this authorization.
|
|
491
|
+
recipient_epoch = runtime.message_bus.registration_epoch(to_agent)
|
|
492
|
+
if recipient_epoch is None:
|
|
493
|
+
return None
|
|
494
|
+
|
|
495
|
+
# Keep the recipient registered until this already-authorized send
|
|
496
|
+
# reaches its commit point. Shutdown acquires the same lock before
|
|
497
|
+
# unregistering agents, so an in-flight persistence/injection cannot be
|
|
498
|
+
# turned into a spurious failure by teardown.
|
|
499
|
+
async with runtime.bus_lock:
|
|
500
|
+
# Removal takes bus_lock before unregistering. A synchronous
|
|
501
|
+
# re-check here therefore closes the auth-to-commit race without
|
|
502
|
+
# taking membership_lock while bus_lock is held (shutdown uses the
|
|
503
|
+
# opposite nested order in its final cleanup phase).
|
|
504
|
+
if (
|
|
505
|
+
self._teams.get(team_id) is not runtime
|
|
506
|
+
or runtime.state != TeamState.ACTIVE
|
|
507
|
+
or team_id in self._shutting_down
|
|
508
|
+
or to_agent not in runtime.teammates
|
|
509
|
+
or runtime.message_bus.registration_epoch(to_agent) != recipient_epoch
|
|
510
|
+
):
|
|
511
|
+
return None
|
|
512
|
+
msg = await runtime.message_bus.send(
|
|
513
|
+
from_agent=from_agent,
|
|
514
|
+
to_agent=to_agent,
|
|
515
|
+
text=text,
|
|
516
|
+
expected_epoch=recipient_epoch,
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
emit_allowed = False
|
|
520
|
+
if msg:
|
|
521
|
+
async with runtime.membership_lock:
|
|
522
|
+
emit_allowed = not (
|
|
523
|
+
self._teams.get(team_id) is not runtime
|
|
524
|
+
or runtime.state != TeamState.ACTIVE
|
|
525
|
+
or team_id in self._shutting_down
|
|
526
|
+
or to_agent not in runtime.teammates
|
|
527
|
+
)
|
|
528
|
+
if emit_allowed:
|
|
529
|
+
await self._emit_team_message(team_id, msg)
|
|
530
|
+
|
|
531
|
+
return msg
|
|
532
|
+
|
|
533
|
+
async def broadcast(
|
|
534
|
+
self,
|
|
535
|
+
team_id: str,
|
|
536
|
+
from_agent: str,
|
|
537
|
+
text: str,
|
|
538
|
+
) -> list[TeamMessage]:
|
|
539
|
+
"""Broadcast a message to all teammates except the sender."""
|
|
540
|
+
runtime = self._teams.get(team_id)
|
|
541
|
+
if runtime is None:
|
|
542
|
+
return []
|
|
543
|
+
if not isinstance(text, str) or not text.strip():
|
|
544
|
+
return []
|
|
545
|
+
recipient_epochs: dict[str, int] = {}
|
|
546
|
+
async with runtime.membership_lock:
|
|
547
|
+
if (
|
|
548
|
+
self._teams.get(team_id) is not runtime
|
|
549
|
+
or runtime.state != TeamState.ACTIVE
|
|
550
|
+
or team_id in self._shutting_down
|
|
551
|
+
):
|
|
552
|
+
return []
|
|
553
|
+
if from_agent and from_agent not in runtime.teammates:
|
|
554
|
+
logger.warning("Broadcast from non-member %s in team %s", from_agent, team_id)
|
|
555
|
+
return []
|
|
556
|
+
for agent_id in runtime.teammates:
|
|
557
|
+
if agent_id == from_agent:
|
|
558
|
+
continue
|
|
559
|
+
epoch = runtime.message_bus.registration_epoch(agent_id)
|
|
560
|
+
if epoch is not None:
|
|
561
|
+
recipient_epochs[agent_id] = epoch
|
|
562
|
+
|
|
563
|
+
async with runtime.bus_lock:
|
|
564
|
+
# Freeze the authorized roster and registration generations before
|
|
565
|
+
# writing. Calling the bus broadcast helper directly could include
|
|
566
|
+
# a teammate added after authorization.
|
|
567
|
+
if (
|
|
568
|
+
self._teams.get(team_id) is not runtime
|
|
569
|
+
or runtime.state != TeamState.ACTIVE
|
|
570
|
+
or team_id in self._shutting_down
|
|
571
|
+
or any(
|
|
572
|
+
agent_id not in runtime.teammates
|
|
573
|
+
or runtime.message_bus.registration_epoch(agent_id) != epoch
|
|
574
|
+
for agent_id, epoch in recipient_epochs.items()
|
|
575
|
+
)
|
|
576
|
+
):
|
|
577
|
+
return []
|
|
578
|
+
messages: list[TeamMessage] = []
|
|
579
|
+
for agent_id, epoch in recipient_epochs.items():
|
|
580
|
+
message = await runtime.message_bus.send(
|
|
581
|
+
from_agent=from_agent,
|
|
582
|
+
to_agent=agent_id,
|
|
583
|
+
text=text,
|
|
584
|
+
expected_epoch=epoch,
|
|
585
|
+
)
|
|
586
|
+
if message is not None:
|
|
587
|
+
messages.append(message)
|
|
588
|
+
|
|
589
|
+
emit_allowed = True
|
|
590
|
+
async with runtime.membership_lock:
|
|
591
|
+
if (
|
|
592
|
+
self._teams.get(team_id) is not runtime
|
|
593
|
+
or runtime.state != TeamState.ACTIVE
|
|
594
|
+
or team_id in self._shutting_down
|
|
595
|
+
):
|
|
596
|
+
emit_allowed = False
|
|
597
|
+
|
|
598
|
+
for msg in messages:
|
|
599
|
+
if emit_allowed:
|
|
600
|
+
await self._emit_team_message(team_id, msg)
|
|
601
|
+
|
|
602
|
+
return messages
|
|
603
|
+
|
|
604
|
+
def get_unread_messages(self, team_id: str, agent_id: str) -> list[TeamMessage]:
|
|
605
|
+
"""Get unread messages for a teammate."""
|
|
606
|
+
runtime = self._teams.get(team_id)
|
|
607
|
+
if runtime is None:
|
|
608
|
+
return []
|
|
609
|
+
return runtime.message_bus.get_unread(agent_id)
|
|
610
|
+
|
|
611
|
+
def get_all_messages(self, team_id: str, agent_id: str) -> list[TeamMessage]:
|
|
612
|
+
"""Get all messages for a teammate."""
|
|
613
|
+
runtime = self._teams.get(team_id)
|
|
614
|
+
if runtime is None:
|
|
615
|
+
return []
|
|
616
|
+
return runtime.message_bus.get_all(agent_id)
|
|
617
|
+
|
|
618
|
+
async def mark_read(self, team_id: str, agent_id: str, message_ids: list[str] | None = None) -> int:
|
|
619
|
+
"""Mark messages as read for a teammate."""
|
|
620
|
+
runtime = self._teams.get(team_id)
|
|
621
|
+
if runtime is None:
|
|
622
|
+
return 0
|
|
623
|
+
# Authorize first, then release the lock while mark_read rewrites the
|
|
624
|
+
# inbox and sends receipts. Receipt injection can update teammate state
|
|
625
|
+
# and therefore re-enter membership_lock.
|
|
626
|
+
async with runtime.membership_lock:
|
|
627
|
+
if (
|
|
628
|
+
self._teams.get(team_id) is not runtime
|
|
629
|
+
or runtime.state != TeamState.ACTIVE
|
|
630
|
+
or team_id in self._shutting_down
|
|
631
|
+
or agent_id not in runtime.teammates
|
|
632
|
+
):
|
|
633
|
+
return 0
|
|
634
|
+
async with runtime.bus_lock:
|
|
635
|
+
count = await runtime.message_bus.mark_read(agent_id, message_ids)
|
|
636
|
+
async with runtime.membership_lock:
|
|
637
|
+
if (
|
|
638
|
+
self._teams.get(team_id) is not runtime
|
|
639
|
+
or runtime.state != TeamState.ACTIVE
|
|
640
|
+
or team_id in self._shutting_down
|
|
641
|
+
or agent_id not in runtime.teammates
|
|
642
|
+
):
|
|
643
|
+
return 0
|
|
644
|
+
return count
|
|
645
|
+
|
|
646
|
+
# ------------------------------------------------------------------
|
|
647
|
+
# Task board
|
|
648
|
+
# ------------------------------------------------------------------
|
|
649
|
+
|
|
650
|
+
async def add_task(self, team_id: str, description: str) -> str:
|
|
651
|
+
"""Add a task to the team's task board."""
|
|
652
|
+
runtime = self._teams.get(team_id)
|
|
653
|
+
if runtime is None:
|
|
654
|
+
raise ValueError(f"Team '{team_id}' not found")
|
|
655
|
+
if not isinstance(description, str) or not description.strip():
|
|
656
|
+
raise ValueError("Task description must not be empty")
|
|
657
|
+
|
|
658
|
+
task = TaskItem(description=description)
|
|
659
|
+
async with runtime.membership_lock:
|
|
660
|
+
if runtime.state != TeamState.ACTIVE or team_id in self._shutting_down:
|
|
661
|
+
raise ValueError(f"Team '{team_id}' is not active")
|
|
662
|
+
async with runtime.task_lock:
|
|
663
|
+
runtime.task_board.append(task)
|
|
664
|
+
|
|
665
|
+
await self._emit_task_update(team_id, task)
|
|
666
|
+
return task.id
|
|
667
|
+
|
|
668
|
+
async def claim_task(self, team_id: str, task_id: str, agent_id: str) -> bool:
|
|
669
|
+
"""Atomically claim a task. Protected by asyncio.Lock."""
|
|
670
|
+
runtime = self._teams.get(team_id)
|
|
671
|
+
if runtime is None:
|
|
672
|
+
return False
|
|
673
|
+
async with runtime.membership_lock:
|
|
674
|
+
if runtime.state != TeamState.ACTIVE or team_id in self._shutting_down:
|
|
675
|
+
return False
|
|
676
|
+
if agent_id and agent_id not in runtime.teammates:
|
|
677
|
+
return False
|
|
678
|
+
async with runtime.task_lock:
|
|
679
|
+
task = next((t for t in runtime.task_board if t.id == task_id), None)
|
|
680
|
+
if task is None:
|
|
681
|
+
return False
|
|
682
|
+
if not task.claim(agent_id):
|
|
683
|
+
return False
|
|
684
|
+
|
|
685
|
+
await self._emit_task_update(team_id, task)
|
|
686
|
+
return True
|
|
687
|
+
|
|
688
|
+
async def complete_task(
|
|
689
|
+
self,
|
|
690
|
+
team_id: str,
|
|
691
|
+
task_id: str,
|
|
692
|
+
result: str = "",
|
|
693
|
+
agent_id: str | None = None,
|
|
694
|
+
) -> bool:
|
|
695
|
+
"""Mark a task as completed, optionally enforcing its assignee."""
|
|
696
|
+
runtime = self._teams.get(team_id)
|
|
697
|
+
if runtime is None:
|
|
698
|
+
return False
|
|
699
|
+
|
|
700
|
+
async with runtime.membership_lock:
|
|
701
|
+
if runtime.state != TeamState.ACTIVE or team_id in self._shutting_down:
|
|
702
|
+
return False
|
|
703
|
+
if agent_id and agent_id not in runtime.teammates:
|
|
704
|
+
return False
|
|
705
|
+
async with runtime.task_lock:
|
|
706
|
+
task = next((t for t in runtime.task_board if t.id == task_id), None)
|
|
707
|
+
if task is None:
|
|
708
|
+
return False
|
|
709
|
+
if agent_id and task.assignee != agent_id:
|
|
710
|
+
return False
|
|
711
|
+
if not task.complete(result):
|
|
712
|
+
return False
|
|
713
|
+
|
|
714
|
+
await self._emit_task_update(team_id, task)
|
|
715
|
+
return True
|
|
716
|
+
|
|
717
|
+
async def fail_task(
|
|
718
|
+
self,
|
|
719
|
+
team_id: str,
|
|
720
|
+
task_id: str,
|
|
721
|
+
reason: str = "",
|
|
722
|
+
agent_id: str | None = None,
|
|
723
|
+
) -> bool:
|
|
724
|
+
"""Mark a task as failed, optionally enforcing its assignee."""
|
|
725
|
+
runtime = self._teams.get(team_id)
|
|
726
|
+
if runtime is None:
|
|
727
|
+
return False
|
|
728
|
+
|
|
729
|
+
async with runtime.membership_lock:
|
|
730
|
+
if runtime.state != TeamState.ACTIVE or team_id in self._shutting_down:
|
|
731
|
+
return False
|
|
732
|
+
if agent_id and agent_id not in runtime.teammates:
|
|
733
|
+
return False
|
|
734
|
+
async with runtime.task_lock:
|
|
735
|
+
task = next((t for t in runtime.task_board if t.id == task_id), None)
|
|
736
|
+
if task is None:
|
|
737
|
+
return False
|
|
738
|
+
if agent_id and task.assignee != agent_id:
|
|
739
|
+
return False
|
|
740
|
+
if not task.fail(reason):
|
|
741
|
+
return False
|
|
742
|
+
|
|
743
|
+
await self._emit_task_update(team_id, task)
|
|
744
|
+
return True
|
|
745
|
+
|
|
746
|
+
def list_tasks(self, team_id: str) -> list[TaskItem]:
|
|
747
|
+
"""List all tasks on the team's task board."""
|
|
748
|
+
runtime = self._teams.get(team_id)
|
|
749
|
+
if runtime is None:
|
|
750
|
+
return []
|
|
751
|
+
return [task.model_copy(deep=True) for task in runtime.task_board]
|
|
752
|
+
|
|
753
|
+
# ------------------------------------------------------------------
|
|
754
|
+
# Status
|
|
755
|
+
# ------------------------------------------------------------------
|
|
756
|
+
|
|
757
|
+
def get_team_status(self, team_id: str) -> dict[str, Any]:
|
|
758
|
+
"""Get a status overview of the team."""
|
|
759
|
+
runtime = self._teams.get(team_id)
|
|
760
|
+
if runtime is None:
|
|
761
|
+
return {}
|
|
762
|
+
|
|
763
|
+
teammates = []
|
|
764
|
+
for agent_id, info in runtime.teammates.items():
|
|
765
|
+
teammates.append({
|
|
766
|
+
"agent_id": agent_id,
|
|
767
|
+
"name": info.name,
|
|
768
|
+
"role": info.role.value,
|
|
769
|
+
"state": info.state.value,
|
|
770
|
+
"model_profile": info.model_profile,
|
|
771
|
+
})
|
|
772
|
+
|
|
773
|
+
tasks_summary = {
|
|
774
|
+
"total": len(runtime.task_board),
|
|
775
|
+
"pending": sum(1 for t in runtime.task_board if t.status == TaskStatus.PENDING),
|
|
776
|
+
"claimed": sum(1 for t in runtime.task_board if t.status == TaskStatus.CLAIMED),
|
|
777
|
+
"completed": sum(1 for t in runtime.task_board if t.status == TaskStatus.COMPLETED),
|
|
778
|
+
"failed": sum(1 for t in runtime.task_board if t.status == TaskStatus.FAILED),
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
return {
|
|
782
|
+
"team_id": team_id,
|
|
783
|
+
"state": runtime.state.value,
|
|
784
|
+
"teammates": teammates,
|
|
785
|
+
"teammate_count": len(runtime.teammates),
|
|
786
|
+
"max_teammates": runtime.config.max_teammates,
|
|
787
|
+
"tasks": tasks_summary,
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
def list_teams(self) -> list[str]:
|
|
791
|
+
"""List all active team IDs."""
|
|
792
|
+
return list(self._teams.keys())
|
|
793
|
+
|
|
794
|
+
def get_teammate(self, team_id: str, agent_id: str) -> TeammateInfo | None:
|
|
795
|
+
"""Get info about a specific teammate."""
|
|
796
|
+
runtime = self._teams.get(team_id)
|
|
797
|
+
if runtime is None:
|
|
798
|
+
return None
|
|
799
|
+
info = runtime.teammates.get(agent_id)
|
|
800
|
+
return info.model_copy(deep=True) if info is not None else None
|
|
801
|
+
|
|
802
|
+
def get_team_for_agent(self, agent_id: str) -> str | None:
|
|
803
|
+
"""Find which team an agent belongs to."""
|
|
804
|
+
for team_id, runtime in self._teams.items():
|
|
805
|
+
if agent_id in runtime.teammates:
|
|
806
|
+
return team_id
|
|
807
|
+
return None
|
|
808
|
+
|
|
809
|
+
def get_lead_agent(self, team_id: str) -> str | None:
|
|
810
|
+
"""Return an explicitly registered lead teammate, if one exists."""
|
|
811
|
+
runtime = self._teams.get(team_id)
|
|
812
|
+
if runtime is None:
|
|
813
|
+
return None
|
|
814
|
+
for agent_id, info in runtime.teammates.items():
|
|
815
|
+
if info.role == TeammateRole.LEAD:
|
|
816
|
+
return agent_id
|
|
817
|
+
return None
|
|
818
|
+
|
|
819
|
+
# ------------------------------------------------------------------
|
|
820
|
+
# Cross-team communication
|
|
821
|
+
# ------------------------------------------------------------------
|
|
822
|
+
|
|
823
|
+
def register_bridge(
|
|
824
|
+
self,
|
|
825
|
+
team_a: str,
|
|
826
|
+
team_b: str,
|
|
827
|
+
policy: BridgePolicy = BridgePolicy.ALLOW_TAGGED,
|
|
828
|
+
) -> None:
|
|
829
|
+
"""Register a bridge policy between two teams."""
|
|
830
|
+
if team_a == team_b:
|
|
831
|
+
raise ValueError("A bridge requires two distinct teams")
|
|
832
|
+
try:
|
|
833
|
+
policy = BridgePolicy(policy)
|
|
834
|
+
except (TypeError, ValueError) as exc:
|
|
835
|
+
raise ValueError(f"Unknown bridge policy: {policy!r}") from exc
|
|
836
|
+
self._bridges[(team_a, team_b)] = policy
|
|
837
|
+
self._bridges[(team_b, team_a)] = policy
|
|
838
|
+
|
|
839
|
+
def get_bridge_policy(self, team_a: str, team_b: str) -> BridgePolicy:
|
|
840
|
+
"""Return the effective bridge policy, defaulting to deny."""
|
|
841
|
+
explicit = self._bridges.get((team_a, team_b))
|
|
842
|
+
if explicit is not None:
|
|
843
|
+
return explicit
|
|
844
|
+
source = self._teams.get(team_a)
|
|
845
|
+
target = self._teams.get(team_b)
|
|
846
|
+
if source is not None and target is not None:
|
|
847
|
+
source_policy = source.config.bridge_policy
|
|
848
|
+
target_policy = target.config.bridge_policy
|
|
849
|
+
if source_policy == target_policy:
|
|
850
|
+
return source_policy
|
|
851
|
+
return BridgePolicy.DENY
|
|
852
|
+
|
|
853
|
+
async def send_cross_team(
|
|
854
|
+
self,
|
|
855
|
+
from_team: str,
|
|
856
|
+
from_agent: str,
|
|
857
|
+
to_team: str,
|
|
858
|
+
to_agent: str,
|
|
859
|
+
text: str,
|
|
860
|
+
) -> CrossTeamMessage | None:
|
|
861
|
+
"""Send a message between teams. Requires an allowed bridge policy."""
|
|
862
|
+
if from_team == to_team:
|
|
863
|
+
logger.warning("Cross-team message requires distinct teams: %s", from_team)
|
|
864
|
+
return None
|
|
865
|
+
source_runtime = self._teams.get(from_team)
|
|
866
|
+
target_runtime = self._teams.get(to_team)
|
|
867
|
+
if source_runtime is None or target_runtime is None:
|
|
868
|
+
logger.warning("Cross-team message references an unknown team: %s -> %s", from_team, to_team)
|
|
869
|
+
return None
|
|
870
|
+
if not isinstance(text, str) or not text.strip():
|
|
871
|
+
return None
|
|
872
|
+
|
|
873
|
+
# Acquire both team locks in stable order for authorization only. The
|
|
874
|
+
# actual bus operation must run outside them: target injection calls
|
|
875
|
+
# back into _mark_teammate_busy(), which needs the target lock.
|
|
876
|
+
ordered = sorted(
|
|
877
|
+
((from_team, source_runtime), (to_team, target_runtime)),
|
|
878
|
+
key=lambda item: item[0],
|
|
879
|
+
)
|
|
880
|
+
target_epochs: dict[str, int] = {}
|
|
881
|
+
async with ordered[0][1].membership_lock:
|
|
882
|
+
async with ordered[1][1].membership_lock:
|
|
883
|
+
if (
|
|
884
|
+
self._teams.get(from_team) is not source_runtime
|
|
885
|
+
or self._teams.get(to_team) is not target_runtime
|
|
886
|
+
or source_runtime.state != TeamState.ACTIVE
|
|
887
|
+
or target_runtime.state != TeamState.ACTIVE
|
|
888
|
+
or from_team in self._shutting_down
|
|
889
|
+
or to_team in self._shutting_down
|
|
890
|
+
):
|
|
891
|
+
return None
|
|
892
|
+
if from_agent and from_agent not in source_runtime.teammates:
|
|
893
|
+
logger.warning(
|
|
894
|
+
"Cross-team message from non-member %s in team %s",
|
|
895
|
+
from_agent,
|
|
896
|
+
from_team,
|
|
897
|
+
)
|
|
898
|
+
return None
|
|
899
|
+
if to_agent and to_agent not in target_runtime.teammates:
|
|
900
|
+
logger.warning(
|
|
901
|
+
"Cross-team message targets non-member %s in team %s",
|
|
902
|
+
to_agent,
|
|
903
|
+
to_team,
|
|
904
|
+
)
|
|
905
|
+
return None
|
|
906
|
+
if to_agent:
|
|
907
|
+
epoch = target_runtime.message_bus.registration_epoch(to_agent)
|
|
908
|
+
if epoch is None:
|
|
909
|
+
return None
|
|
910
|
+
target_epochs[to_agent] = epoch
|
|
911
|
+
else:
|
|
912
|
+
for target_id in target_runtime.teammates:
|
|
913
|
+
if target_id == from_agent:
|
|
914
|
+
continue
|
|
915
|
+
epoch = target_runtime.message_bus.registration_epoch(target_id)
|
|
916
|
+
if epoch is not None:
|
|
917
|
+
target_epochs[target_id] = epoch
|
|
918
|
+
|
|
919
|
+
policy = self._bridges.get((from_team, to_team))
|
|
920
|
+
if policy is None:
|
|
921
|
+
# A non-deny team-level policy can opt into a bridge without
|
|
922
|
+
# explicit pair registration. Require both sides to opt in.
|
|
923
|
+
source_policy = source_runtime.config.bridge_policy
|
|
924
|
+
target_policy = target_runtime.config.bridge_policy
|
|
925
|
+
policy = (
|
|
926
|
+
source_policy
|
|
927
|
+
if source_policy == target_policy
|
|
928
|
+
else BridgePolicy.DENY
|
|
929
|
+
)
|
|
930
|
+
if policy == BridgePolicy.DENY:
|
|
931
|
+
logger.warning(
|
|
932
|
+
"Cross-team message blocked: no bridge between %s and %s",
|
|
933
|
+
from_team,
|
|
934
|
+
to_team,
|
|
935
|
+
)
|
|
936
|
+
return None
|
|
937
|
+
|
|
938
|
+
# Apply the target team's byte limit before recording the audit
|
|
939
|
+
# message so retained and delivered payloads agree.
|
|
940
|
+
max_bytes = target_runtime.config.max_message_size_bytes
|
|
941
|
+
tag_prefix = (
|
|
942
|
+
f"[cross-team:{from_team}] "
|
|
943
|
+
if policy == BridgePolicy.ALLOW_TAGGED
|
|
944
|
+
else ""
|
|
945
|
+
)
|
|
946
|
+
available_bytes = max(0, max_bytes - len(tag_prefix.encode("utf-8")))
|
|
947
|
+
if len(text.encode("utf-8")) > available_bytes:
|
|
948
|
+
text = (
|
|
949
|
+
text.encode("utf-8")[:available_bytes]
|
|
950
|
+
.decode("utf-8", errors="ignore")
|
|
951
|
+
)
|
|
952
|
+
if not text:
|
|
953
|
+
return None
|
|
954
|
+
|
|
955
|
+
cross_msg = CrossTeamMessage(
|
|
956
|
+
from_team=from_team,
|
|
957
|
+
from_agent=from_agent,
|
|
958
|
+
to_team=to_team,
|
|
959
|
+
to_agent=to_agent,
|
|
960
|
+
text=text,
|
|
961
|
+
bridge_policy=policy,
|
|
962
|
+
)
|
|
963
|
+
# ALLOW_TAGGED makes the source explicit to the recipient;
|
|
964
|
+
# ALLOW_ALL preserves the original payload.
|
|
965
|
+
tagged_text = tag_prefix + text
|
|
966
|
+
|
|
967
|
+
# Serialize bus I/O with both teams' teardown paths. Acquire in the
|
|
968
|
+
# same stable team-id order used for membership authorization so two
|
|
969
|
+
# simultaneous cross-team sends cannot deadlock each other.
|
|
970
|
+
async with ordered[0][1].bus_lock:
|
|
971
|
+
async with ordered[1][1].bus_lock:
|
|
972
|
+
# Removal acquires bus_lock before unregistering. Recheck the
|
|
973
|
+
# frozen roster/generations synchronously while both locks are
|
|
974
|
+
# held, then use expected_epoch on every write.
|
|
975
|
+
if (
|
|
976
|
+
self._teams.get(from_team) is not source_runtime
|
|
977
|
+
or self._teams.get(to_team) is not target_runtime
|
|
978
|
+
or source_runtime.state != TeamState.ACTIVE
|
|
979
|
+
or target_runtime.state != TeamState.ACTIVE
|
|
980
|
+
or from_team in self._shutting_down
|
|
981
|
+
or to_team in self._shutting_down
|
|
982
|
+
or any(
|
|
983
|
+
target_id not in target_runtime.teammates
|
|
984
|
+
or target_runtime.message_bus.registration_epoch(target_id) != epoch
|
|
985
|
+
for target_id, epoch in target_epochs.items()
|
|
986
|
+
)
|
|
987
|
+
):
|
|
988
|
+
return None
|
|
989
|
+
delivered_messages = []
|
|
990
|
+
for target_id, epoch in target_epochs.items():
|
|
991
|
+
delivered = await target_runtime.message_bus.send(
|
|
992
|
+
from_agent=from_agent,
|
|
993
|
+
to_agent=target_id,
|
|
994
|
+
text=tagged_text,
|
|
995
|
+
msg_type="cross_team",
|
|
996
|
+
expected_epoch=epoch,
|
|
997
|
+
)
|
|
998
|
+
if delivered is not None:
|
|
999
|
+
delivered_messages.append(delivered)
|
|
1000
|
+
if len(delivered_messages) == 1:
|
|
1001
|
+
# The durable inbox append is the delivery commit
|
|
1002
|
+
# point. Record the audit exactly once while the bus
|
|
1003
|
+
# transaction is still owned so teardown cannot make
|
|
1004
|
+
# an already-delivered message look unsuccessful.
|
|
1005
|
+
self._cross_team_messages.append(cross_msg)
|
|
1006
|
+
if not delivered_messages:
|
|
1007
|
+
return None
|
|
1008
|
+
|
|
1009
|
+
for delivered in delivered_messages:
|
|
1010
|
+
await self._emit_team_message(to_team, delivered)
|
|
1011
|
+
return cross_msg
|
|
1012
|
+
|
|
1013
|
+
# ------------------------------------------------------------------
|
|
1014
|
+
# Recovery
|
|
1015
|
+
# ------------------------------------------------------------------
|
|
1016
|
+
|
|
1017
|
+
async def recover(self) -> list[dict[str, str]]:
|
|
1018
|
+
"""Recover stale teammate state after an interrupted runtime.
|
|
1019
|
+
|
|
1020
|
+
Both ``busy`` and ``cancelling`` are in-flight states. Either can be
|
|
1021
|
+
left behind when a process exits before the AgentManager writes its
|
|
1022
|
+
terminal state, so both are normalized to ``ready``. This method does
|
|
1023
|
+
NOT auto-restart agents and returns the affected team/agent pairs for
|
|
1024
|
+
manual re-engagement.
|
|
1025
|
+
"""
|
|
1026
|
+
interrupted: list[dict[str, str]] = []
|
|
1027
|
+
recovered_states: dict[tuple[str, str], str] = {}
|
|
1028
|
+
|
|
1029
|
+
# Take a stable runtime snapshot under the map lock. A concurrent
|
|
1030
|
+
# shutdown may remove and later recreate the same team id; identity
|
|
1031
|
+
# checks below prevent recovery from mutating or reporting the
|
|
1032
|
+
# replacement runtime.
|
|
1033
|
+
async with self._teams_lock:
|
|
1034
|
+
runtimes = list(self._teams.items())
|
|
1035
|
+
|
|
1036
|
+
for team_id, runtime in runtimes:
|
|
1037
|
+
changes: list[tuple[str, str, str]] = []
|
|
1038
|
+
# A shutdown reserves the team name before waiting on its
|
|
1039
|
+
# membership lock. Check that inexpensive fence first so recovery
|
|
1040
|
+
# does not block behind a lock that shutdown intentionally holds
|
|
1041
|
+
# across cancellation callbacks.
|
|
1042
|
+
async with self._teams_lock:
|
|
1043
|
+
if (
|
|
1044
|
+
self._teams.get(team_id) is not runtime
|
|
1045
|
+
or team_id in self._shutting_down
|
|
1046
|
+
):
|
|
1047
|
+
continue
|
|
1048
|
+
# Acquire the per-runtime lock first. add_teammate() and the
|
|
1049
|
+
# shutdown worker can hold membership_lock across an AgentManager
|
|
1050
|
+
# callback that re-enters create_team() (and therefore needs
|
|
1051
|
+
# _teams_lock); taking the locks in the opposite order here would
|
|
1052
|
+
# deadlock that callback. The map lock is only held for the
|
|
1053
|
+
# synchronous identity/state check and transition.
|
|
1054
|
+
async with runtime.membership_lock:
|
|
1055
|
+
async with self._teams_lock:
|
|
1056
|
+
if (
|
|
1057
|
+
runtime.state != TeamState.ACTIVE
|
|
1058
|
+
or team_id in self._shutting_down
|
|
1059
|
+
or self._teams.get(team_id) is not runtime
|
|
1060
|
+
):
|
|
1061
|
+
continue
|
|
1062
|
+
for agent_id, info in list(runtime.teammates.items()):
|
|
1063
|
+
if info.state in {
|
|
1064
|
+
TeammateState.BUSY,
|
|
1065
|
+
TeammateState.CANCELLING,
|
|
1066
|
+
}:
|
|
1067
|
+
old_state = info.state.value
|
|
1068
|
+
info.transition_to(TeammateState.READY, force=True)
|
|
1069
|
+
changes.append((agent_id, old_state, info.role.value))
|
|
1070
|
+
interrupted.append({"team_id": team_id, "agent_id": agent_id})
|
|
1071
|
+
recovered_states[(team_id, agent_id)] = old_state
|
|
1072
|
+
for agent_id, old_state, role in changes:
|
|
1073
|
+
# Shutdown/removal may begin after the recovery transition but
|
|
1074
|
+
# before its notification is dispatched. Re-check ownership
|
|
1075
|
+
# synchronously so a READY event cannot be emitted for a team
|
|
1076
|
+
# that has already entered its terminal state.
|
|
1077
|
+
async with runtime.membership_lock:
|
|
1078
|
+
info = runtime.teammates.get(agent_id)
|
|
1079
|
+
if (
|
|
1080
|
+
self._teams.get(team_id) is not runtime
|
|
1081
|
+
or runtime.state != TeamState.ACTIVE
|
|
1082
|
+
or team_id in self._shutting_down
|
|
1083
|
+
or info is None
|
|
1084
|
+
or info.state != TeammateState.READY
|
|
1085
|
+
):
|
|
1086
|
+
continue
|
|
1087
|
+
await self._emit_team_state(
|
|
1088
|
+
team_id,
|
|
1089
|
+
agent_id,
|
|
1090
|
+
old_state,
|
|
1091
|
+
TeammateState.READY.value,
|
|
1092
|
+
role,
|
|
1093
|
+
)
|
|
1094
|
+
|
|
1095
|
+
if recovered_states:
|
|
1096
|
+
self._last_recovered_states = recovered_states
|
|
1097
|
+
return interrupted
|
|
1098
|
+
|
|
1099
|
+
# ------------------------------------------------------------------
|
|
1100
|
+
# Cleanup
|
|
1101
|
+
# ------------------------------------------------------------------
|
|
1102
|
+
|
|
1103
|
+
async def close(self) -> None:
|
|
1104
|
+
"""Shutdown all teams."""
|
|
1105
|
+
async with self._teams_lock:
|
|
1106
|
+
if self._closed:
|
|
1107
|
+
return
|
|
1108
|
+
# Cancellation callbacks can re-enter the manager and create another
|
|
1109
|
+
# team. Drain until no active team or detached cleanup remains instead
|
|
1110
|
+
# of relying on one snapshot of _teams.
|
|
1111
|
+
current = asyncio.current_task()
|
|
1112
|
+
while True:
|
|
1113
|
+
for team_id in list(self._teams.keys()):
|
|
1114
|
+
if team_id in self._shutting_down:
|
|
1115
|
+
continue
|
|
1116
|
+
await self.shutdown_team(team_id)
|
|
1117
|
+
|
|
1118
|
+
# A caller may have cancelled shutdown_team after its cleanup task
|
|
1119
|
+
# was scheduled. Do not return while that detached cleanup is
|
|
1120
|
+
# still mutating inboxes or cancelling agents. Exclude the current
|
|
1121
|
+
# cleanup task for re-entrant close() calls from cancellation hooks.
|
|
1122
|
+
pending_shutdowns = [
|
|
1123
|
+
task
|
|
1124
|
+
for task in self._shutdown_tasks.values()
|
|
1125
|
+
if task is not current and not task.done()
|
|
1126
|
+
]
|
|
1127
|
+
if pending_shutdowns:
|
|
1128
|
+
await asyncio.gather(*pending_shutdowns, return_exceptions=True)
|
|
1129
|
+
|
|
1130
|
+
# Also clean up orphaned waiters for agents removed by an external
|
|
1131
|
+
# AgentManager operation. This await is intentionally inside the
|
|
1132
|
+
# drain loop: a cancellation callback can create a team while the
|
|
1133
|
+
# orphan tasks are being cancelled, and that team must be drained
|
|
1134
|
+
# before the manager becomes terminal.
|
|
1135
|
+
await asyncio.gather(
|
|
1136
|
+
*(self._cancel_agent_tasks(agent_id) for agent_id in list(self._delivery_locks)),
|
|
1137
|
+
return_exceptions=True,
|
|
1138
|
+
)
|
|
1139
|
+
|
|
1140
|
+
async with self._teams_lock:
|
|
1141
|
+
remaining = [
|
|
1142
|
+
task
|
|
1143
|
+
for task in self._shutdown_tasks.values()
|
|
1144
|
+
if task is not current and not task.done()
|
|
1145
|
+
]
|
|
1146
|
+
active_teams = any(
|
|
1147
|
+
team_id not in self._shutting_down for team_id in self._teams
|
|
1148
|
+
)
|
|
1149
|
+
if not active_teams and not remaining:
|
|
1150
|
+
# The lock closes the small race between the final drain
|
|
1151
|
+
# check and a stale ToolContext attempting create_team().
|
|
1152
|
+
# A creator that acquired the lock first made the team
|
|
1153
|
+
# visible above and forces another drain iteration; one
|
|
1154
|
+
# that arrives afterwards observes _closed and is rejected.
|
|
1155
|
+
self._closed = True
|
|
1156
|
+
return
|
|
1157
|
+
|
|
1158
|
+
# ------------------------------------------------------------------
|
|
1159
|
+
# Internal helpers
|
|
1160
|
+
# ------------------------------------------------------------------
|
|
1161
|
+
|
|
1162
|
+
def _resolve_inbox_root(self) -> Path:
|
|
1163
|
+
"""Resolve the inbox storage root directory."""
|
|
1164
|
+
team_settings = getattr(self._settings, "team", None)
|
|
1165
|
+
if team_settings and team_settings.inbox_dir:
|
|
1166
|
+
# A configured directory is a parent, rather than a shared inbox:
|
|
1167
|
+
# session-bound managers get stable project/session children so an
|
|
1168
|
+
# old session cannot read or delete a same-named team's files.
|
|
1169
|
+
return self._scope_inbox_root(Path(team_settings.inbox_dir), custom=True)
|
|
1170
|
+
# Default: <config home>/team_inbox/<project_hash>/<session_hash>/.
|
|
1171
|
+
# Keep the project-only path for low-level callers that have not bound a
|
|
1172
|
+
# session yet.
|
|
1173
|
+
return self._scope_inbox_root(
|
|
1174
|
+
get_config_home() / "team_inbox",
|
|
1175
|
+
custom=False,
|
|
1176
|
+
)
|
|
1177
|
+
|
|
1178
|
+
def _scope_inbox_root(self, base: Path, *, custom: bool) -> Path:
|
|
1179
|
+
"""Return a session-isolated inbox root under *base*.
|
|
1180
|
+
|
|
1181
|
+
``custom`` roots retain their historical unscoped location until a
|
|
1182
|
+
session id is available. This keeps low-level TeamManager callers
|
|
1183
|
+
compatible while ensuring every CoreSession-backed manager is isolated.
|
|
1184
|
+
"""
|
|
1185
|
+
project_hash = hashlib.md5(self._cwd.encode()).hexdigest()[:12]
|
|
1186
|
+
if not self._session_id:
|
|
1187
|
+
return base if custom else base / project_hash
|
|
1188
|
+
session_hash = hashlib.md5(self._session_id.encode()).hexdigest()[:12]
|
|
1189
|
+
return base / project_hash / session_hash
|
|
1190
|
+
|
|
1191
|
+
def _make_inject_fn(self) -> Callable:
|
|
1192
|
+
"""Create a message injection function for the bus.
|
|
1193
|
+
|
|
1194
|
+
Injects a message into the recipient's session as a synthetic
|
|
1195
|
+
user message so the LLM actually sees it.
|
|
1196
|
+
"""
|
|
1197
|
+
async def _inject(agent_id: str, from_agent: str, text: str) -> None:
|
|
1198
|
+
await self._deliver_or_queue(agent_id, from_agent, text)
|
|
1199
|
+
return _inject
|
|
1200
|
+
|
|
1201
|
+
async def _deliver_or_queue(self, agent_id: str, from_agent: str, text: str) -> None:
|
|
1202
|
+
"""Inject immediately when idle, otherwise retain ordered pending input."""
|
|
1203
|
+
lock = self._delivery_locks.setdefault(agent_id, asyncio.Lock())
|
|
1204
|
+
async with lock:
|
|
1205
|
+
# A pending batch must be delivered before a new direct message;
|
|
1206
|
+
# otherwise a message arriving just after the previous turn ends
|
|
1207
|
+
# could overtake older queued messages.
|
|
1208
|
+
team_id = self.get_team_for_agent(agent_id)
|
|
1209
|
+
runtime = self._teams.get(team_id) if team_id is not None else None
|
|
1210
|
+
if (
|
|
1211
|
+
runtime is None
|
|
1212
|
+
or self._teams.get(team_id) is not runtime
|
|
1213
|
+
or runtime.state != TeamState.ACTIVE
|
|
1214
|
+
or team_id in self._shutting_down
|
|
1215
|
+
):
|
|
1216
|
+
# A bus write may finish after teardown has started. It is
|
|
1217
|
+
# durable, but it must not start/restart work in a terminal
|
|
1218
|
+
# teammate runtime.
|
|
1219
|
+
self._pending_messages.pop(agent_id, None)
|
|
1220
|
+
return
|
|
1221
|
+
if self._pending_messages.get(agent_id):
|
|
1222
|
+
self._pending_messages.setdefault(agent_id, []).append((from_agent, text))
|
|
1223
|
+
self._ensure_pending_delivery(agent_id)
|
|
1224
|
+
return
|
|
1225
|
+
|
|
1226
|
+
prompt = f"[Message from teammate {from_agent}]: {text}"
|
|
1227
|
+
delivered = False
|
|
1228
|
+
try:
|
|
1229
|
+
delivered = bool(
|
|
1230
|
+
await self._agent_manager.send_input(
|
|
1231
|
+
agent_id,
|
|
1232
|
+
prompt,
|
|
1233
|
+
interrupt=False,
|
|
1234
|
+
)
|
|
1235
|
+
)
|
|
1236
|
+
except Exception:
|
|
1237
|
+
logger.debug("Failed to inject message to agent %s", agent_id, exc_info=True)
|
|
1238
|
+
|
|
1239
|
+
if delivered:
|
|
1240
|
+
await self._mark_teammate_busy(agent_id)
|
|
1241
|
+
self._ensure_teammate_watcher_for_agent(agent_id)
|
|
1242
|
+
return
|
|
1243
|
+
|
|
1244
|
+
# Busy (or a transient race with completion): wait for the current
|
|
1245
|
+
# run and then deliver all buffered messages in one ordered prompt.
|
|
1246
|
+
self._pending_messages.setdefault(agent_id, []).append((from_agent, text))
|
|
1247
|
+
self._ensure_pending_delivery(agent_id)
|
|
1248
|
+
|
|
1249
|
+
def _ensure_pending_delivery(self, agent_id: str) -> None:
|
|
1250
|
+
task = self._pending_delivery_tasks.get(agent_id)
|
|
1251
|
+
if task is not None and not task.done():
|
|
1252
|
+
return
|
|
1253
|
+
self._pending_delivery_tasks[agent_id] = asyncio.create_task(
|
|
1254
|
+
self._wait_and_deliver_pending(agent_id)
|
|
1255
|
+
)
|
|
1256
|
+
|
|
1257
|
+
async def _wait_and_deliver_pending(self, agent_id: str) -> None:
|
|
1258
|
+
"""Wait for a busy run and drain pending messages without reordering."""
|
|
1259
|
+
failed_attempts = 0
|
|
1260
|
+
try:
|
|
1261
|
+
while self._pending_messages.get(agent_id):
|
|
1262
|
+
team_id = self.get_team_for_agent(agent_id)
|
|
1263
|
+
runtime = self._teams.get(team_id) if team_id is not None else None
|
|
1264
|
+
if (
|
|
1265
|
+
runtime is None
|
|
1266
|
+
or self._teams.get(team_id) is not runtime
|
|
1267
|
+
or runtime.state != TeamState.ACTIVE
|
|
1268
|
+
or team_id in self._shutting_down
|
|
1269
|
+
):
|
|
1270
|
+
self._pending_messages.pop(agent_id, None)
|
|
1271
|
+
return
|
|
1272
|
+
wait_agent = getattr(self._agent_manager, "wait_agent", None)
|
|
1273
|
+
if wait_agent is None:
|
|
1274
|
+
logger.warning("Agent manager cannot wait for pending teammate %s", agent_id)
|
|
1275
|
+
return
|
|
1276
|
+
snapshot = await wait_agent(agent_id)
|
|
1277
|
+
if snapshot is None:
|
|
1278
|
+
# The recipient may have been removed by a team/session
|
|
1279
|
+
# shutdown while this waiter was asleep. Do not leave a
|
|
1280
|
+
# permanent in-memory retry task (or repeatedly requeue
|
|
1281
|
+
# the same batch) when AgentManager can no longer observe
|
|
1282
|
+
# that run. The message bus remains the durable source
|
|
1283
|
+
# of truth and will replay unread messages if the agent is
|
|
1284
|
+
# registered again.
|
|
1285
|
+
self._pending_messages.pop(agent_id, None)
|
|
1286
|
+
return
|
|
1287
|
+
|
|
1288
|
+
# If a new run is already active, wait for it rather than
|
|
1289
|
+
# calling send_input and dropping the batch again.
|
|
1290
|
+
is_active = getattr(self._agent_manager, "is_agent_active", None)
|
|
1291
|
+
if callable(is_active) and is_active(agent_id):
|
|
1292
|
+
await asyncio.sleep(0.05)
|
|
1293
|
+
continue
|
|
1294
|
+
|
|
1295
|
+
lock = self._delivery_locks.setdefault(agent_id, asyncio.Lock())
|
|
1296
|
+
async with lock:
|
|
1297
|
+
pending = list(self._pending_messages.get(agent_id, []))
|
|
1298
|
+
if not pending:
|
|
1299
|
+
return
|
|
1300
|
+
# Do not spin forever when callback-enabled terminal runs
|
|
1301
|
+
# intentionally reject follow-up input.
|
|
1302
|
+
if (
|
|
1303
|
+
getattr(snapshot, "callback_enabled", False)
|
|
1304
|
+
and getattr(snapshot, "callback_state", "") in {"pending", "injected"}
|
|
1305
|
+
):
|
|
1306
|
+
# The completion dispatcher still owns the terminal
|
|
1307
|
+
# callback. Keep messages that arrived while this
|
|
1308
|
+
# teammate was busy instead of silently dropping the
|
|
1309
|
+
# in-memory delivery record. The durable inbox remains
|
|
1310
|
+
# the source of truth; a later explicit re-engagement
|
|
1311
|
+
# or lifecycle event will retry this batch.
|
|
1312
|
+
logger.info(
|
|
1313
|
+
"Deferring pending messages until callback delivery for teammate %s",
|
|
1314
|
+
agent_id,
|
|
1315
|
+
)
|
|
1316
|
+
return
|
|
1317
|
+
combined = "\n".join(
|
|
1318
|
+
f"[Message from teammate {sender}]: {message}"
|
|
1319
|
+
for sender, message in pending
|
|
1320
|
+
)
|
|
1321
|
+
try:
|
|
1322
|
+
delivered = bool(
|
|
1323
|
+
await self._agent_manager.send_input(
|
|
1324
|
+
agent_id,
|
|
1325
|
+
combined,
|
|
1326
|
+
interrupt=False,
|
|
1327
|
+
)
|
|
1328
|
+
)
|
|
1329
|
+
except Exception:
|
|
1330
|
+
delivered = False
|
|
1331
|
+
logger.debug(
|
|
1332
|
+
"Failed to drain pending messages for agent %s",
|
|
1333
|
+
agent_id,
|
|
1334
|
+
exc_info=True,
|
|
1335
|
+
)
|
|
1336
|
+
if delivered:
|
|
1337
|
+
# New arrivals block on this lock, so clearing the
|
|
1338
|
+
# exact batch cannot lose or reorder a message.
|
|
1339
|
+
self._pending_messages.pop(agent_id, None)
|
|
1340
|
+
await self._mark_teammate_busy(agent_id)
|
|
1341
|
+
self._ensure_teammate_watcher_for_agent(agent_id)
|
|
1342
|
+
return
|
|
1343
|
+
failed_attempts += 1
|
|
1344
|
+
# A terminal AgentManager implementation can reject input
|
|
1345
|
+
# forever. Bound retries so a broken recipient does not leave
|
|
1346
|
+
# a hot background task; the durable inbox remains available
|
|
1347
|
+
# for a later explicit retry.
|
|
1348
|
+
if failed_attempts >= 5:
|
|
1349
|
+
logger.warning("Giving up temporary delivery retries for teammate %s", agent_id)
|
|
1350
|
+
return
|
|
1351
|
+
await asyncio.sleep(min(0.05 * (2 ** (failed_attempts - 1)), 1.0))
|
|
1352
|
+
except asyncio.CancelledError:
|
|
1353
|
+
raise
|
|
1354
|
+
finally:
|
|
1355
|
+
current = asyncio.current_task()
|
|
1356
|
+
if self._pending_delivery_tasks.get(agent_id) is current:
|
|
1357
|
+
self._pending_delivery_tasks.pop(agent_id, None)
|
|
1358
|
+
|
|
1359
|
+
def _ensure_teammate_watcher(self, team_id: str, agent_id: str) -> None:
|
|
1360
|
+
runtime = self._teams.get(team_id)
|
|
1361
|
+
if (
|
|
1362
|
+
runtime is None
|
|
1363
|
+
or runtime.state != TeamState.ACTIVE
|
|
1364
|
+
or team_id in self._shutting_down
|
|
1365
|
+
or agent_id not in runtime.teammates
|
|
1366
|
+
):
|
|
1367
|
+
return
|
|
1368
|
+
task = self._teammate_watch_tasks.get(agent_id)
|
|
1369
|
+
if task is not None and not task.done():
|
|
1370
|
+
return
|
|
1371
|
+
self._teammate_watch_tasks[agent_id] = asyncio.create_task(
|
|
1372
|
+
self._watch_teammate(team_id, agent_id)
|
|
1373
|
+
)
|
|
1374
|
+
|
|
1375
|
+
def _ensure_teammate_watcher_for_agent(self, agent_id: str) -> None:
|
|
1376
|
+
team_id = self.get_team_for_agent(agent_id)
|
|
1377
|
+
if team_id is not None:
|
|
1378
|
+
self._ensure_teammate_watcher(team_id, agent_id)
|
|
1379
|
+
|
|
1380
|
+
async def _watch_teammate(self, team_id: str, agent_id: str) -> None:
|
|
1381
|
+
"""Mirror AgentManager terminal states into TeamStatus."""
|
|
1382
|
+
try:
|
|
1383
|
+
wait_agent = getattr(self._agent_manager, "wait_agent", None)
|
|
1384
|
+
if wait_agent is None:
|
|
1385
|
+
return
|
|
1386
|
+
while True:
|
|
1387
|
+
snapshot = await wait_agent(agent_id)
|
|
1388
|
+
if snapshot is None:
|
|
1389
|
+
return
|
|
1390
|
+
# A new send_input can start a replacement run immediately
|
|
1391
|
+
# after done_event is set but before this watcher resumes. Do
|
|
1392
|
+
# not mark that replacement READY; wait for its own terminal
|
|
1393
|
+
# state first.
|
|
1394
|
+
is_active = getattr(self._agent_manager, "is_agent_active", None)
|
|
1395
|
+
if not callable(is_active) or not is_active(agent_id):
|
|
1396
|
+
break
|
|
1397
|
+
await asyncio.sleep(0.05)
|
|
1398
|
+
runtime = self._teams.get(team_id)
|
|
1399
|
+
if runtime is None:
|
|
1400
|
+
return
|
|
1401
|
+
state_event: tuple[str, str] | None = None
|
|
1402
|
+
should_rearm = False
|
|
1403
|
+
# A teammate can be removed or a team can enter shutdown after
|
|
1404
|
+
# wait_agent() wakes but before this watcher updates the model.
|
|
1405
|
+
# Serialize the final transition with membership changes so a
|
|
1406
|
+
# stale completion cannot publish READY for a removed member.
|
|
1407
|
+
async with runtime.membership_lock:
|
|
1408
|
+
if (
|
|
1409
|
+
self._teams.get(team_id) is not runtime
|
|
1410
|
+
or runtime.state != TeamState.ACTIVE
|
|
1411
|
+
or team_id in self._shutting_down
|
|
1412
|
+
):
|
|
1413
|
+
return
|
|
1414
|
+
info = runtime.teammates.get(agent_id)
|
|
1415
|
+
if info is None:
|
|
1416
|
+
return
|
|
1417
|
+
if info.state == TeammateState.BUSY:
|
|
1418
|
+
old_state = info.state.value
|
|
1419
|
+
if info.transition_to(TeammateState.READY, force=True):
|
|
1420
|
+
state_event = (old_state, info.role.value)
|
|
1421
|
+
# A pending delivery may have started another run while this
|
|
1422
|
+
# watcher was settling. Ensure a fresh watcher observes it.
|
|
1423
|
+
should_rearm = bool(
|
|
1424
|
+
getattr(self._agent_manager, "is_agent_active", lambda _id: False)(agent_id)
|
|
1425
|
+
)
|
|
1426
|
+
if state_event is not None:
|
|
1427
|
+
await self._emit_team_state(
|
|
1428
|
+
team_id,
|
|
1429
|
+
agent_id,
|
|
1430
|
+
state_event[0],
|
|
1431
|
+
TeammateState.READY.value,
|
|
1432
|
+
state_event[1],
|
|
1433
|
+
)
|
|
1434
|
+
if should_rearm:
|
|
1435
|
+
self._ensure_teammate_watcher(team_id, agent_id)
|
|
1436
|
+
except asyncio.CancelledError:
|
|
1437
|
+
raise
|
|
1438
|
+
except Exception:
|
|
1439
|
+
logger.debug("Failed to update teammate state for %s", agent_id, exc_info=True)
|
|
1440
|
+
finally:
|
|
1441
|
+
current = asyncio.current_task()
|
|
1442
|
+
if self._teammate_watch_tasks.get(agent_id) is current:
|
|
1443
|
+
self._teammate_watch_tasks.pop(agent_id, None)
|
|
1444
|
+
# A replacement run may have started in the tiny window between
|
|
1445
|
+
# the terminal snapshot and watcher cleanup. Re-arm a watcher so
|
|
1446
|
+
# TeamStatus cannot remain READY while that run is active.
|
|
1447
|
+
is_active = getattr(self._agent_manager, "is_agent_active", None)
|
|
1448
|
+
runtime = self._teams.get(team_id)
|
|
1449
|
+
info = runtime.teammates.get(agent_id) if runtime else None
|
|
1450
|
+
if (
|
|
1451
|
+
callable(is_active)
|
|
1452
|
+
and is_active(agent_id)
|
|
1453
|
+
and runtime is not None
|
|
1454
|
+
and runtime.state == TeamState.ACTIVE
|
|
1455
|
+
and team_id not in self._shutting_down
|
|
1456
|
+
and info is not None
|
|
1457
|
+
and info.state != TeammateState.SHUTDOWN
|
|
1458
|
+
):
|
|
1459
|
+
self._ensure_teammate_watcher_for_agent(agent_id)
|
|
1460
|
+
|
|
1461
|
+
async def _mark_teammate_busy(self, agent_id: str) -> None:
|
|
1462
|
+
team_id = self.get_team_for_agent(agent_id)
|
|
1463
|
+
runtime = self._teams.get(team_id) if team_id else None
|
|
1464
|
+
if runtime is None or team_id is None:
|
|
1465
|
+
return
|
|
1466
|
+
state_event: tuple[str, str] | None = None
|
|
1467
|
+
async with runtime.membership_lock:
|
|
1468
|
+
if (
|
|
1469
|
+
self._teams.get(team_id) is not runtime
|
|
1470
|
+
or runtime.state != TeamState.ACTIVE
|
|
1471
|
+
or team_id in self._shutting_down
|
|
1472
|
+
):
|
|
1473
|
+
return
|
|
1474
|
+
info = runtime.teammates.get(agent_id)
|
|
1475
|
+
if info is None or info.state == TeammateState.SHUTDOWN:
|
|
1476
|
+
return
|
|
1477
|
+
old_state = info.state.value
|
|
1478
|
+
if info.state != TeammateState.BUSY and info.transition_to(TeammateState.BUSY, force=True):
|
|
1479
|
+
state_event = (old_state, info.role.value)
|
|
1480
|
+
if state_event is not None:
|
|
1481
|
+
await self._emit_team_state(
|
|
1482
|
+
team_id,
|
|
1483
|
+
agent_id,
|
|
1484
|
+
state_event[0],
|
|
1485
|
+
TeammateState.BUSY.value,
|
|
1486
|
+
state_event[1],
|
|
1487
|
+
)
|
|
1488
|
+
|
|
1489
|
+
async def _cancel_agent_tasks(self, agent_id: str) -> None:
|
|
1490
|
+
"""Cancel pending delivery/state waiters and discard buffered input."""
|
|
1491
|
+
self._pending_messages.pop(agent_id, None)
|
|
1492
|
+
tasks = []
|
|
1493
|
+
for task_map in (self._pending_delivery_tasks, self._teammate_watch_tasks):
|
|
1494
|
+
task = task_map.pop(agent_id, None)
|
|
1495
|
+
if task is not None and task is not asyncio.current_task() and not task.done():
|
|
1496
|
+
task.cancel()
|
|
1497
|
+
tasks.append(task)
|
|
1498
|
+
if tasks:
|
|
1499
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
1500
|
+
self._delivery_locks.pop(agent_id, None)
|
|
1501
|
+
|
|
1502
|
+
async def _rollback_teammate(
|
|
1503
|
+
self,
|
|
1504
|
+
runtime: TeamRuntime,
|
|
1505
|
+
team_id: str,
|
|
1506
|
+
agent_id: str,
|
|
1507
|
+
) -> None:
|
|
1508
|
+
"""Undo a partially-created teammate and stop its AgentManager run."""
|
|
1509
|
+
# Remove membership first under the same lock used by send/remove/
|
|
1510
|
+
# shutdown. This fences any late delivery while cancellation runs.
|
|
1511
|
+
async with runtime.bus_lock:
|
|
1512
|
+
async with runtime.membership_lock:
|
|
1513
|
+
info = runtime.teammates.pop(agent_id, None)
|
|
1514
|
+
if info is not None:
|
|
1515
|
+
info.transition_to(TeammateState.SHUTDOWN, force=True)
|
|
1516
|
+
runtime.message_bus.unregister_agent(agent_id)
|
|
1517
|
+
|
|
1518
|
+
await self._cancel_agent_tasks(agent_id)
|
|
1519
|
+
try:
|
|
1520
|
+
await self._agent_manager.cancel_agent(agent_id)
|
|
1521
|
+
except Exception:
|
|
1522
|
+
logger.warning(
|
|
1523
|
+
"Failed to clean up spawned teammate %s in team %s",
|
|
1524
|
+
agent_id,
|
|
1525
|
+
team_id,
|
|
1526
|
+
exc_info=True,
|
|
1527
|
+
)
|
|
1528
|
+
|
|
1529
|
+
async def _shielded_rollback_teammate(
|
|
1530
|
+
self,
|
|
1531
|
+
runtime: TeamRuntime,
|
|
1532
|
+
team_id: str,
|
|
1533
|
+
agent_id: str,
|
|
1534
|
+
) -> None:
|
|
1535
|
+
"""Run teammate rollback to completion despite caller cancellation."""
|
|
1536
|
+
cleanup_task = asyncio.create_task(
|
|
1537
|
+
self._rollback_teammate(runtime, team_id, agent_id)
|
|
1538
|
+
)
|
|
1539
|
+
cancelled = False
|
|
1540
|
+
# ``shield`` protects the child, but every cancellation request still
|
|
1541
|
+
# interrupts the parent await. Keep joining until the child settles;
|
|
1542
|
+
# otherwise a second cancellation can make the rollback task
|
|
1543
|
+
# untracked, leaving an AgentManager run and inbox registration behind.
|
|
1544
|
+
while not cleanup_task.done():
|
|
1545
|
+
try:
|
|
1546
|
+
await asyncio.shield(cleanup_task)
|
|
1547
|
+
except asyncio.CancelledError:
|
|
1548
|
+
cancelled = True
|
|
1549
|
+
continue
|
|
1550
|
+
|
|
1551
|
+
# Retrieve and propagate a cleanup failure (including cancellation of
|
|
1552
|
+
# the child itself). Checking done() above avoids an infinite loop if
|
|
1553
|
+
# some external owner explicitly cancels the cleanup task.
|
|
1554
|
+
cleanup_task.result()
|
|
1555
|
+
if cancelled:
|
|
1556
|
+
raise asyncio.CancelledError
|
|
1557
|
+
|
|
1558
|
+
async def handle_agent_event(self, event: Any) -> None:
|
|
1559
|
+
"""Consume an AgentStateEvent when CoreSession wires one in.
|
|
1560
|
+
|
|
1561
|
+
Kept as a public, optional hook so lifecycle integration can be added
|
|
1562
|
+
without coupling TeamManager to CoreSession's event dispatcher.
|
|
1563
|
+
"""
|
|
1564
|
+
# TeamStateEvent/AgentOutputEvent also carry an agent_id but are not
|
|
1565
|
+
# lifecycle notifications. Restrict this hook to AgentStateEvent to
|
|
1566
|
+
# avoid flipping a teammate back to READY on its own team event.
|
|
1567
|
+
from crabcode_core.types.event import AgentStateEvent
|
|
1568
|
+
if not isinstance(event, AgentStateEvent):
|
|
1569
|
+
return
|
|
1570
|
+
agent_id = getattr(event, "agent_id", None)
|
|
1571
|
+
if not agent_id:
|
|
1572
|
+
return
|
|
1573
|
+
team_id = self.get_team_for_agent(agent_id)
|
|
1574
|
+
runtime = self._teams.get(team_id) if team_id else None
|
|
1575
|
+
if runtime is None or team_id is None:
|
|
1576
|
+
return
|
|
1577
|
+
status = getattr(event, "status", "")
|
|
1578
|
+
target = TeammateState.BUSY if status in {"queued", "running"} else TeammateState.READY
|
|
1579
|
+
state_event: tuple[str, str] | None = None
|
|
1580
|
+
async with runtime.membership_lock:
|
|
1581
|
+
if (
|
|
1582
|
+
self._teams.get(team_id) is not runtime
|
|
1583
|
+
or runtime.state != TeamState.ACTIVE
|
|
1584
|
+
or team_id in self._shutting_down
|
|
1585
|
+
):
|
|
1586
|
+
return
|
|
1587
|
+
info = runtime.teammates.get(agent_id)
|
|
1588
|
+
if info is None or info.state == TeammateState.SHUTDOWN:
|
|
1589
|
+
return
|
|
1590
|
+
old_state = info.state.value
|
|
1591
|
+
if info.state != target and info.transition_to(target, force=True):
|
|
1592
|
+
state_event = (old_state, info.role.value)
|
|
1593
|
+
if state_event is not None:
|
|
1594
|
+
await self._emit_team_state(
|
|
1595
|
+
team_id,
|
|
1596
|
+
agent_id,
|
|
1597
|
+
state_event[0],
|
|
1598
|
+
target.value,
|
|
1599
|
+
state_event[1],
|
|
1600
|
+
)
|
|
1601
|
+
if target == TeammateState.BUSY:
|
|
1602
|
+
self._ensure_teammate_watcher(team_id, agent_id)
|
|
1603
|
+
elif self._pending_messages.get(agent_id):
|
|
1604
|
+
self._ensure_pending_delivery(agent_id)
|
|
1605
|
+
|
|
1606
|
+
def _make_wake_fn(self) -> Callable:
|
|
1607
|
+
"""Create an auto-wake function for the bus.
|
|
1608
|
+
|
|
1609
|
+
If the recipient is idle (done_event set), re-engages it by
|
|
1610
|
+
sending a wake message via send_input.
|
|
1611
|
+
"""
|
|
1612
|
+
async def _wake(agent_id: str, from_agent: str) -> None:
|
|
1613
|
+
# The inject_fn already handles re-engagement by calling send_input,
|
|
1614
|
+
# which will restart the prompt loop if the agent is idle.
|
|
1615
|
+
# This function exists as a separate hook for future use (e.g.,
|
|
1616
|
+
# starting a fresh prompt loop without injecting a message).
|
|
1617
|
+
pass
|
|
1618
|
+
return _wake
|
|
1619
|
+
|
|
1620
|
+
async def _emit_team_message(self, team_id: str, msg: TeamMessage) -> None:
|
|
1621
|
+
from crabcode_core.types.event import TeamMessageEvent
|
|
1622
|
+
try:
|
|
1623
|
+
await self._event_sink(TeamMessageEvent(
|
|
1624
|
+
team_id=team_id,
|
|
1625
|
+
from_agent=msg.from_agent,
|
|
1626
|
+
to_agent=msg.to_agent,
|
|
1627
|
+
text=msg.text,
|
|
1628
|
+
msg_type=msg.msg_type,
|
|
1629
|
+
message_id=msg.id,
|
|
1630
|
+
))
|
|
1631
|
+
except Exception:
|
|
1632
|
+
logger.warning("Failed to emit team message event for %s", msg.id, exc_info=True)
|
|
1633
|
+
|
|
1634
|
+
async def _emit_team_state(
|
|
1635
|
+
self, team_id: str, agent_id: str, old_state: str, new_state: str, role: str
|
|
1636
|
+
) -> None:
|
|
1637
|
+
from crabcode_core.types.event import TeamStateEvent
|
|
1638
|
+
try:
|
|
1639
|
+
await self._event_sink(TeamStateEvent(
|
|
1640
|
+
team_id=team_id,
|
|
1641
|
+
agent_id=agent_id,
|
|
1642
|
+
old_state=old_state,
|
|
1643
|
+
new_state=new_state,
|
|
1644
|
+
role=role,
|
|
1645
|
+
))
|
|
1646
|
+
except Exception:
|
|
1647
|
+
logger.warning("Failed to emit team state event for %s", agent_id, exc_info=True)
|
|
1648
|
+
|
|
1649
|
+
async def _emit_task_update(self, team_id: str, task: TaskItem) -> None:
|
|
1650
|
+
from crabcode_core.types.event import TaskUpdateEvent
|
|
1651
|
+
try:
|
|
1652
|
+
await self._event_sink(TaskUpdateEvent(
|
|
1653
|
+
team_id=team_id,
|
|
1654
|
+
task_id=task.id,
|
|
1655
|
+
status=task.status.value,
|
|
1656
|
+
assignee=task.assignee,
|
|
1657
|
+
description=task.description,
|
|
1658
|
+
))
|
|
1659
|
+
except Exception:
|
|
1660
|
+
logger.warning("Failed to emit task update event for %s", task.id, exc_info=True)
|