camel-ai 0.2.67__py3-none-any.whl → 0.2.69a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of camel-ai might be problematic. Click here for more details.

Files changed (43) hide show
  1. camel/__init__.py +1 -1
  2. camel/agents/chat_agent.py +170 -11
  3. camel/configs/vllm_config.py +2 -0
  4. camel/datagen/self_improving_cot.py +1 -1
  5. camel/environments/__init__.py +12 -0
  6. camel/environments/rlcards_env.py +860 -0
  7. camel/interpreters/docker/Dockerfile +2 -5
  8. camel/loaders/firecrawl_reader.py +4 -4
  9. camel/memories/blocks/vectordb_block.py +8 -1
  10. camel/memories/context_creators/score_based.py +185 -39
  11. camel/models/anthropic_model.py +114 -2
  12. camel/runtimes/configs.py +11 -11
  13. camel/runtimes/daytona_runtime.py +4 -4
  14. camel/runtimes/docker_runtime.py +6 -6
  15. camel/runtimes/remote_http_runtime.py +5 -5
  16. camel/societies/workforce/prompts.py +55 -21
  17. camel/societies/workforce/single_agent_worker.py +274 -14
  18. camel/societies/workforce/task_channel.py +9 -2
  19. camel/societies/workforce/utils.py +10 -2
  20. camel/societies/workforce/worker.py +74 -16
  21. camel/societies/workforce/workforce.py +90 -35
  22. camel/tasks/task.py +18 -12
  23. camel/toolkits/__init__.py +2 -0
  24. camel/toolkits/aci_toolkit.py +19 -19
  25. camel/toolkits/arxiv_toolkit.py +6 -6
  26. camel/toolkits/dappier_toolkit.py +5 -5
  27. camel/toolkits/file_write_toolkit.py +10 -10
  28. camel/toolkits/github_toolkit.py +3 -3
  29. camel/toolkits/non_visual_browser_toolkit/__init__.py +18 -0
  30. camel/toolkits/non_visual_browser_toolkit/actions.py +196 -0
  31. camel/toolkits/non_visual_browser_toolkit/agent.py +278 -0
  32. camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py +363 -0
  33. camel/toolkits/non_visual_browser_toolkit/nv_browser_session.py +175 -0
  34. camel/toolkits/non_visual_browser_toolkit/snapshot.js +188 -0
  35. camel/toolkits/non_visual_browser_toolkit/snapshot.py +164 -0
  36. camel/toolkits/pptx_toolkit.py +4 -4
  37. camel/toolkits/sympy_toolkit.py +1 -1
  38. camel/toolkits/task_planning_toolkit.py +3 -3
  39. camel/toolkits/thinking_toolkit.py +1 -1
  40. {camel_ai-0.2.67.dist-info → camel_ai-0.2.69a1.dist-info}/METADATA +2 -1
  41. {camel_ai-0.2.67.dist-info → camel_ai-0.2.69a1.dist-info}/RECORD +43 -35
  42. {camel_ai-0.2.67.dist-info → camel_ai-0.2.69a1.dist-info}/WHEEL +0 -0
  43. {camel_ai-0.2.67.dist-info → camel_ai-0.2.69a1.dist-info}/licenses/LICENSE +0 -0
@@ -49,17 +49,6 @@ The information returned should be concise and clear.
49
49
  ASSIGN_TASK_PROMPT = TextPrompt(
50
50
  """You need to assign multiple tasks to worker nodes based on the information below.
51
51
 
52
- Here are the tasks to be assigned:
53
- ==============================
54
- {tasks_info}
55
- ==============================
56
-
57
- Following is the information of the existing worker nodes. The format is <ID>:<description>:<additional_info>. Choose the most capable worker node ID for each task.
58
-
59
- ==============================
60
- {child_nodes_info}
61
- ==============================
62
-
63
52
  For each task, you need to:
64
53
  1. Choose the most capable worker node ID for that task
65
54
  2. Identify any dependencies between tasks (if task B requires results from task A, then task A is a dependency of task B)
@@ -80,24 +69,37 @@ Example valid response:
80
69
  ]
81
70
  }}
82
71
 
83
- IMPORTANT: Only add dependencies when one task truly needs the output/result of another task to complete successfully. Don't add dependencies unless they are logically necessary.
72
+ ***CRITICAL: DEPENDENCY MANAGEMENT IS YOUR IMPORTANT RESPONSIBILITY.***
73
+ Carefully analyze the sequence of tasks. A task's dependencies MUST include the IDs of all prior tasks whose outputs are necessary for its execution. For example, a task to 'Summarize Paper X' MUST depend on the task that 'Finds/Retrieves Paper X'. Similarly, a task that 'Compiles a report from summaries' MUST depend on all 'Summarize Paper X' tasks. **Incorrect or missing dependencies will lead to critical operational failures and an inability to complete the overall objective.** Be meticulous in defining these relationships.
84
74
 
85
75
  Do not include any other text, explanations, justifications, or conversational filler before or after the JSON object. Return ONLY the JSON object.
76
+
77
+ Here are the tasks to be assigned:
78
+ ==============================
79
+ {tasks_info}
80
+ ==============================
81
+
82
+ Following is the information of the existing worker nodes. The format is <ID>:<description>:<additional_info>. Choose the most capable worker node ID for each task.
83
+
84
+ ==============================
85
+ {child_nodes_info}
86
+ ==============================
86
87
  """
87
88
  )
88
89
 
89
90
  PROCESS_TASK_PROMPT = TextPrompt(
90
91
  """You need to process one given task.
91
- Here are results of some prerequisite tasks that you can refer to:
92
+
93
+ Please keep in mind the task you are going to process, the content of the task that you need to do is:
92
94
 
93
95
  ==============================
94
- {dependency_tasks_info}
96
+ {content}
95
97
  ==============================
96
98
 
97
- The content of the task that you need to do is:
99
+ Here are results of some prerequisite tasks that you can refer to:
98
100
 
99
101
  ==============================
100
- {content}
102
+ {dependency_tasks_info}
101
103
  ==============================
102
104
 
103
105
  Here are some additional information about the task:
@@ -182,11 +184,43 @@ Now you should summarize the scenario and return the result of the task.
182
184
  """
183
185
  )
184
186
 
185
- WF_TASK_DECOMPOSE_PROMPT = r"""You need to decompose the given task into subtasks according to the workers available in the group, following these important principles:
187
+ WF_TASK_DECOMPOSE_PROMPT = r"""You need to decompose the given task into subtasks according to the workers available in the group, following these important principles to maximize efficiency and parallelism:
188
+
189
+ 1. **Strategic Grouping for Sequential Work**:
190
+ * If a series of steps must be done in order *and* can be handled by the same worker type, group them into a single subtask to maintain flow and minimize handoffs.
191
+
192
+ 2. **Aggressive Parallelization**:
193
+ * **Across Different Worker Specializations**: If distinct phases of the overall task require different types of workers (e.g., research by a 'SearchAgent', then content creation by a 'DocumentAgent'), define these as separate subtasks.
194
+ * **Within a Single Phase (Data/Task Parallelism)**: If a phase involves repetitive operations on multiple items (e.g., processing 10 documents, fetching 5 web pages, analyzing 3 datasets):
195
+ * Decompose this into parallel subtasks, one for each item or a small batch of items.
196
+ * This applies even if the same type of worker handles these parallel subtasks. The goal is to leverage multiple available workers or allow concurrent processing.
197
+
198
+ 3. **Subtask Design for Efficiency**:
199
+ * **Actionable and Well-Defined**: Each subtask should have a clear, achievable goal.
200
+ * **Balanced Granularity**: Make subtasks large enough to be meaningful but small enough to enable parallelism and quick feedback. Avoid overly large subtasks that hide parallel opportunities.
201
+ * **Consider Dependencies**: While you list tasks sequentially, think about the true dependencies. The workforce manager will handle execution based on these implied dependencies and worker availability.
202
+
203
+ These principles aim to reduce overall completion time by maximizing concurrent work and effectively utilizing all available worker capabilities.
204
+
205
+ **EXAMPLE FORMAT ONLY** (DO NOT use this example content for actual task decomposition):
206
+
207
+ If given a hypothetical task requiring research, analysis, and reporting with multiple items to process, you should decompose it to maximize parallelism:
208
+
209
+ * Poor decomposition (monolithic):
210
+ `<tasks><task>Do all research, analysis, and write final report.</task></tasks>`
211
+
212
+ * Better decomposition (parallel structure):
213
+ ```
214
+ <tasks>
215
+ <task>Subtask 1 (ResearchAgent): Gather initial data and resources.</task>
216
+ <task>Subtask 2.1 (AnalysisAgent): Analyze Item A from Subtask 1 results.</task>
217
+ <task>Subtask 2.2 (AnalysisAgent): Analyze Item B from Subtask 1 results.</task>
218
+ <task>Subtask 2.N (AnalysisAgent): Analyze Item N from Subtask 1 results.</task>
219
+ <task>Subtask 3 (ReportAgent): Compile all analyses into final report.</task>
220
+ </tasks>
221
+ ```
186
222
 
187
- 1. Keep tasks that are sequential and require the same type of worker together in one subtask
188
- 2. Only decompose tasks that can be handled in parallel and require different types of workers
189
- 3. This ensures efficient execution by minimizing context switching between workers
223
+ **END OF FORMAT EXAMPLE** - Now apply this structure to your actual task below.
190
224
 
191
225
  The content of the task is:
192
226
 
@@ -207,7 +241,7 @@ Following are the available workers, given in the format <ID>: <description>.
207
241
  {child_nodes_info}
208
242
  ==============================
209
243
 
210
- You must return the subtasks in the format of a numbered list within <tasks> tags, as shown below:
244
+ You must return the subtasks as a list of individual subtasks within <tasks> tags. If your decomposition, following the principles and detailed example above (e.g., for summarizing multiple papers), results in several parallelizable actions, EACH of those actions must be represented as a separate <task> entry. For instance, the general format is:
211
245
 
212
246
  <tasks>
213
247
  <task>Subtask 1</task>
@@ -13,9 +13,12 @@
13
13
  # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
14
14
  from __future__ import annotations
15
15
 
16
+ import asyncio
16
17
  import datetime
17
18
  import json
18
- from typing import Any, List
19
+ import time
20
+ from collections import deque
21
+ from typing import Any, List, Optional
19
22
 
20
23
  from colorama import Fore
21
24
 
@@ -26,36 +29,237 @@ from camel.societies.workforce.worker import Worker
26
29
  from camel.tasks.task import Task, TaskState, validate_task_content
27
30
 
28
31
 
32
+ class AgentPool:
33
+ r"""A pool of agent instances for efficient reuse.
34
+
35
+ This pool manages a collection of pre-cloned agents. It supports
36
+ auto-scaling based ondemand and intelligent reuse of existing agents.
37
+
38
+ Args:
39
+ base_agent (ChatAgent): The base agent to clone from.
40
+ initial_size (int): Initial number of agents in the pool.
41
+ (default: :obj:`1`)
42
+ max_size (int): Maximum number of agents in the pool.
43
+ (default: :obj:`10`)
44
+ auto_scale (bool): Whether to automatically scale the pool size.
45
+ (default: :obj:`True`)
46
+ scale_factor (float): Factor by which to scale the pool when needed.
47
+ (default: :obj:`1.5`)
48
+ idle_timeout (float): Time in seconds after which idle agents are
49
+ removed. (default: :obj:`180.0`)
50
+ """
51
+
52
+ def __init__(
53
+ self,
54
+ base_agent: ChatAgent,
55
+ initial_size: int = 1,
56
+ max_size: int = 10,
57
+ auto_scale: bool = True,
58
+ scale_factor: float = 1.5,
59
+ idle_timeout: float = 180.0, # 3 minutes
60
+ ):
61
+ self.base_agent = base_agent
62
+ self.max_size = max_size
63
+ self.auto_scale = auto_scale
64
+ self.scale_factor = scale_factor
65
+ self.idle_timeout = idle_timeout
66
+
67
+ # Pool management
68
+ self._available_agents: deque = deque()
69
+ self._in_use_agents: set = set()
70
+ self._agent_last_used: dict = {}
71
+ self._lock = asyncio.Lock()
72
+
73
+ # Statistics
74
+ self._total_borrows = 0
75
+ self._total_clones_created = 0
76
+ self._pool_hits = 0
77
+
78
+ # Initialize pool
79
+ self._initialize_pool(initial_size)
80
+
81
+ def _initialize_pool(self, size: int) -> None:
82
+ r"""Initialize the pool with the specified number of agents."""
83
+ for _ in range(min(size, self.max_size)):
84
+ agent = self._create_fresh_agent()
85
+ self._available_agents.append(agent)
86
+
87
+ def _create_fresh_agent(self) -> ChatAgent:
88
+ r"""Create a fresh agent instance."""
89
+ agent = self.base_agent.clone(with_memory=False)
90
+ self._total_clones_created += 1
91
+ return agent
92
+
93
+ async def get_agent(self) -> ChatAgent:
94
+ r"""Get an agent from the pool, creating one if necessary."""
95
+ async with self._lock:
96
+ self._total_borrows += 1
97
+
98
+ # Try to get from available agents first
99
+ if self._available_agents:
100
+ agent = self._available_agents.popleft()
101
+ self._in_use_agents.add(id(agent))
102
+ self._pool_hits += 1
103
+
104
+ # Reset the agent state
105
+ agent.reset()
106
+ return agent
107
+
108
+ # Check if we can create new agents
109
+ total_agents = len(self._available_agents) + len(
110
+ self._in_use_agents
111
+ )
112
+ if total_agents < self.max_size:
113
+ agent = self._create_fresh_agent()
114
+ self._in_use_agents.add(id(agent))
115
+ return agent
116
+
117
+ # Pool exhausted, wait and retry or create temporary agent
118
+ if self.auto_scale:
119
+ # Create a temporary agent that won't be returned to pool
120
+ return self._create_fresh_agent()
121
+ else:
122
+ # Wait for an agent to become available
123
+ while not self._available_agents:
124
+ await asyncio.sleep(0.1)
125
+
126
+ agent = self._available_agents.popleft()
127
+ self._in_use_agents.add(id(agent))
128
+ agent.reset()
129
+ return agent
130
+
131
+ async def return_agent(self, agent: ChatAgent) -> None:
132
+ r"""Return an agent to the pool."""
133
+ async with self._lock:
134
+ agent_id = id(agent)
135
+
136
+ if agent_id in self._in_use_agents:
137
+ self._in_use_agents.remove(agent_id)
138
+
139
+ # Only return to pool if we're under max size
140
+ if len(self._available_agents) < self.max_size:
141
+ # Reset agent state before returning to pool
142
+ agent.reset()
143
+ self._available_agents.append(agent)
144
+ self._agent_last_used[agent_id] = time.time()
145
+
146
+ async def cleanup_idle_agents(self) -> None:
147
+ r"""Remove idle agents from the pool to free memory."""
148
+ if not self.auto_scale:
149
+ return
150
+
151
+ async with self._lock:
152
+ current_time = time.time()
153
+ agents_to_remove = []
154
+
155
+ for agent in list(self._available_agents):
156
+ agent_id = id(agent)
157
+ last_used = self._agent_last_used.get(agent_id, current_time)
158
+
159
+ if current_time - last_used > self.idle_timeout:
160
+ agents_to_remove.append(agent)
161
+
162
+ for agent in agents_to_remove:
163
+ self._available_agents.remove(agent)
164
+ agent_id = id(agent)
165
+ self._agent_last_used.pop(agent_id, None)
166
+
167
+ def get_stats(self) -> dict:
168
+ r"""Get pool statistics."""
169
+ return {
170
+ "available_agents": len(self._available_agents),
171
+ "in_use_agents": len(self._in_use_agents),
172
+ "total_borrows": self._total_borrows,
173
+ "total_clones_created": self._total_clones_created,
174
+ "pool_hits": self._pool_hits,
175
+ "hit_rate": self._pool_hits / max(self._total_borrows, 1),
176
+ }
177
+
178
+
29
179
  class SingleAgentWorker(Worker):
30
180
  r"""A worker node that consists of a single agent.
31
181
 
32
182
  Args:
33
183
  description (str): Description of the node.
34
184
  worker (ChatAgent): Worker of the node. A single agent.
185
+ use_agent_pool (bool): Whether to use agent pool for efficiency.
186
+ (default: :obj:`True`)
187
+ pool_initial_size (int): Initial size of the agent pool.
188
+ (default: :obj:`1`)
189
+ pool_max_size (int): Maximum size of the agent pool.
190
+ (default: :obj:`10`)
191
+ auto_scale_pool (bool): Whether to auto-scale the agent pool.
192
+ (default: :obj:`True`)
35
193
  """
36
194
 
37
195
  def __init__(
38
196
  self,
39
197
  description: str,
40
198
  worker: ChatAgent,
199
+ use_agent_pool: bool = True,
200
+ pool_initial_size: int = 1,
201
+ pool_max_size: int = 10,
202
+ auto_scale_pool: bool = True,
41
203
  ) -> None:
42
204
  node_id = worker.agent_id
43
- super().__init__(description, node_id=node_id)
205
+ super().__init__(
206
+ description,
207
+ node_id=node_id,
208
+ )
44
209
  self.worker = worker
210
+ self.use_agent_pool = use_agent_pool
211
+
212
+ self.agent_pool: Optional[AgentPool] = None
213
+ self._cleanup_task: Optional[asyncio.Task] = None
214
+ # Initialize agent pool if enabled
215
+ if self.use_agent_pool:
216
+ self.agent_pool = AgentPool(
217
+ base_agent=worker,
218
+ initial_size=pool_initial_size,
219
+ max_size=pool_max_size,
220
+ auto_scale=auto_scale_pool,
221
+ )
45
222
 
46
223
  def reset(self) -> Any:
47
224
  r"""Resets the worker to its initial state."""
48
225
  super().reset()
49
226
  self.worker.reset()
50
227
 
228
+ # Reset agent pool if it exists
229
+ if self.agent_pool:
230
+ # Stop cleanup task
231
+ if self._cleanup_task and not self._cleanup_task.done():
232
+ self._cleanup_task.cancel()
233
+
234
+ # Reinitialize pool
235
+ self.agent_pool = AgentPool(
236
+ base_agent=self.worker,
237
+ )
238
+
239
+ async def _get_worker_agent(self) -> ChatAgent:
240
+ r"""Get a worker agent, either from pool or by cloning."""
241
+ if self.use_agent_pool and self.agent_pool:
242
+ return await self.agent_pool.get_agent()
243
+ else:
244
+ # Fallback to original cloning approach
245
+ return self.worker.clone(with_memory=False)
246
+
247
+ async def _return_worker_agent(self, agent: ChatAgent) -> None:
248
+ r"""Return a worker agent to the pool if pooling is enabled."""
249
+ if self.use_agent_pool and self.agent_pool:
250
+ await self.agent_pool.return_agent(agent)
251
+ # If not using pool, agent will be garbage collected
252
+
51
253
  async def _process_task(
52
254
  self, task: Task, dependencies: List[Task]
53
255
  ) -> TaskState:
54
- r"""Processes a task with its dependencies.
256
+ r"""Processes a task with its dependencies using an efficient agent
257
+ management system.
55
258
 
56
259
  This method asynchronously processes a given task, considering its
57
- dependencies, by sending a generated prompt to a worker. It updates
58
- the task's result based on the agent's response.
260
+ dependencies, by sending a generated prompt to a worker agent.
261
+ Uses an agent pool for efficiency when enabled, or falls back to
262
+ cloning when pool is disabled.
59
263
 
60
264
  Args:
61
265
  task (Task): The task to process, which includes necessary details
@@ -66,14 +270,18 @@ class SingleAgentWorker(Worker):
66
270
  TaskState: `TaskState.DONE` if processed successfully, otherwise
67
271
  `TaskState.FAILED`.
68
272
  """
69
- dependency_tasks_info = self._get_dep_tasks_info(dependencies)
70
- prompt = PROCESS_TASK_PROMPT.format(
71
- content=task.content,
72
- dependency_tasks_info=dependency_tasks_info,
73
- additional_info=task.additional_info,
74
- )
273
+ # Get agent efficiently (from pool or by cloning)
274
+ worker_agent = await self._get_worker_agent()
275
+
75
276
  try:
76
- response = await self.worker.astep(
277
+ dependency_tasks_info = self._get_dep_tasks_info(dependencies)
278
+ prompt = PROCESS_TASK_PROMPT.format(
279
+ content=task.content,
280
+ dependency_tasks_info=dependency_tasks_info,
281
+ additional_info=task.additional_info,
282
+ )
283
+
284
+ response = await worker_agent.astep(
77
285
  prompt, response_format=TaskResult
78
286
  )
79
287
  except Exception as e:
@@ -82,6 +290,16 @@ class SingleAgentWorker(Worker):
82
290
  f"\n{e}{Fore.RESET}"
83
291
  )
84
292
  return TaskState.FAILED
293
+ finally:
294
+ # Return agent to pool or let it be garbage collected
295
+ await self._return_worker_agent(worker_agent)
296
+
297
+ # Get actual token usage from the agent that processed this task
298
+ try:
299
+ _, total_token_count = worker_agent.memory.get_context()
300
+ except Exception:
301
+ # Fallback if memory context unavailable
302
+ total_token_count = 0
85
303
 
86
304
  # Populate additional_info with worker attempt details
87
305
  if task.additional_info is None:
@@ -90,14 +308,20 @@ class SingleAgentWorker(Worker):
90
308
  # Create worker attempt details with descriptive keys
91
309
  worker_attempt_details = {
92
310
  "agent_id": getattr(
311
+ worker_agent, "agent_id", worker_agent.role_name
312
+ ),
313
+ "original_worker_id": getattr(
93
314
  self.worker, "agent_id", self.worker.role_name
94
315
  ),
95
316
  "timestamp": str(datetime.datetime.now()),
96
317
  "description": f"Attempt by "
97
- f"{getattr(self.worker, 'agent_id', self.worker.role_name)} "
318
+ f"{getattr(worker_agent, 'agent_id', worker_agent.role_name)} "
319
+ f"(from pool/clone of "
320
+ f"{getattr(self.worker, 'agent_id', self.worker.role_name)}) "
98
321
  f"to process task {task.content}",
99
322
  "response_content": response.msg.content,
100
- "tool_calls": response.info["tool_calls"],
323
+ "tool_calls": response.info.get("tool_calls"),
324
+ "total_token_count": total_token_count,
101
325
  }
102
326
 
103
327
  # Store the worker attempt in additional_info
@@ -105,6 +329,11 @@ class SingleAgentWorker(Worker):
105
329
  task.additional_info["worker_attempts"] = []
106
330
  task.additional_info["worker_attempts"].append(worker_attempt_details)
107
331
 
332
+ # Store the actual token usage for this specific task
333
+ task.additional_info["token_usage"] = {
334
+ "total_tokens": total_token_count
335
+ }
336
+
108
337
  print(f"======\n{Fore.GREEN}Reply from {self}:{Fore.RESET}")
109
338
 
110
339
  result_dict = json.loads(response.msg.content)
@@ -127,3 +356,34 @@ class SingleAgentWorker(Worker):
127
356
 
128
357
  task.result = task_result.content
129
358
  return TaskState.DONE
359
+
360
+ async def _listen_to_channel(self):
361
+ r"""Override to start cleanup task when pool is enabled."""
362
+ # Start cleanup task for agent pool
363
+ if self.use_agent_pool and self.agent_pool:
364
+ self._cleanup_task = asyncio.create_task(self._periodic_cleanup())
365
+
366
+ # Call parent implementation
367
+ await super()._listen_to_channel()
368
+
369
+ # Stop cleanup task
370
+ if self._cleanup_task and not self._cleanup_task.done():
371
+ self._cleanup_task.cancel()
372
+
373
+ async def _periodic_cleanup(self):
374
+ r"""Periodically clean up idle agents from the pool."""
375
+ while True:
376
+ try:
377
+ await asyncio.sleep(60) # Cleanup every minute
378
+ if self.agent_pool:
379
+ await self.agent_pool.cleanup_idle_agents()
380
+ except asyncio.CancelledError:
381
+ break
382
+ except Exception as e:
383
+ print(f"Error in pool cleanup: {e}")
384
+
385
+ def get_pool_stats(self) -> Optional[dict]:
386
+ r"""Get agent pool statistics if pool is enabled."""
387
+ if self.use_agent_pool and self.agent_pool:
388
+ return self.agent_pool.get_stats()
389
+ return None
@@ -23,6 +23,8 @@ class PacketStatus(Enum):
23
23
  states:
24
24
 
25
25
  - ``SENT``: The packet has been sent to a worker.
26
+ - ``PROCESSING``: The packet has been claimed by a worker and is being
27
+ processed.
26
28
  - ``RETURNED``: The packet has been returned by the worker, meaning that
27
29
  the status of the task inside has been updated.
28
30
  - ``ARCHIVED``: The packet has been archived, meaning that the content of
@@ -31,6 +33,7 @@ class PacketStatus(Enum):
31
33
  """
32
34
 
33
35
  SENT = "SENT"
36
+ PROCESSING = "PROCESSING"
34
37
  RETURNED = "RETURNED"
35
38
  ARCHIVED = "ARCHIVED"
36
39
 
@@ -97,8 +100,9 @@ class TaskChannel:
97
100
  await self._condition.wait()
98
101
 
99
102
  async def get_assigned_task_by_assignee(self, assignee_id: str) -> Task:
100
- r"""Get a task from the channel that has been assigned to the
101
- assignee.
103
+ r"""Atomically get and claim a task from the channel that has been
104
+ assigned to the assignee. This prevents race conditions where multiple
105
+ concurrent calls might retrieve the same task.
102
106
  """
103
107
  async with self._condition:
104
108
  while True:
@@ -107,6 +111,9 @@ class TaskChannel:
107
111
  packet.status == PacketStatus.SENT
108
112
  and packet.assignee_id == assignee_id
109
113
  ):
114
+ # Atomically claim the task by changing its status
115
+ packet.status = PacketStatus.PROCESSING
116
+ self._condition.notify_all()
110
117
  return packet.task
111
118
  await self._condition.wait()
112
119
 
@@ -50,7 +50,8 @@ class TaskAssignment(BaseModel):
50
50
  )
51
51
  dependencies: List[str] = Field(
52
52
  default_factory=list,
53
- description="List of task IDs that must complete before this task.",
53
+ description="List of task IDs that must complete before this task. "
54
+ "This is critical for the task decomposition and execution.",
54
55
  )
55
56
 
56
57
 
@@ -156,7 +157,14 @@ def check_if_running(
156
157
  )
157
158
  return None
158
159
  else:
159
- raise last_exception
160
+ raise (
161
+ last_exception
162
+ if last_exception
163
+ else RuntimeError(
164
+ f"Unexpected failure in {func.__name__} "
165
+ "with no exception captured."
166
+ )
167
+ )
160
168
 
161
169
  return wrapper
162
170
 
@@ -13,9 +13,10 @@
13
13
  # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
14
14
  from __future__ import annotations
15
15
 
16
+ import asyncio
16
17
  import logging
17
18
  from abc import ABC, abstractmethod
18
- from typing import List, Optional
19
+ from typing import List, Optional, Set
19
20
 
20
21
  from colorama import Fore
21
22
 
@@ -43,6 +44,7 @@ class Worker(BaseNode, ABC):
43
44
  node_id: Optional[str] = None,
44
45
  ) -> None:
45
46
  super().__init__(description, node_id=node_id)
47
+ self._active_task_ids: Set[str] = set()
46
48
 
47
49
  def __repr__(self):
48
50
  return f"Worker node {self.node_id} ({self.description})"
@@ -60,7 +62,7 @@ class Worker(BaseNode, ABC):
60
62
  pass
61
63
 
62
64
  async def _get_assigned_task(self) -> Task:
63
- r"""Get the task assigned to this node from the channel."""
65
+ r"""Get a task assigned to this node from the channel."""
64
66
  return await self._channel.get_assigned_task_by_assignee(self.node_id)
65
67
 
66
68
  @staticmethod
@@ -77,20 +79,10 @@ class Worker(BaseNode, ABC):
77
79
  def set_channel(self, channel: TaskChannel):
78
80
  self._channel = channel
79
81
 
80
- @check_if_running(False)
81
- async def _listen_to_channel(self):
82
- """Continuously listen to the channel, process the task that are
83
- assigned to this node, and update the result and status of the task.
84
-
85
- This method should be run in an event loop, as it will run
86
- indefinitely.
87
- """
88
- self._running = True
89
- logger.info(f"{self} started.")
90
-
91
- while True:
92
- # Get the earliest task assigned to this node
93
- task = await self._get_assigned_task()
82
+ async def _process_single_task(self, task: Task) -> None:
83
+ r"""Process a single task and handle its completion/failure."""
84
+ try:
85
+ self._active_task_ids.add(task.id)
94
86
  print(
95
87
  f"{Fore.YELLOW}{self} get task {task.id}: {task.content}"
96
88
  f"{Fore.RESET}"
@@ -109,6 +101,72 @@ class Worker(BaseNode, ABC):
109
101
  task.set_state(task_state)
110
102
 
111
103
  await self._channel.return_task(task.id)
104
+ except Exception as e:
105
+ logger.error(f"Error processing task {task.id}: {e}")
106
+ task.set_state(TaskState.FAILED)
107
+ await self._channel.return_task(task.id)
108
+ finally:
109
+ self._active_task_ids.discard(task.id)
110
+
111
+ @check_if_running(False)
112
+ async def _listen_to_channel(self):
113
+ r"""Continuously listen to the channel and process assigned tasks.
114
+
115
+ This method supports parallel task execution without artificial limits.
116
+ """
117
+ self._running = True
118
+ logger.info(f"{self} started.")
119
+
120
+ # Keep track of running task coroutines
121
+ running_tasks: Set[asyncio.Task] = set()
122
+
123
+ while self._running:
124
+ try:
125
+ # Clean up completed tasks
126
+ completed_tasks = [t for t in running_tasks if t.done()]
127
+ for completed_task in completed_tasks:
128
+ running_tasks.remove(completed_task)
129
+ # Check for exceptions in completed tasks
130
+ try:
131
+ await completed_task
132
+ except Exception as e:
133
+ logger.error(f"Task processing failed: {e}")
134
+
135
+ # Try to get a new task (with short timeout to avoid blocking)
136
+ try:
137
+ task = await asyncio.wait_for(
138
+ self._get_assigned_task(), timeout=1.0
139
+ )
140
+
141
+ # Create and start processing task
142
+ task_coroutine = asyncio.create_task(
143
+ self._process_single_task(task)
144
+ )
145
+ running_tasks.add(task_coroutine)
146
+
147
+ except asyncio.TimeoutError:
148
+ # No tasks available, continue loop
149
+ if not running_tasks:
150
+ # No tasks running and none available, short sleep
151
+ await asyncio.sleep(0.1)
152
+ continue
153
+
154
+ except Exception as e:
155
+ logger.error(
156
+ f"Error in worker {self.node_id} listen loop: {e}"
157
+ )
158
+ await asyncio.sleep(0.1)
159
+ continue
160
+
161
+ # Wait for all remaining tasks to complete when stopping
162
+ if running_tasks:
163
+ logger.info(
164
+ f"{self} stopping, waiting for {len(running_tasks)} "
165
+ f"tasks to complete..."
166
+ )
167
+ await asyncio.gather(*running_tasks, return_exceptions=True)
168
+
169
+ logger.info(f"{self} stopped.")
112
170
 
113
171
  @check_if_running(False)
114
172
  async def start(self):