camel-ai 0.2.67__py3-none-any.whl → 0.2.68__py3-none-any.whl

camel/interpreters/docker/Dockerfile

@@ -55,11 +55,8 @@ RUN curl -fsSL https://install.python-poetry.org | python3.10 - && \
 # Upgrade pip and install base Python packages
 RUN python3.10 -m pip install --upgrade pip setuptools wheel
 
-# Install uv
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \
-    mv /root/.local/bin/uv /usr/local/bin/uv && \
-    mv /root/.local/bin/uvx /usr/local/bin/uvx && \
-    chmod +x /usr/local/bin/uv /usr/local/bin/uvx
+# Install uv using pip instead of the shell script
+RUN pip install uv
 
 # Setup working directory
 WORKDIR /workspace

camel/loaders/firecrawl_reader.py

@@ -98,8 +98,8 @@ class Firecrawl:
     def scrape(
         self,
         url: str,
-        params: Optional[Dict[str, Any]] = None,
-    ) -> Dict:
+        params: Optional[Dict[str, str]] = None,
+    ) -> Dict[str, str]:
         r"""To scrape a single URL. This function supports advanced scraping
         by setting different parameters and returns the full scraped data as a
         dictionary.
@@ -108,11 +108,11 @@ class Firecrawl:
 
         Args:
             url (str): The URL to read.
-            params (Optional[Dict[str, Any]]): Additional parameters for the
+            params (Optional[Dict[str, str]]): Additional parameters for the
                 scrape request.
 
         Returns:
-            Dict: The scraped data.
+            Dict[str, str]: The scraped data.
 
         Raises:
             RuntimeError: If the scrape process fails.

camel/memories/blocks/vectordb_block.py

@@ -89,13 +89,20 @@ class VectorDBBlock(MemoryBlock):
             records (List[MemoryRecord]): Memory records to be added to the
                 memory.
         """
+        # Filter out records with empty message content
+        valid_records = [
+            record
+            for record in records
+            if record.message.content and record.message.content.strip()
+        ]
+
         v_records = [
             VectorRecord(
                 vector=self.embedding.embed(record.message.content),
                 payload=record.to_dict(),
                 id=str(record.uuid),
             )
-            for record in records
+            for record in valid_records
         ]
         self.storage.add(v_records)
 

camel/memories/context_creators/score_based.py

@@ -11,7 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from pydantic import BaseModel
 
@@ -74,6 +74,8 @@ class ScoreBasedContextCreator(BaseContextCreator):
         3. Final output maintains chronological order and in history memory,
            the score of each message decreases according to keep_rate. The
            newer the message, the higher the score.
+        4. Tool calls and their responses are kept together to maintain
+           API compatibility
 
         Args:
             records (List[ContextRecord]): List of context records with scores
@@ -126,12 +128,17 @@ class ScoreBasedContextCreator(BaseContextCreator):
             )
 
         # ======================
-        # 3. Token Calculation
+        # 3. Tool Call Relationship Mapping
+        # ======================
+        tool_call_groups = self._group_tool_calls_and_responses(regular_units)
+
+        # ======================
+        # 4. Token Calculation
         # ======================
         total_tokens = system_tokens + sum(u.num_tokens for u in regular_units)
 
         # ======================
-        # 4. Early Return if Within Limit
+        # 5. Early Return if Within Limit
         # ======================
         if total_tokens <= self.token_limit:
             sorted_units = sorted(
@@ -140,7 +147,7 @@ class ScoreBasedContextCreator(BaseContextCreator):
             return self._assemble_output(sorted_units, system_unit)
 
         # ======================
-        # 5. Truncation Logic
+        # 6. Truncation Logic with Tool Call Awareness
         # ======================
         logger.warning(
             f"Context truncation required "
@@ -148,24 +155,12 @@ class ScoreBasedContextCreator(BaseContextCreator):
             f"pruning low-score messages."
         )
 
-        # Sort for truncation: high scores first, older messages first at same
-        # score
-        sorted_for_truncation = sorted(
-            regular_units, key=self._truncation_sort_key
+        remaining_units = self._truncate_with_tool_call_awareness(
+            regular_units, tool_call_groups, system_tokens
         )
 
-        # Reverse to process from lowest score (end of sorted list)
-        remaining_units = []
-        current_total = system_tokens
-
-        for unit in sorted_for_truncation:
-            potential_total = current_total + unit.num_tokens
-            if potential_total <= self.token_limit:
-                remaining_units.append(unit)
-                current_total = potential_total
-
         # ======================
-        # 6. Output Assembly
+        # 7. Output Assembly
         # ======================
 
         # In case system message is the only message in memory when sorted
@@ -180,6 +175,115 @@ class ScoreBasedContextCreator(BaseContextCreator):
         final_units = sorted(remaining_units, key=self._conversation_sort_key)
         return self._assemble_output(final_units, system_unit)
 
+    def _group_tool_calls_and_responses(
+        self, units: List[_ContextUnit]
+    ) -> Dict[str, List[_ContextUnit]]:
+        r"""Groups tool calls with their corresponding responses.
+
+        Args:
+            units (List[_ContextUnit]): List of context units to analyze
+
+        Returns:
+            Dict[str, List[_ContextUnit]]: Mapping from tool_call_id to list of
+                related units (tool call + responses)
+        """
+        tool_call_groups: Dict[str, List[_ContextUnit]] = {}
+
+        for unit in units:
+            message = unit.record.memory_record.message
+            backend_role = unit.record.memory_record.role_at_backend
+
+            # Check if this is a tool call message
+            if hasattr(message, 'func_name') and hasattr(
+                message, 'tool_call_id'
+            ):
+                tool_call_id = getattr(message, 'tool_call_id', None)
+                if tool_call_id:
+                    if tool_call_id not in tool_call_groups:
+                        tool_call_groups[tool_call_id] = []
+                    tool_call_groups[tool_call_id].append(unit)
+
+            # Check if this is a tool response message
+            elif backend_role == OpenAIBackendRole.FUNCTION:
+                tool_call_id = None
+                if hasattr(message, 'tool_call_id'):
+                    tool_call_id = getattr(message, 'tool_call_id', None)
+                elif hasattr(message, 'result') and hasattr(
+                    message, 'tool_call_id'
+                ):
+                    tool_call_id = getattr(message, 'tool_call_id', None)
+
+                if tool_call_id:
+                    if tool_call_id not in tool_call_groups:
+                        tool_call_groups[tool_call_id] = []
+                    tool_call_groups[tool_call_id].append(unit)
+
+        return tool_call_groups
+
+    def _truncate_with_tool_call_awareness(
+        self,
+        regular_units: List[_ContextUnit],
+        tool_call_groups: Dict[str, List[_ContextUnit]],
+        system_tokens: int,
+    ) -> List[_ContextUnit]:
+        r"""Truncates messages while preserving tool call-response pairs.
+
+        Args:
+            regular_units (List[_ContextUnit]): All regular message units
+            tool_call_groups (Dict[str, List[_ContextUnit]]): Grouped tool
+                calls
+            system_tokens (int): Tokens used by system message
+
+        Returns:
+            List[_ContextUnit]: Units that fit within token limit
+        """
+        # Create sets for quick lookup of tool call related units
+        tool_call_unit_ids = set()
+        for group in tool_call_groups.values():
+            for unit in group:
+                tool_call_unit_ids.add(unit.record.memory_record.uuid)
+
+        # Separate tool call groups and standalone units
+        standalone_units = [
+            u
+            for u in regular_units
+            if u.record.memory_record.uuid not in tool_call_unit_ids
+        ]
+
+        # Sort standalone units for truncation (high scores first)
+        standalone_units.sort(key=self._truncation_sort_key)
+
+        # Sort tool call groups by their best (highest) score
+        sorted_tool_groups = []
+        for _tool_call_id, group in tool_call_groups.items():
+            # Use the highest score in the group as the group's score
+            best_score = max(unit.record.score for unit in group)
+            latest_timestamp = max(unit.record.timestamp for unit in group)
+            group_tokens = sum(unit.num_tokens for unit in group)
+            sorted_tool_groups.append(
+                ((-best_score, -latest_timestamp), group, group_tokens)
+            )
+
+        sorted_tool_groups.sort(key=lambda x: x[0])
+
+        # Greedy selection to fit within token limit
+        remaining_units = []
+        current_tokens = system_tokens
+
+        # First, try to include complete tool call groups
+        for _, group, group_tokens in sorted_tool_groups:
+            if current_tokens + group_tokens <= self.token_limit:
+                remaining_units.extend(group)
+                current_tokens += group_tokens
+
+        # Then, include standalone units
+        for unit in standalone_units:
+            if current_tokens + unit.num_tokens <= self.token_limit:
+                remaining_units.append(unit)
+                current_tokens += unit.num_tokens
+
+        return remaining_units
+
     def _extract_system_message(
         self, records: List[ContextRecord]
     ) -> Tuple[Optional[_ContextUnit], List[_ContextUnit]]:

camel/models/anthropic_model.py

@@ -12,11 +12,14 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 import os
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Union
+
+from openai import AsyncStream, Stream
 
 from camel.configs import ANTHROPIC_API_PARAMS, AnthropicConfig
+from camel.messages import OpenAIMessage
 from camel.models.openai_compatible_model import OpenAICompatibleModel
-from camel.types import ModelType
+from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
 from camel.utils import (
     AnthropicTokenCounter,
     BaseTokenCounter,
@@ -25,6 +28,47 @@ from camel.utils import (
 )
 
 
+def strip_trailing_whitespace_from_messages(
+    messages: List[OpenAIMessage],
+) -> List[OpenAIMessage]:
+    r"""Strip trailing whitespace from all message contents in a list of
+    messages. This is necessary because the Anthropic API doesn't allow
+    trailing whitespace in message content.
+
+    Args:
+        messages (List[OpenAIMessage]): List of messages to process
+
+    Returns:
+        List[OpenAIMessage]: The processed messages with trailing whitespace
+            removed
+    """
+    if not messages:
+        return messages
+
+    # Create a deep copy to avoid modifying the original messages
+    processed_messages = [dict(msg) for msg in messages]
+
+    # Process each message
+    for msg in processed_messages:
+        if "content" in msg and msg["content"] is not None:
+            if isinstance(msg["content"], str):
+                msg["content"] = msg["content"].rstrip()
+            elif isinstance(msg["content"], list):
+                # Handle content that's a list of content parts (e.g., for
+                # multimodal content)
+                for i, part in enumerate(msg["content"]):
+                    if (
+                        isinstance(part, dict)
+                        and "text" in part
+                        and isinstance(part["text"], str)
+                    ):
+                        part["text"] = part["text"].rstrip()
+                    elif isinstance(part, str):
+                        msg["content"][i] = part.rstrip()
+
+    return processed_messages  # type: ignore[return-value]
+
+
 class AnthropicModel(OpenAICompatibleModel):
     r"""Anthropic API in a unified OpenAICompatibleModel interface.
 
@@ -89,6 +133,9 @@ class AnthropicModel(OpenAICompatibleModel):
             **kwargs,
         )
 
+        # Monkey patch the AnthropicTokenCounter to handle trailing whitespace
+        self._patch_anthropic_token_counter()
+
     @property
     def token_counter(self) -> BaseTokenCounter:
         r"""Initialize the token counter for the model backend.
@@ -115,3 +162,68 @@ class AnthropicModel(OpenAICompatibleModel):
                     f"Unexpected argument `{param}` is "
                     "input into Anthropic model backend."
                 )
+
+    def _request_chat_completion(
+        self,
+        messages: List[OpenAIMessage],
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+        # Strip trailing whitespace from all message contents to prevent
+        # Anthropic API errors
+        processed_messages = strip_trailing_whitespace_from_messages(messages)
+
+        # Call the parent class method
+        return super()._request_chat_completion(processed_messages, tools)
+
+    async def _arequest_chat_completion(
+        self,
+        messages: List[OpenAIMessage],
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
+        # Strip trailing whitespace from all message contents to prevent
+        # Anthropic API errors
+        processed_messages = strip_trailing_whitespace_from_messages(messages)
+
+        # Call the parent class method
+        return await super()._arequest_chat_completion(
+            processed_messages, tools
+        )
+
+    def _patch_anthropic_token_counter(self):
+        r"""Monkey patch the AnthropicTokenCounter class to handle trailing
+        whitespace.
+
+        This patches the count_tokens_from_messages method to strip trailing
+        whitespace from message content before sending to the Anthropic API.
+        """
+        import functools
+
+        from anthropic.types import MessageParam
+
+        from camel.utils import AnthropicTokenCounter
+
+        original_count_tokens = (
+            AnthropicTokenCounter.count_tokens_from_messages
+        )
+
+        @functools.wraps(original_count_tokens)
+        def patched_count_tokens(self, messages):
+            # Process messages to remove trailing whitespace
+            processed_messages = strip_trailing_whitespace_from_messages(
+                messages
+            )
+
+            # Use the processed messages with the original method
+            return self.client.messages.count_tokens(
+                messages=[
+                    MessageParam(
+                        content=str(msg["content"]),
+                        role="user" if msg["role"] == "user" else "assistant",
+                    )
+                    for msg in processed_messages
+                ],
+                model=self.model,
+            ).input_tokens
+
+        # Apply the monkey patch
+        AnthropicTokenCounter.count_tokens_from_messages = patched_count_tokens

camel/societies/workforce/prompts.py

@@ -88,16 +88,17 @@ Do not include any other text, explanations, justifications, or conversational f
 
 PROCESS_TASK_PROMPT = TextPrompt(
     """You need to process one given task.
-Here are results of some prerequisite tasks that you can refer to:
+
+Please keep in mind the task you are going to process, the content of the task that you need to do is:
 
 ==============================
-{dependency_tasks_info}
+{content}
 ==============================
 
-The content of the task that you need to do is:
+Here are results of some prerequisite tasks that you can refer to:
 
 ==============================
-{content}
+{dependency_tasks_info}
 ==============================
 
 Here are some additional information about the task:
@@ -182,11 +183,43 @@ Now you should summarize the scenario and return the result of the task.
 """
 )
 
-WF_TASK_DECOMPOSE_PROMPT = r"""You need to decompose the given task into subtasks according to the workers available in the group, following these important principles:
+WF_TASK_DECOMPOSE_PROMPT = r"""You need to decompose the given task into subtasks according to the workers available in the group, following these important principles to maximize efficiency and parallelism:
+
+1.  **Strategic Grouping for Sequential Work**:
+    *   If a series of steps must be done in order *and* can be handled by the same worker type, group them into a single subtask to maintain flow and minimize handoffs.
+
+2.  **Aggressive Parallelization**:
+    *   **Across Different Worker Specializations**: If distinct phases of the overall task require different types of workers (e.g., research by a 'SearchAgent', then content creation by a 'DocumentAgent'), define these as separate subtasks.
+    *   **Within a Single Phase (Data/Task Parallelism)**: If a phase involves repetitive operations on multiple items (e.g., processing 10 documents, fetching 5 web pages, analyzing 3 datasets):
+        *   Decompose this into parallel subtasks, one for each item or a small batch of items.
+        *   This applies even if the same type of worker handles these parallel subtasks. The goal is to leverage multiple available workers or allow concurrent processing.
+
+3.  **Subtask Design for Efficiency**:
+    *   **Actionable and Well-Defined**: Each subtask should have a clear, achievable goal.
+    *   **Balanced Granularity**: Make subtasks large enough to be meaningful but small enough to enable parallelism and quick feedback. Avoid overly large subtasks that hide parallel opportunities.
+    *   **Consider Dependencies**: While you list tasks sequentially, think about the true dependencies. The workforce manager will handle execution based on these implied dependencies and worker availability.
+
+These principles aim to reduce overall completion time by maximizing concurrent work and effectively utilizing all available worker capabilities.
+
+**EXAMPLE FORMAT ONLY** (DO NOT use this example content for actual task decomposition):
+
+If given a hypothetical task requiring research, analysis, and reporting with multiple items to process, you should decompose it to maximize parallelism:
+
+*   Poor decomposition (monolithic):
+    `<tasks><task>Do all research, analysis, and write final report.</task></tasks>`
+
+*   Better decomposition (parallel structure):
+    ```
+    <tasks>
+    <task>Subtask 1 (ResearchAgent): Gather initial data and resources.</task>
+    <task>Subtask 2.1 (AnalysisAgent): Analyze Item A from Subtask 1 results.</task>
+    <task>Subtask 2.2 (AnalysisAgent): Analyze Item B from Subtask 1 results.</task>
+    <task>Subtask 2.N (AnalysisAgent): Analyze Item N from Subtask 1 results.</task>
+    <task>Subtask 3 (ReportAgent): Compile all analyses into final report.</task>
+    </tasks>
+    ```
 
-1. Keep tasks that are sequential and require the same type of worker together in one subtask
-2. Only decompose tasks that can be handled in parallel and require different types of workers
-3. This ensures efficient execution by minimizing context switching between workers
+**END OF FORMAT EXAMPLE** - Now apply this structure to your actual task below.
 
 The content of the task is:
 
@@ -207,7 +240,7 @@ Following are the available workers, given in the format <ID>: <description>.
 {child_nodes_info}
 ==============================
 
-You must return the subtasks in the format of a numbered list within <tasks> tags, as shown below:
+You must return the subtasks as a list of individual subtasks within <tasks> tags. If your decomposition, following the principles and detailed example above (e.g., for summarizing multiple papers), results in several parallelizable actions, EACH of those actions must be represented as a separate <task> entry. For instance, the general format is:
 
 <tasks>
 <task>Subtask 1</task>

camel/societies/workforce/single_agent_worker.py

@@ -32,15 +32,22 @@ class SingleAgentWorker(Worker):
     Args:
         description (str): Description of the node.
         worker (ChatAgent): Worker of the node. A single agent.
+        max_concurrent_tasks (int): Maximum number of tasks this worker can
+            process concurrently. (default: :obj:`10`)
     """
 
     def __init__(
         self,
         description: str,
         worker: ChatAgent,
+        max_concurrent_tasks: int = 10,
     ) -> None:
         node_id = worker.agent_id
-        super().__init__(description, node_id=node_id)
+        super().__init__(
+            description,
+            node_id=node_id,
+            max_concurrent_tasks=max_concurrent_tasks,
+        )
         self.worker = worker
 
     def reset(self) -> Any:
@@ -51,11 +58,12 @@ class SingleAgentWorker(Worker):
     async def _process_task(
         self, task: Task, dependencies: List[Task]
     ) -> TaskState:
-        r"""Processes a task with its dependencies.
+        r"""Processes a task with its dependencies using a cloned agent.
 
         This method asynchronously processes a given task, considering its
-        dependencies, by sending a generated prompt to a worker. It updates
-        the task's result based on the agent's response.
+        dependencies, by sending a generated prompt to a cloned worker agent.
+        Using a cloned agent ensures that concurrent tasks don't interfere
+        with each other's state.
 
         Args:
             task (Task): The task to process, which includes necessary details
@@ -66,6 +74,10 @@ class SingleAgentWorker(Worker):
             TaskState: `TaskState.DONE` if processed successfully, otherwise
                 `TaskState.FAILED`.
         """
+        # Clone the agent for this specific task to avoid state conflicts
+        # when processing multiple tasks concurrently
+        worker_agent = self.worker.clone(with_memory=False)
+
         dependency_tasks_info = self._get_dep_tasks_info(dependencies)
         prompt = PROCESS_TASK_PROMPT.format(
             content=task.content,
@@ -73,7 +85,7 @@ class SingleAgentWorker(Worker):
             additional_info=task.additional_info,
         )
         try:
-            response = await self.worker.astep(
+            response = await worker_agent.astep(
                 prompt, response_format=TaskResult
             )
         except Exception as e:
@@ -83,6 +95,13 @@ class SingleAgentWorker(Worker):
             )
             return TaskState.FAILED
 
+        # Get actual token usage from the cloned agent that processed this task
+        try:
+            _, total_token_count = worker_agent.memory.get_context()
+        except Exception:
+            # Fallback if memory context unavailable
+            total_token_count = 0
+
         # Populate additional_info with worker attempt details
         if task.additional_info is None:
             task.additional_info = {}
@@ -90,14 +109,20 @@ class SingleAgentWorker(Worker):
         # Create worker attempt details with descriptive keys
         worker_attempt_details = {
             "agent_id": getattr(
+                worker_agent, "agent_id", worker_agent.role_name
+            ),
+            "original_worker_id": getattr(
                 self.worker, "agent_id", self.worker.role_name
             ),
             "timestamp": str(datetime.datetime.now()),
             "description": f"Attempt by "
-            f"{getattr(self.worker, 'agent_id', self.worker.role_name)} "
+            f"{getattr(worker_agent, 'agent_id', worker_agent.role_name)} "
+            f"(cloned from "
+            f"{getattr(self.worker, 'agent_id', self.worker.role_name)}) "
             f"to process task {task.content}",
             "response_content": response.msg.content,
             "tool_calls": response.info["tool_calls"],
+            "total_token_count": total_token_count,
         }
 
         # Store the worker attempt in additional_info
@@ -105,6 +130,11 @@ class SingleAgentWorker(Worker):
             task.additional_info["worker_attempts"] = []
         task.additional_info["worker_attempts"].append(worker_attempt_details)
 
+        # Store the actual token usage for this specific task
+        task.additional_info["token_usage"] = {
+            "total_tokens": total_token_count
+        }
+
         print(f"======\n{Fore.GREEN}Reply from {self}:{Fore.RESET}")
 
         result_dict = json.loads(response.msg.content)

camel/societies/workforce/task_channel.py

@@ -23,6 +23,8 @@ class PacketStatus(Enum):
     states:
 
     - ``SENT``: The packet has been sent to a worker.
+    - ``PROCESSING``: The packet has been claimed by a worker and is being
+    processed.
     - ``RETURNED``: The packet has been returned by the worker, meaning that
       the status of the task inside has been updated.
     - ``ARCHIVED``: The packet has been archived, meaning that the content of
@@ -31,6 +33,7 @@ class PacketStatus(Enum):
     """
 
     SENT = "SENT"
+    PROCESSING = "PROCESSING"
     RETURNED = "RETURNED"
     ARCHIVED = "ARCHIVED"
 
@@ -97,8 +100,9 @@ class TaskChannel:
                 await self._condition.wait()
 
     async def get_assigned_task_by_assignee(self, assignee_id: str) -> Task:
-        r"""Get a task from the channel that has been assigned to the
-        assignee.
+        r"""Atomically get and claim a task from the channel that has been
+        assigned to the assignee. This prevents race conditions where multiple
+        concurrent calls might retrieve the same task.
         """
         async with self._condition:
             while True:
@@ -107,6 +111,9 @@ class TaskChannel:
                         packet.status == PacketStatus.SENT
                         and packet.assignee_id == assignee_id
                     ):
+                        # Atomically claim the task by changing its status
+                        packet.status = PacketStatus.PROCESSING
+                        self._condition.notify_all()
                         return packet.task
                 await self._condition.wait()