camel-ai 0.2.59__py3-none-any.whl → 0.2.82__py3-none-any.whl

camel/benchmarks/__init__.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 from .apibank import APIBankBenchmark
 from .apibench import APIBenchBenchmark

camel/benchmarks/apibank.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import json
 import logging
@@ -542,9 +542,9 @@ replace the ApiName with the actual API name, and \
 replace the key and value with the actual parameters. \
 Your output should start with a square bracket "[" \
 and end with a square bracket "]". Do not output any \
-other explanation or prompt or the result of the API call in your output. 
+other explanation or prompt or the result of the API call in your output.
 This year is 2023.
-Input: 
+Input:
 User: [User's utterence]
 AI: [AI's utterence]
 
@@ -559,7 +559,7 @@ Based on the given API description and the existing \
 conversation history 1..t, please generate the next \
 dialog that the AI should response after the API call t.
 This year is 2023.
-Input: 
+Input:
 User: [User's utterence]
 AI: [AI's utterence]
 [ApiName(key1='value1', key2='value2', …)]

camel/benchmarks/apibench.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import json
 import logging

camel/benchmarks/base.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import logging
 from abc import ABC, abstractmethod

camel/benchmarks/browsecomp.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import base64
 import hashlib
@@ -55,7 +55,7 @@ class QueryResponse(BaseModel):
     )
     exact_answer: str = Field(description="""your succinct, final answer.""")
     confidence: str = Field(
-        description="""
+        description=r"""
 your confidence score between 0|\%| and 100|\%| for your answer.
 """
     )
@@ -72,27 +72,27 @@ class GradingResponse(BaseModel):
     extracted_final_answer: str = Field(
         description="""
 The final exact answer extracted from the [response].
-Put the extracted answer as 'None' if there is no exact, final answer to 
+Put the extracted answer as 'None' if there is no exact, final answer to
 extract from the response."""
     )
     reasoning: str = Field(
         description="""
-Explain why the extracted_final_answer is correct or incorrect 
-based on [correct_answer], focusing only on if there are meaningful 
-differences between [correct_answer] and the extracted_final_answer. 
-Do not comment on any background to the problem, do not attempt 
-to solve the problem, do not argue for any answer different 
+Explain why the extracted_final_answer is correct or incorrect
+based on [correct_answer], focusing only on if there are meaningful
+differences between [correct_answer] and the extracted_final_answer.
+Do not comment on any background to the problem, do not attempt
+to solve the problem, do not argue for any answer different
 than [correct_answer], focus only on whether the answers match."""
     )
     correct: str = Field(
-        description="""Answer 'yes' if extracted_final_answer matches the 
-[correct_answer] given above, or is within a small margin of error for 
-numerical problems. Answer 'no' otherwise, i.e. if there if there is any 
-inconsistency, ambiguity, non-equivalency, or if the extracted answer is 
+        description="""Answer 'yes' if extracted_final_answer matches the
+[correct_answer] given above, or is within a small margin of error for
+numerical problems. Answer 'no' otherwise, i.e. if there if there is any
+inconsistency, ambiguity, non-equivalency, or if the extracted answer is
 incorrect."""
     )
     confidence: str = Field(
-        description="""The extracted confidence score between 0|\%| 
+        description=r"""The extracted confidence score between 0|\%|
 and 100|\%| from [response]. Put 100 if there is no confidence score available.
 """
     )
@@ -160,8 +160,8 @@ format content into json:
 {content}
 """
 
-GRADER_TEMPLATE = """
-Judge whether the following [response] to [question] is correct or not 
+GRADER_TEMPLATE = r"""
+Judge whether the following [response] to [question] is correct or not
 based on the precise and unambiguous [correct_answer] below.
 
 [question]: {question}
@@ -171,26 +171,37 @@ based on the precise and unambiguous [correct_answer] below.
 Your judgement must be in the format and criteria specified below:
 
 extracted_final_answer: The final exact answer extracted from the [response].
-Put the extracted answer as 'None' if there is no exact, final answer to 
+Put the extracted answer as 'None' if there is no exact, final answer to
+Put the extracted answer as 'None' if there is no exact, final answer to
 extract from the response.
 
 [correct_answer]: {correct_answer}
 
-reasoning: Explain why the extracted_final_answer is correct or incorrect 
-based on [correct_answer], focusing only on if there are meaningful 
-differences between [correct_answer] and the extracted_final_answer. 
-Do not comment on any background to the problem, do not attempt 
-to solve the problem, do not argue for any answer different 
+reasoning: Explain why the extracted_final_answer is correct or incorrect
+based on [correct_answer], focusing only on if there are meaningful
+differences between [correct_answer] and the extracted_final_answer.
+Do not comment on any background to the problem, do not attempt
+to solve the problem, do not argue for any answer different
+reasoning: Explain why the extracted_final_answer is correct or incorrect
+based on [correct_answer], focusing only on if there are meaningful
+differences between [correct_answer] and the extracted_final_answer.
+Do not comment on any background to the problem, do not attempt
+to solve the problem, do not argue for any answer different
 than [correct_answer], focus only on whether the answers match.
 
-correct: Answer 'yes' if extracted_final_answer matches the 
-[correct_answer] given above, or is within a small margin of error for 
-numerical problems. Answer 'no' otherwise, i.e. if there is any 
-inconsistency, ambiguity, non-equivalency, or if the extracted answer is 
+correct: Answer 'yes' if extracted_final_answer matches the
+[correct_answer] given above, or is within a small margin of error for
+numerical problems. Answer 'no' otherwise, i.e. if there is any
+inconsistency, ambiguity, non-equivalency, or if the extracted answer is
+correct: Answer 'yes' if extracted_final_answer matches the
+[correct_answer] given above, or is within a small margin of error for
+numerical problems. Answer 'no' otherwise, i.e. if there is any
+inconsistency, ambiguity, non-equivalency, or if the extracted answer is
 incorrect.
 
 
-confidence: The extracted confidence score between 0|\%| and 100|\%| 
+confidence: The extracted confidence score between 0|\%| and 100|\%|
+confidence: The extracted confidence score between 0|\%| and 100|\%|
 from [response]. Put 100 if there is no confidence score available.
 """.strip()
 
@@ -619,20 +630,20 @@ class BrowseCompBenchmark(BaseBenchmark):
                         assistant_response, user_response = pipeline.step(
                             input_msg
                         )
-                        if assistant_response.terminated:  # type: ignore[attr-defined]
+                        if assistant_response.terminated:  # type: ignore[union-attr]
                             break
-                        if user_response.terminated:  # type: ignore[attr-defined]
+                        if user_response.terminated:  # type: ignore[union-attr]
                             break
-                        if "CAMEL_TASK_DONE" in user_response.msg.content:  # type: ignore[attr-defined]
+                        if "CAMEL_TASK_DONE" in user_response.msg.content:  # type: ignore[union-attr]
                             break
 
                         chat_history.append(
-                            f"AI User: {user_response.msg.content}"  # type: ignore[attr-defined]
+                            f"AI User: {user_response.msg.content}"  # type: ignore[union-attr]
                         )
                         chat_history.append(
-                            f"AI Assistant: {assistant_response.msg.content}"  # type: ignore[attr-defined]
+                            f"AI Assistant: {assistant_response.msg.content}"  # type: ignore[union-attr]
                         )
-                        input_msg = assistant_response.msg  # type: ignore[attr-defined]
+                        input_msg = assistant_response.msg  # type: ignore[union-attr]
 
                     chat_history_str = "\n".join(chat_history)
                     if roleplaying_summarizer:

camel/benchmarks/gaia.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import json
 import logging
@@ -165,6 +165,8 @@ class GAIABenchmark(BaseBenchmark):
             force_download (bool, optional): Whether to
                 force download the data.
         """
+        import pandas as pd
+
         if force_download:
             logger.info("Force downloading data.")
             self.download()
@@ -181,15 +183,17 @@ class GAIABenchmark(BaseBenchmark):
         # Load metadata for both validation and test datasets
         for path, label in zip([valid_dir, test_dir], ["valid", "test"]):
             self._data[label] = []
-            with open(path / "metadata.jsonl", "r") as f:
-                lines = f.readlines()
-                for line in lines:
-                    data = json.loads(line)
-                    if data["task_id"] == "0-0-0-0-0":
-                        continue
-                    if data["file_name"]:
-                        data["file_name"] = path / data["file_name"]
-                    self._data[label].append(data)
+            metadata_file = path / "metadata.parquet"
+            df = pd.read_parquet(metadata_file)
+            for _, row in df.iterrows():
+                data = row.to_dict()
+                if data["task_id"] == "0-0-0-0-0":
+                    continue
+                # convert level to int (parquet stores as string)
+                data["Level"] = int(data["Level"])
+                if data["file_name"]:
+                    data["file_name"] = path / data["file_name"]
+                self._data[label].append(data)
         return self
 
     @property
@@ -333,7 +337,7 @@ class GAIABenchmark(BaseBenchmark):
         }
         self._results.append(result_data)
         file_obj.write(
-            json.dumps(result_data, indent=2) + "\n", ensure_ascii=False
+            json.dumps(result_data, indent=2, ensure_ascii=False) + "\n"
         )
         file_obj.flush()
 
@@ -354,7 +358,7 @@ class GAIABenchmark(BaseBenchmark):
         }
         self._results.append(error_data)
         file_obj.write(
-            json.dumps(error_data, indent=2) + "\n", ensure_ascii=False
+            json.dumps(error_data, indent=2, ensure_ascii=False) + "\n"
         )
         file_obj.flush()
 

camel/benchmarks/mock_website/README.md

@@ -0,0 +1,94 @@
+# Mock Website Benchmarks for Web Agent Testing
+
+This project provides a framework for testing web agents against various mock websites. It features a central dispatcher (`mock_web.py`) that manages different test environments, each simulating a specific type of website (e.g., an e-commerce site). The initial example project is an Amazon-like shopping website.
+
+## Core Concepts
+
+*   **Dispatcher (`mock_web.py`):** The single entry point for running any benchmark. It handles:
+    *   Downloading necessary web assets (HTML templates, CSS, JS) for a specific project from Hugging Face (https://huggingface.co/datasets/camel-ai/mock_websites).
+    *   Reading the task configuration from `task.json`.
+    *   Launching the project's dedicated Flask web server as a background process.
+    *   Monitoring the server for task completion via API polling.
+    *   Reporting the final results (success status, number of operations) and shutting down the server.
+*   **Projects (e.g., `shopping_mall/`):** Each project is a self-contained Flask application representing a unique website.
+*   **Task Configuration (`task.json`):** A central JSON file that defines the environment and the goal for the agent.
+
+## Example Project: `shopping_mall`
+
+The included `shopping_mall` project simulates an e-commerce website with the following features:
+*   **Product Display:** View a list of products with images, descriptions, ratings, and prices.
+*   **Product Detail Pages:** Click on a product to see its dedicated detail page.
+*   **Shopping Cart:** Add products, view the cart, and manage its contents.
+*   **API-Driven:** The backend provides API endpoints for all state-changing actions.
+
+## Setup and Usage
+
+1.  **Install Dependencies:**
+    ```bash
+    pip install -r requirements.txt
+    ```
+    This will install `Flask`, `huggingface-hub`, `requests`, and other necessary packages.
+
+2.  **Configure the Task:**
+    Edit the `task.json` file to define the products available on the website and the agent's goal. The structure is as follows:
+    ```json
+    {
+        "products": [
+            {
+                "id": 1,
+                "name": "Gaming Laptop",
+                "price": 1200,
+                "image": "assets/img/products/laptop.jpg",
+                "category": "Electronics",
+                "rating": 4.5,
+                "description": "High-performance gaming laptop with latest specs."
+            }
+        ],
+        "ground_truth_cart": [
+            {
+                "id": 1,
+                "quantity": 1
+            }
+        ]
+    }
+    ```
+    *   `products`: A list of all product objects available in the environment.
+    *   `ground_truth_cart`: A list of items that defines the target state of the shopping cart for the task to be considered complete.
+
+3.  **Running the Benchmark:**
+    Use the `mock_web.py` dispatcher to run a project.
+    ```bash
+    python mock_web.py --project shopping_mall
+    ```
+    *   `--project`: Specifies which mock website project to run (default: `shopping_mall`).
+    *   `--port`: Specifies the port to run the server on (default: `5001`).
+
+    The dispatcher will start the server and begin polling for task completion. You or your agent can then interact with the website at `http://127.0.0.1:5001/`. Once the conditions defined in `ground_truth_cart` are met, the dispatcher will automatically detect it, report the results, and shut down. You can also stop it early by pressing `Ctrl+C`.
+
+## Logging
+
+*   `dispatcher.log`: High-level log from the dispatcher, showing setup, status, and final results.
+*   `shopping_mall/app.log`: Detailed internal log from the Flask application for the `shopping_mall` project.
+
+## Project Structure
+
+```
+.
+├── mock_web.py                # Main dispatcher for running benchmarks
+├── task.json                  # Task configuration file
+├── requirements.txt           # Python dependencies
+├── shopping_mall/             # Example project: shopping_mall
+│   └── app.py                 # Flask application for this project
+├── dispatcher.log             # (Generated at runtime)
+└── README.md                  # This file
+```
+The dispatcher will automatically download project-specific `templates` and `static` folders from Hugging Face Hub and place them inside the corresponding project directory at runtime.
+
+## TODO: Automated Question Generation Module
+
+A planned future module for this project is the development of an automated question generation system. This system would analyze the current state of the web application environment (e.g., visible elements, available products, cart status) and generate relevant questions or tasks for a web agent to solve.
+
+This could involve:
+*   Identifying interactable elements and their states.
+*   Understanding the current context (e.g., on product page, in cart).
+*   Formulating natural language questions or goal descriptions based on this context (e.g., "Find a product under $50 in the Electronics category and add it to the cart," or "What is the current subtotal of the cart after adding two units of item X?").