clawbench-cli 0.1.2__py3-none-any.whl

clawbench/data/test-cases/179-dev-tech-github-ops-github/extra_info/config.json

@@ -0,0 +1,13 @@
+{
+  "repo_name": "openclaw-agent-benchmark",
+  "visibility": "public",
+  "license": "MIT",
+  "description": "A benchmark for evaluating browser agents on real-world web tasks",
+  "topics": [
+    "benchmark",
+    "browser-agent",
+    "evaluation"
+  ],
+  "has_wiki": false,
+  "has_issues": true
+}

clawbench/data/test-cases/179-dev-tech-github-ops-github/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 179,
+    "metaclass": "dev-tech",
+    "class": "github-ops",
+    "description": "Create repo \"openclaw-agent-benchmark\" on GitHub + README + MIT LICENSE + .gitignore",
+    "sites_involved": [
+      "github.com"
+    ],
+    "platform": "github",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create repo \"openclaw-agent-benchmark\" on GitHub + README + MIT LICENSE + .gitignore",
+  "eval_schema": {
+    "url_pattern": "github\\.com/repositories$",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/config.json",
+      "description": "Configuration details for the task"
+    }
+  ]
+}

clawbench/data/test-cases/180-dev-tech-github-ops-github/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 180,
+    "metaclass": "dev-tech",
+    "class": "github-ops",
+    "description": "Fork huggingface/transformers on GitHub and create branch \"fix-tokenizer-bug\"",
+    "sites_involved": [
+      "github.com"
+    ],
+    "platform": "github",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Fork huggingface/transformers on GitHub and create branch \"fix-tokenizer-bug\"",
+  "eval_schema": {
+    "url_pattern": "github\\.com/[^/]+/transformers/branches$",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/215-academia-research-paper-tables-overleaf/extra_info/raw_results.json

@@ -0,0 +1,47 @@
+{
+  "models": [
+    "GPT-4o",
+    "Claude 3.5 Sonnet",
+    "Gemini 1.5 Pro",
+    "Llama 3.1 70B",
+    "Mistral Large"
+  ],
+  "metrics": [
+    "Accuracy",
+    "F1-Score",
+    "Latency (ms)",
+    "Cost ($/1K tokens)"
+  ],
+  "results": {
+    "GPT-4o": [
+      0.923,
+      0.918,
+      342,
+      0.015
+    ],
+    "Claude 3.5 Sonnet": [
+      0.931,
+      0.926,
+      298,
+      0.018
+    ],
+    "Gemini 1.5 Pro": [
+      0.908,
+      0.902,
+      410,
+      0.007
+    ],
+    "Llama 3.1 70B": [
+      0.887,
+      0.881,
+      520,
+      0.001
+    ],
+    "Mistral Large": [
+      0.901,
+      0.895,
+      380,
+      0.008
+    ]
+  }
+}

clawbench/data/test-cases/215-academia-research-paper-tables-overleaf/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 215,
+    "metaclass": "academia-research",
+    "class": "paper-tables",
+    "description": "On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table.",
+    "sites_involved": [
+      "overleaf.com"
+    ],
+    "platform": "overleaf",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table.",
+  "eval_schema": {
+    "url_pattern": "overleaf\\.com/project/[a-f0-9]+/compile",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/raw_results.json",
+      "description": "Raw benchmark results: 5 models × 4 metrics"
+    }
+  ]
+}

clawbench/data/test-cases/242-academia-research-research-tools-overleaf/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 242,
+    "metaclass": "academia-research",
+    "class": "research-tools",
+    "description": "Overleaf: Create project \"CVPR2026_Submission\" using CVPR 2026 template",
+    "sites_involved": [
+      "overleaf.com"
+    ],
+    "platform": "overleaf",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Overleaf: Create project \"CVPR2026_Submission\" using CVPR 2026 template",
+  "eval_schema": {
+    "url_pattern": "overleaf\\.com/project/[a-f0-9]+/settings",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/246-academia-research-research-tools-zotero/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 246,
+    "metaclass": "academia-research",
+    "class": "research-tools",
+    "description": "Zotero Web: Batch import papers from 5 URLs with auto metadata",
+    "sites_involved": [
+      "zotero.org"
+    ],
+    "platform": "zotero",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Zotero Web: Batch import papers from 5 URLs with auto metadata",
+  "eval_schema": {
+    "url_pattern": "api\\.zotero\\.org/users/\\d+/items",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/247-academia-research-research-tools-semantic-scholar/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 247,
+    "metaclass": "academia-research",
+    "class": "research-tools",
+    "description": "Semantic Scholar: Create feed for \"LLMs\"",
+    "sites_involved": [
+      "semanticscholar.org"
+    ],
+    "platform": "semantic-scholar",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Semantic Scholar: Create feed for \"LLMs\"",
+  "eval_schema": {
+    "url_pattern": "semanticscholar\\.org/api/1/alert",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/265-education-learning-general-coursera/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 265,
+    "metaclass": "education-learning",
+    "class": "general",
+    "description": "On Coursera, enroll in the \"Deep Learning Specialization\" by reaching the checkout or payment page with enrollment details filled in.",
+    "sites_involved": [
+      "coursera.org"
+    ],
+    "platform": "coursera",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Coursera, enroll in the \"Deep Learning Specialization\" by reaching the checkout or payment page with enrollment details filled in.",
+  "eval_schema": {
+    "url_pattern": "coursera\\.org/api/subscriptions\\.v1/\\?action=createCart",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/266-education-learning-general-leetcode/extra_info/solution_code.py

@@ -0,0 +1,9 @@
+class Solution:
+    def twoSum(self, nums: list[int], target: int) -> list[int]:
+        seen = {}
+        for i, num in enumerate(nums):
+            complement = target - num
+            if complement in seen:
+                return [seen[complement], i]
+            seen[num] = i
+        return []

clawbench/data/test-cases/266-education-learning-general-leetcode/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 266,
+    "metaclass": "education-learning",
+    "class": "general",
+    "description": "On LeetCode, open the \"Two Sum\" problem, write a correct Python solution in the code editor, and click the Submit button.",
+    "sites_involved": [
+      "leetcode.com"
+    ],
+    "platform": "leetcode",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On LeetCode, open the \"Two Sum\" problem, write a correct Python solution in the code editor, and click the Submit button.",
+  "eval_schema": {
+    "url_pattern": "leetcode\\.com/problems/two-sum/submit",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/solution_code.py",
+      "description": "Python solution for the Two Sum problem"
+    }
+  ]
+}

clawbench/data/test-cases/273-education-learning-general-edx/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 273,
+    "metaclass": "education-learning",
+    "class": "general",
+    "description": "On edX, enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible.",
+    "sites_involved": [
+      "edx.org"
+    ],
+    "platform": "edx",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On edX, enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible.",
+  "eval_schema": {
+    "url_pattern": "www\\.edx\\.org/track-select/",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/274-education-learning-general-udemy/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 274,
+    "metaclass": "education-learning",
+    "class": "general",
+    "description": "Udemy: Purchase \"Complete Python Bootcamp\"",
+    "sites_involved": [
+      "udemy.com"
+    ],
+    "platform": "udemy",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Udemy: Purchase \"Complete Python Bootcamp\"",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/279-travel-general-airbnb/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 279,
+    "metaclass": "travel",
+    "class": "general",
+    "description": "On Airbnb, search for an apartment in Shibuya, Tokyo for July 01–07 next year for 2 guests, select a listing, and proceed to the \"Confirm and pay\" page with all guest and payment details filled in.",
+    "sites_involved": [
+      "airbnb.com"
+    ],
+    "platform": "airbnb",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Airbnb, search for an apartment in Shibuya, Tokyo for July 01–07 next year for 2 guests, select a listing, and proceed to the \"Confirm and pay\" page with all guest and payment details filled in.",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/280-travel-general-booking-com/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 280,
+    "metaclass": "travel",
+    "class": "general",
+    "description": "Booking.com: Book a hotel in central Paris for 3 nights next August, double room, breakfast included, free cancellation",
+    "sites_involved": [
+      "booking.com"
+    ],
+    "platform": "booking-com",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Booking.com: Book a hotel in central Paris for 3 nights next August, double room, breakfast included, free cancellation",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/363-entertainment-hobbies-general-ticketmaster/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 363,
+    "metaclass": "entertainment-hobbies",
+    "class": "general",
+    "description": "Ticketmaster: Buy 2 tickets for an upcoming Raptors home game",
+    "sites_involved": [
+      "ticketmaster.ca"
+    ],
+    "platform": "ticketmaster",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Ticketmaster: Buy 2 tickets for an upcoming Raptors home game",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/369-entertainment-hobbies-general-goodreads/extra_info/book_list.json

@@ -0,0 +1,14 @@
+{
+  "books": [
+    "Deep Learning by Ian Goodfellow",
+    "Pattern Recognition and Machine Learning by Christopher Bishop",
+    "The Hundred-Page Machine Learning Book by Andriy Burkov",
+    "Hands-On Machine Learning with Scikit-Learn, Keras, and TensorFlow by Aur\u00e9lien G\u00e9ron",
+    "Machine Learning: A Probabilistic Perspective by Kevin Murphy",
+    "Reinforcement Learning: An Introduction by Richard Sutton",
+    "Speech and Language Processing by Dan Jurafsky",
+    "Information Theory, Inference and Learning Algorithms by David MacKay",
+    "Probabilistic Graphical Models by Daphne Koller",
+    "The Elements of Statistical Learning by Trevor Hastie"
+  ]
+}

clawbench/data/test-cases/369-entertainment-hobbies-general-goodreads/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 369,
+    "metaclass": "entertainment-hobbies",
+    "class": "general",
+    "description": "Goodreads: Create a shelf \"ML Must-Reads\" and add 10 books",
+    "sites_involved": [
+      "goodreads.com"
+    ],
+    "platform": "goodreads",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Goodreads: Create a shelf \"ML Must-Reads\" and add 10 books",
+  "eval_schema": {
+    "url_pattern": "goodreads\\.com/shelf/add_to_shelf",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/book_list.json",
+      "description": "List of 10 ML books to add to shelf"
+    }
+  ]
+}

clawbench/data/test-cases/372-entertainment-hobbies-general-eventbrite/extra_info/event_details.json

@@ -0,0 +1,10 @@
+{
+  "event_name": "ML Paper Reading Group",
+  "event_type": "free",
+  "date": "2026-04-15",
+  "time": "6:00 PM - 8:00 PM ET",
+  "location": "Online (Zoom)",
+  "description": "Weekly paper reading group focusing on recent ML research. This week: attention mechanisms and transformer architectures. All levels welcome.",
+  "capacity": 50,
+  "organizer": "Alex Green"
+}

clawbench/data/test-cases/372-entertainment-hobbies-general-eventbrite/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 372,
+    "metaclass": "entertainment-hobbies",
+    "class": "general",
+    "description": "Eventbrite: Create a free event \"ML Paper Reading Group\"",
+    "sites_involved": [
+      "eventbrite.com"
+    ],
+    "platform": "eventbrite",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Eventbrite: Create a free event \"ML Paper Reading Group\"",
+  "eval_schema": {
+    "url_pattern": "www\\.eventbrite\\.com/api/v3/organizations/\\d+/events/auto-create",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/event_details.json",
+      "description": "Event details including name, date, time, location, and description"
+    }
+  ]
+}

clawbench/data/test-cases/403-personal-management-account-security-1password-web/extra_info/credentials.json

@@ -0,0 +1,34 @@
+{
+  "entries": [
+    {
+      "site": "github.com",
+      "username": "alexgreen-dev",
+      "password": "Gh$ecure2026!ax",
+      "notes": "Personal GitHub account"
+    },
+    {
+      "site": "aws.amazon.com",
+      "username": "alex.green@pinecresttech.com",
+      "password": "AwS#Pr0d2026!kx",
+      "notes": "Work AWS console"
+    },
+    {
+      "site": "slack.com",
+      "username": "alex.green@pinecresttech.com",
+      "password": "Sl@ckW0rk2026!mn",
+      "notes": "Pinecrest Slack"
+    },
+    {
+      "site": "jira.atlassian.com",
+      "username": "alex.green@pinecresttech.com",
+      "password": "J1r@Pr0j2026!vz",
+      "notes": "Project management"
+    },
+    {
+      "site": "notion.so",
+      "username": "alex.green.uoft@clawbench.cc",
+      "password": "N0t10n2026!Pers",
+      "notes": "Personal Notion"
+    }
+  ]
+}

clawbench/data/test-cases/403-personal-management-account-security-1password-web/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 403,
+    "metaclass": "personal-management",
+    "class": "account-security",
+    "description": "Add 5 login entries to 1Password",
+    "sites_involved": [
+      "1password.com"
+    ],
+    "platform": "1password-web",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Add 5 login entries to 1Password",
+  "eval_schema": {
+    "url_pattern": "my\\.1password\\.com/api/v3/vault/[a-z0-9]+/\\d+/items",
+    "method": "PATCH"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/credentials.json",
+      "description": "Login credentials for 5 services to add to password manager"
+    }
+  ]
+}

clawbench/data/test-cases/413-personal-management-personal-tools-todoist/extra_info/task_list.json

@@ -0,0 +1,52 @@
+{
+  "tasks": [
+    {
+      "name": "Define Q3 OKRs",
+      "assignee": "Alex Green",
+      "due_date": "2026-04-07",
+      "priority": "high"
+    },
+    {
+      "name": "Review architecture RFC",
+      "assignee": "Jordan Peters",
+      "due_date": "2026-04-10",
+      "priority": "high"
+    },
+    {
+      "name": "Set up CI/CD pipeline",
+      "assignee": "Alex Green",
+      "due_date": "2026-04-14",
+      "priority": "medium"
+    },
+    {
+      "name": "Write integration tests",
+      "assignee": "Alex Green",
+      "due_date": "2026-04-18",
+      "priority": "medium"
+    },
+    {
+      "name": "Prepare sprint demo",
+      "assignee": "Jordan Peters",
+      "due_date": "2026-04-21",
+      "priority": "low"
+    },
+    {
+      "name": "Update documentation",
+      "assignee": "Alex Green",
+      "due_date": "2026-04-25",
+      "priority": "low"
+    },
+    {
+      "name": "Security audit review",
+      "assignee": "Alex Green",
+      "due_date": "2026-04-28",
+      "priority": "high"
+    },
+    {
+      "name": "Performance benchmarking",
+      "assignee": "Alex Green",
+      "due_date": "2026-04-30",
+      "priority": "medium"
+    }
+  ]
+}