clawbench-cli 0.1.2__py3-none-any.whl

clawbench/data/test-cases/413-personal-management-personal-tools-todoist/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 413,
+    "metaclass": "personal-management",
+    "class": "personal-tools",
+    "description": "Create project \"Q2 Engineering Goals\" with 8 tasks, due dates, and priorities in Todoist",
+    "sites_involved": [
+      "todoist.com"
+    ],
+    "platform": "todoist",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create project \"Q2 Engineering Goals\" with 8 tasks, due dates, and priorities in Todoist",
+  "eval_schema": {
+    "url_pattern": "app\\.todoist\\.com/api/v1/sync",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/task_list.json",
+      "description": "Task list with names, assignees, due dates, and priorities"
+    }
+  ]
+}

clawbench/data/test-cases/468-rating-voting-general-glassdoor/extra_info/interview_experience.json

@@ -0,0 +1,10 @@
+{
+  "company": "TechCorp Inc.",
+  "position": "Senior Software Engineer",
+  "date": "2026-02-15",
+  "interview_type": "On-site (4 rounds)",
+  "difficulty": "Medium-Hard",
+  "experience": "Positive overall. The team was friendly and the questions were relevant to the role. Had a system design round focusing on distributed caching, two coding rounds (one on graph algorithms, one on API design), and a behavioral round.",
+  "offer_received": true,
+  "would_recommend": true
+}

clawbench/data/test-cases/468-rating-voting-general-glassdoor/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 468,
+    "metaclass": "rating-voting",
+    "class": "general",
+    "description": "Write an interview review on Glassdoor",
+    "sites_involved": [
+      "glassdoor.com"
+    ],
+    "platform": "glassdoor",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Write an interview review on Glassdoor",
+  "eval_schema": {
+    "url_pattern": "glassdoor\\.com/(graph|api|a/interview)",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/interview_experience.json",
+      "description": "Interview experience details for review submission"
+    }
+  ]
+}

clawbench/data/test-cases/469-rating-voting-general-tripadvisor/extra_info/review_content.json

@@ -0,0 +1,6 @@
+{
+  "review_text": "Excellent service and product quality. The user experience is intuitive and well-designed. Would recommend to colleagues.",
+  "rating": 5,
+  "pros": "Easy to use, great customer support, reliable service",
+  "cons": "Pricing could be more competitive for small teams"
+}

clawbench/data/test-cases/469-rating-voting-general-tripadvisor/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 469,
+    "metaclass": "rating-voting",
+    "class": "general",
+    "description": "Write a 4-star review for a tour on TripAdvisor",
+    "sites_involved": [
+      "tripadvisor.com"
+    ],
+    "platform": "tripadvisor",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Write a 4-star review for a tour on TripAdvisor",
+  "eval_schema": {
+    "url_pattern": "tripadvisor\\.com/data/graphql/ids",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/review_content.json",
+      "description": "Review content for submission"
+    }
+  ]
+}

clawbench/data/test-cases/470-rating-voting-general-trustpilot/extra_info/review_content.json

@@ -0,0 +1,6 @@
+{
+  "review_text": "Excellent service and product quality. The user experience is intuitive and well-designed. Would recommend to colleagues.",
+  "rating": 5,
+  "pros": "Easy to use, great customer support, reliable service",
+  "cons": "Pricing could be more competitive for small teams"
+}

clawbench/data/test-cases/470-rating-voting-general-trustpilot/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 470,
+    "metaclass": "rating-voting",
+    "class": "general",
+    "description": "Write a 5-star review for Wealthsimple on Trustpilot",
+    "sites_involved": [
+      "trustpilot.com"
+    ],
+    "platform": "trustpilot",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Write a 5-star review for Wealthsimple on Trustpilot",
+  "eval_schema": {
+    "url_pattern": "trustpilot\\.com/api/consumer/review",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/review_content.json",
+      "description": "Review content for submission"
+    }
+  ]
+}

clawbench/data/test-cases/474-rating-voting-general-capterra/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 474,
+    "metaclass": "rating-voting",
+    "class": "general",
+    "description": "Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: \"Flexible workspace, steep learning curve.\"",
+    "sites_involved": [
+      "capterra.com"
+    ],
+    "platform": "capterra",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: \"Flexible workspace, steep learning curve.\"",
+  "eval_schema": {
+    "url_pattern": "reviews\\.capterra\\.com/api/v1/products/[^/]+/partial-review",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/475-rating-voting-general-g2/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 475,
+    "metaclass": "rating-voting",
+    "class": "general",
+    "description": "Write a 4-star review for Jira on G2 with exactly this text: \"Powerful tracking, complex UI.\"",
+    "sites_involved": [
+      "g2.com"
+    ],
+    "platform": "g2",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Write a 4-star review for Jira on G2 with exactly this text: \"Powerful tracking, complex UI.\"",
+  "eval_schema": {
+    "url_pattern": "g2\\.com/survey_responses/.+/autosave_answers\\.json",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/482-creation-init-general-confluence/extra_info/content.json

@@ -0,0 +1,3 @@
+{
+  "content": "Sharing our latest research on scaling language models \u2014 key findings on training dynamics, emergent capabilities, and practical deployment considerations. #MachineLearning #AI #Research"
+}

clawbench/data/test-cases/482-creation-init-general-confluence/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 482,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Create a Confluence space \"Research Wiki\" with a home page and 3 sub-pages",
+    "sites_involved": [
+      "atlassian.com/software/confluence"
+    ],
+    "platform": "confluence",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create a Confluence space \"Research Wiki\" with a home page and 3 sub-pages",
+  "eval_schema": {
+    "url_pattern": "atlassian\\.net/wiki/rest/create-dialog/1\\.0/space-blueprint/create-space",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/content.json",
+      "description": "Content and links for the task"
+    }
+  ]
+}

clawbench/data/test-cases/483-creation-init-general-airtable/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 483,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Create an Airtable base \"Conference Tracker\" with schema",
+    "sites_involved": [
+      "airtable.com"
+    ],
+    "platform": "airtable",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create an Airtable base \"Conference Tracker\" with schema",
+  "eval_schema": {
+    "url_pattern": "airtable\\.com/v0\\.3/application/[^/]+/create",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/484-creation-init-general-clickup/task.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 484,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "On ClickUp, create a new workspace, then create a project named \"Benchmark Development\" with at least one custom field added to the project.",
+    "sites_involved": [
+      "clickup.com"
+    ],
+    "platform": "clickup",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On ClickUp, create a new workspace, then create a project named \"Benchmark Development\" with at least one custom field added to the project.",
+  "eval_schema": {
+    "url_pattern": "clickup\\.com/hierarchy/v1/project",
+    "method": "POST",
+    "body": {
+      "name": "Benchmark Development"
+    }
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/485-creation-init-general-webflow/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 485,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Create a portfolio website project on Webflow",
+    "sites_involved": [
+      "webflow.com"
+    ],
+    "platform": "webflow",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create a portfolio website project on Webflow",
+  "eval_schema": {
+    "url_pattern": "webflow\\.com/api/workspaces/[^/]+/sites",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/486-creation-init-general-mailchimp/extra_info/content.json

@@ -0,0 +1,3 @@
+{
+  "content": "Sharing our latest research on scaling language models \u2014 key findings on training dynamics, emergent capabilities, and practical deployment considerations. #MachineLearning #AI #Research"
+}

clawbench/data/test-cases/486-creation-init-general-mailchimp/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 486,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Create a Mailchimp email campaign with audience set to research newsletter",
+    "sites_involved": [
+      "mailchimp.com"
+    ],
+    "platform": "mailchimp",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create a Mailchimp email campaign with audience set to research newsletter",
+  "eval_schema": {
+    "url_pattern": "admin\\.mailchimp\\.com/campaigns/(wizard|send|schedule)",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/content.json",
+      "description": "Content and links for the task"
+    }
+  ]
+}

clawbench/data/test-cases/487-creation-init-general-typeform/extra_info/survey_questions.json

@@ -0,0 +1,85 @@
+{
+  "title": "User Research Survey: Developer Productivity",
+  "questions": [
+    {
+      "id": 1,
+      "type": "multiple_choice",
+      "text": "What is your primary programming language?",
+      "options": [
+        "Python",
+        "JavaScript/TypeScript",
+        "Go",
+        "Java",
+        "Rust",
+        "Other"
+      ]
+    },
+    {
+      "id": 2,
+      "type": "rating",
+      "text": "How satisfied are you with your current IDE?",
+      "scale": "1-5"
+    },
+    {
+      "id": 3,
+      "type": "short_text",
+      "text": "What is the biggest bottleneck in your daily workflow?"
+    },
+    {
+      "id": 4,
+      "type": "multiple_choice",
+      "text": "How many hours per week do you spend on code reviews?",
+      "options": [
+        "0-2",
+        "3-5",
+        "6-10",
+        "10+"
+      ]
+    },
+    {
+      "id": 5,
+      "type": "yes_no",
+      "text": "Do you use AI-assisted coding tools?"
+    },
+    {
+      "id": 6,
+      "type": "long_text",
+      "text": "Describe your ideal development environment."
+    },
+    {
+      "id": 7,
+      "type": "multiple_choice",
+      "text": "What deployment method do you use most?",
+      "options": [
+        "Docker/K8s",
+        "Serverless",
+        "VMs",
+        "PaaS",
+        "Other"
+      ]
+    },
+    {
+      "id": 8,
+      "type": "rating",
+      "text": "How would you rate your team's CI/CD pipeline?",
+      "scale": "1-5"
+    },
+    {
+      "id": 9,
+      "type": "short_text",
+      "text": "What tool do you wish existed?"
+    },
+    {
+      "id": 10,
+      "type": "multiple_choice",
+      "text": "How do you prefer to learn new technologies?",
+      "options": [
+        "Documentation",
+        "Video tutorials",
+        "Hands-on projects",
+        "Courses",
+        "Peer learning"
+      ]
+    }
+  ]
+}

clawbench/data/test-cases/487-creation-init-general-typeform/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 487,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Create a Typeform user research survey with 10 questions",
+    "sites_involved": [
+      "typeform.com"
+    ],
+    "platform": "typeform",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create a Typeform user research survey with 10 questions",
+  "eval_schema": {
+    "url_pattern": "admin\\.typeform\\.com/bff/bob-the-builder/forms/",
+    "method": "PUT"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/survey_questions.json",
+      "description": "Survey questions for user research with 10 questions of various types"
+    }
+  ]
+}

clawbench/data/test-cases/488-creation-init-general-substack/extra_info/content.json

@@ -0,0 +1,3 @@
+{
+  "content": "Sharing our latest research on scaling language models \u2014 key findings on training dynamics, emergent capabilities, and practical deployment considerations. #MachineLearning #AI #Research"
+}

clawbench/data/test-cases/488-creation-init-general-substack/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 488,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Create a Substack newsletter \"ML Research Roundup\" and publish the first issue",
+    "sites_involved": [
+      "substack.com"
+    ],
+    "platform": "substack",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create a Substack newsletter \"ML Research Roundup\" and publish the first issue",
+  "eval_schema": {
+    "url_pattern": "substack\\.com/api/v1/drafts/\\d+/publish",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/content.json",
+      "description": "Content and links for the task"
+    }
+  ]
+}

clawbench/data/test-cases/489-creation-init-general-ghost/extra_info/content.json

@@ -0,0 +1,3 @@
+{
+  "content": "Sharing our latest research on scaling language models \u2014 key findings on training dynamics, emergent capabilities, and practical deployment considerations. #MachineLearning #AI #Research"
+}

clawbench/data/test-cases/489-creation-init-general-ghost/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 489,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Set up a Ghost blog and publish a \"Hello World\" post",
+    "sites_involved": [
+      "ghost.org"
+    ],
+    "platform": "ghost",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Set up a Ghost blog and publish a \"Hello World\" post",
+  "eval_schema": {
+    "url_pattern": "ghost\\.io/ghost/api/(admin|content)/posts",
+    "method": "PUT"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/content.json",
+      "description": "Content and links for the task"
+    }
+  ]
+}

clawbench/data/test-cases/501-creation-init-general-asana/extra_info/project_description.json

@@ -0,0 +1,8 @@
+{
+  "project_type": "bathroom renovation",
+  "description": "Full bathroom renovation for a standard-size condo bathroom (approx. 5x8 ft). Looking to replace the bathtub with a walk-in shower, update the vanity and mirror, install new tile flooring, and add a heated towel rack. Prefer modern minimalist style with neutral tones.",
+  "budget_range": "$8,000 - $15,000 CAD",
+  "timeline": "4-6 weeks",
+  "location": "Toronto, ON (condo building)",
+  "special_requirements": "Must comply with condo renovation rules. Need quiet hours (no work before 9am or after 5pm)."
+}

clawbench/data/test-cases/501-creation-init-general-asana/task.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 501,
+    "metaclass": "creation-init",
+    "class": "general",
+    "description": "Create an Asana portfolio \"Research Projects 2026\" with 3 sub-projects",
+    "sites_involved": [
+      "app.asana.com"
+    ],
+    "platform": "asana",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Create an Asana portfolio \"Research Projects 2026\" with 3 sub-projects",
+  "eval_schema": {
+    "url_pattern": "app\\.asana\\.com/app/asana/-/report_execution_context_activity",
+    "method": "POST",
+    "body": {
+      "context_identifier": "PotGridRoot"
+    }
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/project_description.json",
+      "description": "Project description with scope, budget, and requirements"
+    }
+  ]
+}

clawbench/data/test-cases/529-daily-life-shopping-delivery-king-arthur-baking/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 529,
+    "metaclass": "daily-life",
+    "class": "shopping-delivery",
+    "description": "On the King Arthur Baking website, purchase a Sourdough Starter Kit, select standard size, and complete checkout",
+    "sites_involved": [
+      "kingarthurbaking.com"
+    ],
+    "platform": "king-arthur-baking",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On the King Arthur Baking website, purchase a Sourdough Starter Kit, select standard size, and complete checkout",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/533-daily-life-utilities-inmyarea/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 533,
+    "metaclass": "daily-life",
+    "class": "utilities",
+    "description": "On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription",
+    "sites_involved": [
+      "inmyarea.com"
+    ],
+    "platform": "inmyarea",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription",
+  "eval_schema": {
+    "url_pattern": "att\\.com/msapi/salesapi/wireless-sales-eapi/v2/addtocart",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}