clawbench-cli 0.1.2__py3-none-any.whl

clawbench/data/test-cases/877-entertainment-hobbies-show-tickets-stubhub/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 877,
+    "metaclass": "entertainment-hobbies",
+    "class": "show-tickets",
+    "description": "On StubHub, buy 2 NHL Toronto Maple Leafs home game tickets at Scotiabank Arena, select seats in sections 118–122 (behind the goal), rows 10 or closer",
+    "sites_involved": [
+      "stubhub.ca"
+    ],
+    "platform": "stubhub",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On StubHub, buy 2 NHL Toronto Maple Leafs home game tickets at Scotiabank Arena, select seats in sections 118–122 (behind the goal), rows 10 or closer",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/878-travel-outdoor-ontario-parks/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 878,
+    "metaclass": "travel",
+    "class": "outdoor",
+    "description": "On Ontario Parks, reserve a waterfront campsite at Algonquin Provincial Park (Canisbay Lake campground) for 3 nights, arriving the upcoming Friday, filter for electric hookup sites",
+    "sites_involved": [
+      "reservations.ontarioparks.ca"
+    ],
+    "platform": "ontario-parks",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Ontario Parks, reserve a waterfront campsite at Algonquin Provincial Park (Canisbay Lake campground) for 3 nights, arriving the upcoming Friday, filter for electric hookup sites",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/883-education-learning-hobby-class-sur-la-table/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 883,
+    "metaclass": "education-learning",
+    "class": "hobby-class",
+    "description": "On Sur La Table, register and book an upcoming weekend afternoon in-store cooking class (pasta making) at the closest location to downtown San Francisco for 1 person",
+    "sites_involved": [
+      "surlatable.com"
+    ],
+    "platform": "sur-la-table",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Sur La Table, register and book an upcoming weekend afternoon in-store cooking class (pasta making) at the closest location to downtown San Francisco for 1 person",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/884-entertainment-hobbies-experience-breakout-games/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 884,
+    "metaclass": "entertainment-hobbies",
+    "class": "experience",
+    "description": "On Breakout Games, book an escape room for 4 people at the nearest location to downtown Toronto, select a horror/thriller themed room, the upcoming Saturday evening time slot between 7–9 PM",
+    "sites_involved": [
+      "breakoutgames.com"
+    ],
+    "platform": "breakout-games",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Breakout Games, book an escape room for 4 people at the nearest location to downtown Toronto, select a horror/thriller themed room, the upcoming Saturday evening time slot between 7–9 PM",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/885-entertainment-hobbies-experience-bowlero/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 885,
+    "metaclass": "entertainment-hobbies",
+    "class": "experience",
+    "description": "On Bowlero, reserve a bowling lane for 6 people at the nearest location to Midtown Manhattan, the upcoming Friday night at 8 PM, add the 2-hour unlimited bowling package",
+    "sites_involved": [
+      "bowlero.com"
+    ],
+    "platform": "bowlero",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Bowlero, reserve a bowling lane for 6 people at the nearest location to Midtown Manhattan, the upcoming Friday night at 8 PM, add the 2-hour unlimited bowling package",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/886-entertainment-hobbies-experience-topgolf/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../task.schema.json",
+  "metadata": {
+    "task_id": 886,
+    "metaclass": "entertainment-hobbies",
+    "class": "experience",
+    "description": "On Topgolf, book a bay for 4 players at the Topgolf nearest to downtown Toronto, the upcoming Saturday afternoon between 2–4 PM, select the longest available session",
+    "sites_involved": [
+      "topgolf.com"
+    ],
+    "platform": "topgolf",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Topgolf, book a bay for 4 players at the Topgolf nearest to downtown Toronto, the upcoming Saturday afternoon between 2–4 PM, select the longest available session",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/data/test-cases/lite.json

@@ -0,0 +1,226 @@
+{
+  "$schema": "./lite.schema.json",
+  "version": "0.1",
+  "created": "2026-04-12",
+  "rubric_max": 12,
+  "total_tasks": 20,
+  "notes": "ClawBench-Lite v0.1 — a 20-task curated subset of the 153-task ClawBench. Designed to match the 20-tasks-per-source convention of browser-use/benchmark (https://github.com/browser-use/benchmark) so the subset can eventually be upstreamed as a 6th source alongside WebBench, Mind2Web 2, GAIA, BrowseComp, and Custom. All Lite tasks are judged by eval/agentic_eval.md regardless of whether eval_schema.url_pattern is a concrete regex or __PLACEHOLDER_WILL_NOT_MATCH__ — tasks were selected on task quality / popularity / real-world relevance, not on url_pattern shape. Selection rubric: 4 axes × 3 points each (site popularity, task realism, difficulty, category ambassador), max 12. Binary gates: accessibility (no hard CAPTCHA / geo-lock / mandatory-phone-step-1 / payment wall before satisfiable) and soft novelty (not a clear duplicate of existing benchmark coverage at the same site). All 20 selections use distinct domains for site-level diversity. Swap history during validation: (1) shopping-commerce pool lacked household-name retailers so the slot was swapped to daily-life/housing/zillow 011; (2) zillow 011 was then dropped after recorded runs showed PerimeterX CAPTCHA walls on 4/4 models in batch-20260327 (borderline accessibility-gate violation), and the slot was reallocated to daily-life/food/instacart 007 — housing is therefore no longer a represented vertical in Lite v0.1.",
+  "skipped_categories": [
+    "nonprofit-charity",
+    "home-services-maintenance",
+    "finance-investment",
+    "automotive-vehicle-services",
+    "automation-workflows",
+    "government-civic",
+    "deletion-revocation",
+    "shopping-commerce",
+    "daily-life-housing"
+  ],
+  "tier_distribution": {
+    "flagship": 9,
+    "core": 8,
+    "wildcard": 3
+  },
+  "tasks": [
+    {
+      "task_id": 872,
+      "dir": "872-daily-life-food-opentable",
+      "tier": "flagship",
+      "metaclass": "daily-life",
+      "platform": "opentable",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "OpenTable is the household-name leader for restaurant reservations; dining-out booking is canonical daily-life behavior with multi-step restaurant filtering and seating selection, and the eval_schema has a concrete url_pattern."
+    },
+    {
+      "task_id": 2,
+      "dir": "002-daily-life-food-doordash",
+      "tier": "flagship",
+      "metaclass": "daily-life",
+      "platform": "doordash",
+      "url_pattern_concrete": false,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 1, "ambassador": 3, "total": 10 },
+      "rationale": "DoorDash is the canonical food-delivery flagship on weekly real-world use; simpler flow than OpenTable but an irreplaceable everyday-life ambassador."
+    },
+    {
+      "task_id": 47,
+      "dir": "047-daily-life-personal-care-taskrabbit",
+      "tier": "flagship",
+      "metaclass": "daily-life",
+      "platform": "taskrabbit",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 2, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 10 },
+      "rationale": "TaskRabbit is the vertical leader for gig-services marketplace; booking a helper is a multi-step realistic flow with date/time/service selection, plus concrete url_pattern and extra_info support."
+    },
+    {
+      "task_id": 372,
+      "dir": "372-entertainment-hobbies-general-eventbrite",
+      "tier": "flagship",
+      "metaclass": "entertainment-hobbies",
+      "platform": "eventbrite",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "Eventbrite is the household-name event platform; event creation is a canonical creator-side task with multi-field form, extra_info support, and concrete url_pattern — a rare combination of popularity and eval quality."
+    },
+    {
+      "task_id": 369,
+      "dir": "369-entertainment-hobbies-general-goodreads",
+      "tier": "flagship",
+      "metaclass": "entertainment-hobbies",
+      "platform": "goodreads",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 2, "difficulty": 2, "ambassador": 3, "total": 10 },
+      "rationale": "Goodreads is the household-name reading community; shelf/list creation is realistic hobby curation, with concrete url_pattern and extra_info — distinct vertical from Eventbrite and Fandango."
+    },
+    {
+      "task_id": 867,
+      "dir": "867-entertainment-hobbies-movies-fandango",
+      "tier": "flagship",
+      "metaclass": "entertainment-hobbies",
+      "platform": "fandango",
+      "url_pattern_concrete": false,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "Fandango is the household-name movie-ticket flagship; buying a ticket with theater, showtime, and seat selection is canonical cinema behavior and a distinct entertainment vertical from event creation and reading lists."
+    },
+    {
+      "task_id": 501,
+      "dir": "501-creation-init-general-asana",
+      "tier": "flagship",
+      "metaclass": "creation-init",
+      "platform": "asana",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 2, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 10 },
+      "rationale": "Asana is a B2B SaaS leader for project management; creating a portfolio with three sub-projects is a canonical multi-object creation task, with concrete url_pattern and extra_info."
+    },
+    {
+      "task_id": 486,
+      "dir": "486-creation-init-general-mailchimp",
+      "tier": "flagship",
+      "metaclass": "creation-init",
+      "platform": "mailchimp",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 2, "total": 10 },
+      "rationale": "Mailchimp is the household-name email-marketing leader; authoring a campaign exercises audience selection, content editing, and concrete url_pattern evaluation — a distinct creation vertical from project tooling and website builders."
+    },
+    {
+      "task_id": 712,
+      "dir": "712-creation-init-website-create-squarespace",
+      "tier": "flagship",
+      "metaclass": "creation-init",
+      "platform": "squarespace",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "Squarespace is the household-name website builder; creating a portfolio site is a canonical self-publishing task with meaningful UI depth and concrete url_pattern."
+    },
+    {
+      "task_id": 142,
+      "dir": "142-office-secretary-tasks-collab-trello",
+      "tier": "core",
+      "metaclass": "office-secretary-tasks",
+      "platform": "trello",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 2, "realism": 2, "difficulty": 2, "ambassador": 3, "total": 9 },
+      "rationale": "Trello is the vertical leader for Kanban collaboration; creating a board with lists and cards is canonical office-collaboration behavior with concrete url_pattern — strongest office-secretary task that is not email-client-specific."
+    },
+    {
+      "task_id": 469,
+      "dir": "469-rating-voting-general-tripadvisor",
+      "tier": "core",
+      "metaclass": "rating-voting",
+      "platform": "tripadvisor",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "TripAdvisor is the household-name global travel-review platform; writing a restaurant review is canonical rating behavior with concrete url_pattern — a natural ambassador for user-generated-content tasks."
+    },
+    {
+      "task_id": 266,
+      "dir": "266-education-learning-general-leetcode",
+      "tier": "core",
+      "metaclass": "education-learning",
+      "platform": "leetcode",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 3, "ambassador": 3, "total": 12 },
+      "rationale": "LeetCode is the canonical coding-education platform and one of the highest-difficulty ClawBench tasks — opening a problem, writing a solution, and submitting exercises 5+ steps of dynamic UI plus code editing. The only 12/12 task in the shortlist."
+    },
+    {
+      "task_id": 279,
+      "dir": "279-travel-general-airbnb",
+      "tier": "core",
+      "metaclass": "travel",
+      "platform": "airbnb",
+      "url_pattern_concrete": false,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 3, "ambassador": 3, "total": 12 },
+      "rationale": "Airbnb is the household-name home-rental platform; searching for an apartment by city/date/guests, selecting a listing, and driving to the Confirm-and-Pay page with all fields filled is one of the deepest multi-step flows in ClawBench. Chosen over Momondo (flight search) because Airbnb is a global household name and end-to-end booking is a distinctive ClawBench contribution vs existing benchmark coverage."
+    },
+    {
+      "task_id": 91,
+      "dir": "091-job-search-hr-job-apply-indeed",
+      "tier": "core",
+      "metaclass": "job-search-hr",
+      "platform": "indeed",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "Indeed is the global household-name job board; applying to a posting is canonical job-seeker behavior and the strongest job-search-hr ambassador, with concrete url_pattern."
+    },
+    {
+      "task_id": 783,
+      "dir": "783-beauty-personal-care-beauty-booking-ulta-beauty",
+      "tier": "core",
+      "metaclass": "beauty-personal-care",
+      "platform": "ulta-beauty",
+      "url_pattern_concrete": false,
+      "scores": { "popularity": 2, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 10 },
+      "rationale": "Ulta Beauty is the vertical leader for beauty retail and services; booking a salon appointment exercises location/stylist/time selection — a realistic multi-step everyday task that represents the beauty vertical without being purely e-commerce."
+    },
+    {
+      "task_id": 7,
+      "dir": "007-daily-life-food-instacart",
+      "tier": "core",
+      "metaclass": "daily-life",
+      "platform": "instacart",
+      "url_pattern_concrete": false,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 3, "ambassador": 3, "total": 12 },
+      "rationale": "Instacart is the household-name leader for grocery delivery in North America; purchasing all ingredients for a multi-day meal plan with recipes, auto-selected nearest store, and checkout is a 5+ step weekly real-world flow with meaningful UI depth plus extra_info (meal_plan.json) for cross-recipe ingredient matching. Occupies a distinct food vertical from DoorDash restaurant delivery. Placeholder url_pattern is acceptable under all-agentic eval. Swapped in after the planned Zillow 011 housing pick showed PerimeterX CAPTCHA walls across 4/4 models in the 03-27 recorded batch, borderline-violating the accessibility gate."
+    },
+    {
+      "task_id": 809,
+      "dir": "809-pet-animal-care-pet-adopt-petfinder",
+      "tier": "core",
+      "metaclass": "pet-animal-care",
+      "platform": "petfinder",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 2, "realism": 2, "difficulty": 2, "ambassador": 3, "total": 9 },
+      "rationale": "Petfinder is the vertical leader for pet adoption in the US; submitting an adoption inquiry is the canonical pet-animal-care task and uses a concrete GraphQL url_pattern."
+    },
+    {
+      "task_id": 179,
+      "dir": "179-dev-tech-github-ops-github",
+      "tier": "wildcard",
+      "metaclass": "dev-tech",
+      "platform": "github",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "GitHub is the household-name code-hosting flagship; creating a repository with README + LICENSE + .gitignore is the canonical dev onboarding flow. Wildcard slot: dev-tech has only 2 tasks but GitHub is a first-tier brand. Concrete url_pattern (POST github.com/repositories). No other source benchmark in browser-use/benchmark covers this, so it is a distinctive ClawBench contribution."
+    },
+    {
+      "task_id": 215,
+      "dir": "215-academia-research-paper-tables-overleaf",
+      "tier": "wildcard",
+      "metaclass": "academia-research",
+      "platform": "overleaf",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 2, "realism": 3, "difficulty": 3, "ambassador": 3, "total": 11 },
+      "rationale": "Overleaf is the vertical leader for collaborative LaTeX; creating a project, writing a booktabs table, and compiling exercises multi-step code-editing UI — one of the deepest tasks in the entire benchmark, with concrete url_pattern and extra_info."
+    },
+    {
+      "task_id": 403,
+      "dir": "403-personal-management-account-security-1password-web",
+      "tier": "wildcard",
+      "metaclass": "personal-management",
+      "platform": "1password-web",
+      "url_pattern_concrete": true,
+      "scores": { "popularity": 3, "realism": 3, "difficulty": 2, "ambassador": 3, "total": 11 },
+      "rationale": "1Password is the household-name password manager; adding five login entries exercises repetitive structured data entry — canonical personal-security behavior with concrete url_pattern and extra_info credentials."
+    }
+  ]
+}

clawbench/data/test-cases/lite.schema.json

@@ -0,0 +1,105 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "lite.schema.json",
+  "title": "ClawBench-Lite Manifest",
+  "description": "Schema for ClawBench-Lite — a curated 20-task subset of ClawBench. The manifest lists selected task_ids without duplicating task data; each entry references a test-cases/<dir>/task.json that must exist.",
+  "type": "object",
+  "properties": {
+    "$schema": true,
+    "version": {
+      "type": "string",
+      "description": "Semantic version of the Lite selection (e.g., \"0.1\"). Bump when tasks are added, removed, or re-scored."
+    },
+    "created": {
+      "type": "string",
+      "format": "date",
+      "description": "ISO date (YYYY-MM-DD) when this version of the manifest was locked."
+    },
+    "rubric_max": {
+      "type": "integer",
+      "description": "Maximum possible rubric total. For v0.1 this is 12 (4 axes × 3 points each).",
+      "const": 12
+    },
+    "total_tasks": {
+      "type": "integer",
+      "description": "Number of tasks in the Lite subset. Must equal the length of the tasks array.",
+      "minimum": 1
+    },
+    "notes": {
+      "type": "string",
+      "description": "Free-text rationale and provenance notes for this Lite version."
+    },
+    "skipped_categories": {
+      "type": "array",
+      "description": "Metaclasses intentionally excluded from Lite v0.1. Documented here so the omission reads as a curatorial choice, not an oversight.",
+      "items": {
+        "type": "string"
+      }
+    },
+    "tier_distribution": {
+      "type": "object",
+      "description": "Per-tier slot allocation — flagship/core/wildcard, summing to total_tasks.",
+      "properties": {
+        "flagship": { "type": "integer", "minimum": 0 },
+        "core": { "type": "integer", "minimum": 0 },
+        "wildcard": { "type": "integer", "minimum": 0 }
+      },
+      "required": ["flagship", "core", "wildcard"],
+      "additionalProperties": false
+    },
+    "tasks": {
+      "type": "array",
+      "description": "Selected tasks. Each entry references a task.json under test-cases/ and records its score and rationale.",
+      "items": {
+        "type": "object",
+        "properties": {
+          "task_id": {
+            "type": "integer",
+            "description": "Matches metadata.task_id in the referenced task.json"
+          },
+          "dir": {
+            "type": "string",
+            "description": "Directory name under test-cases/ (e.g., \"179-dev-tech-github-ops-github\")"
+          },
+          "tier": {
+            "type": "string",
+            "enum": ["flagship", "core", "wildcard"],
+            "description": "Which tier this task fills"
+          },
+          "metaclass": {
+            "type": "string",
+            "description": "Matches metadata.metaclass in the referenced task.json"
+          },
+          "platform": {
+            "type": "string",
+            "description": "Short site/brand identifier (e.g., \"github\", \"airbnb\", \"1password\")"
+          },
+          "url_pattern_concrete": {
+            "type": "boolean",
+            "description": "True if eval_schema.url_pattern is a real regex (machine-scorable); false if it is __PLACEHOLDER_WILL_NOT_MATCH__ (agentic-scoring only). Both are allowed in Lite v0.1 because eval is fully agentic."
+          },
+          "scores": {
+            "type": "object",
+            "properties": {
+              "popularity": { "type": "integer", "minimum": 0, "maximum": 3 },
+              "realism": { "type": "integer", "minimum": 0, "maximum": 3 },
+              "difficulty": { "type": "integer", "minimum": 0, "maximum": 3 },
+              "ambassador": { "type": "integer", "minimum": 0, "maximum": 3 },
+              "total": { "type": "integer", "minimum": 0, "maximum": 12 }
+            },
+            "required": ["popularity", "realism", "difficulty", "ambassador", "total"],
+            "additionalProperties": false
+          },
+          "rationale": {
+            "type": "string",
+            "description": "One-sentence justification for inclusion."
+          }
+        },
+        "required": ["task_id", "dir", "tier", "metaclass", "platform", "url_pattern_concrete", "scores", "rationale"],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": ["version", "created", "rubric_max", "total_tasks", "notes", "skipped_categories", "tier_distribution", "tasks"],
+  "additionalProperties": false
+}

clawbench/data/test-cases/task.schema.json

@@ -0,0 +1,132 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "task.schema.json",
+  "title": "ClawBench Test Case",
+  "description": "Schema for ClawBench test case task.json files",
+  "type": "object",
+  "properties": {
+    "$schema": true,
+    "metadata": {
+      "type": "object",
+      "description": "Human-readable metadata for documentation purposes (not read by the agent)",
+      "properties": {
+        "task_id": {
+          "type": "integer",
+          "description": "Unique numeric identifier for the test case"
+        },
+        "metaclass": {
+          "type": "string",
+          "description": "High-level category of the test case"
+        },
+        "class": {
+          "type": "string",
+          "description": "Granular sub-category of the test case"
+        },
+        "description": {
+          "type": "string",
+          "description": "Human-readable description of the test case"
+        },
+        "sites_involved": {
+          "type": "array",
+          "description": "Site domains involved in the test case (e.g., google.com, uber.com, github.com, etc.)",
+          "items": {
+            "type": "string"
+          }
+        },
+        "platform": {
+          "type": "string",
+          "description": "Platform involved in the test case (e.g., google, uber, github, etc.)"
+        },
+        "common_info": {
+          "type": "object",
+          "description": "Common information that is shared among all test cases",
+          "properties": {
+            "email_credentials": {
+              "const": "credentials to use the assigned disposable email account"
+            },
+            "user_info": {
+              "const": "alex_green_personal_info.json; the dummy user's personal information"
+            },
+            "user_resume": {
+              "const": "PDF resume with disposable email account injected"
+            }
+          },
+          "required": ["email_credentials", "user_info", "user_resume"],
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "task_id",
+        "metaclass",
+        "class",
+        "description",
+        "sites_involved",
+        "platform",
+        "common_info"
+      ]
+    },
+    "instruction": {
+      "type": "string",
+      "description": "Task prompt sent to the agent"
+    },
+    "eval_schema": {
+      "type": "object",
+      "description": "Configuration for the request interceptor. The interceptor blocks HTTP requests matching the URL pattern, method, and optional body/params filters, preventing irreversible actions (checkout, submission, etc.) from reaching the server.",
+      "properties": {
+        "url_pattern": {
+          "type": "string",
+          "description": "Regex pattern the request URL must match to be blocked by the interceptor"
+        },
+        "method": {
+          "type": "string",
+          "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"],
+          "description": "HTTP method the request must match to be blocked"
+        },
+        "body": {
+          "type": "object",
+          "description": "Key-value pairs that must match exactly in the request body. Used to disambiguate when URL + method alone isn't specific enough (e.g., same endpoint for login vs send)."
+        },
+        "params": {
+          "type": "object",
+          "description": "Key-value pairs that must match exactly in the URL query parameters. Used to disambiguate when URL + method alone isn't specific enough."
+        }
+      },
+      "required": ["url_pattern", "method"],
+      "additionalProperties": false
+    },
+    "time_limit": {
+      "type": "number",
+      "description": "Maximum time in minutes before the driver stops the container",
+      "minimum": 1
+    },
+    "extra_info": {
+      "type": "array",
+      "description": "Additional context injected into the agent prompt",
+      "items": {
+        "type": "object",
+        "properties": {
+          "path": {
+            "type": "string",
+            "description": "Relative path to a file in the test case directory (optional)"
+          },
+          "description": {
+            "type": "string",
+            "description": "Description text injected into the agent prompt"
+          }
+        },
+        "required": [
+          "path",
+          "description"
+        ],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": [
+    "instruction",
+    "eval_schema",
+    "time_limit"
+  ],
+  "additionalProperties": false
+}