clawbenchmark 0.2.0__py3-none-any.whl

clawbench/__init__.py

@@ -0,0 +1 @@
+"""ClawBench Python package."""

clawbench/_bundled/.env

@@ -0,0 +1,7 @@
+PURELY_MAIL_API_KEY="pm-live-4c2a5524-392b-4117-a722-0aab7a3bf885"
+PURELY_MAIL_DOMAIN="clawbench.cc"
+THINKING_LEVEL="medium"
+
+# Optional: HuggingFace dataset upload
+# HF_TOKEN="hf_..."
+# HF_REPO_ID="your-org/your-dataset-name"

clawbench/_bundled/models/model.schema.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "description": "Schema for a single model entry in models/models.yaml. The YAML key is the model name.",
+  "type": "object",
+  "required": ["base_url", "api_type"],
+  "properties": {
+    "api_key": {
+      "type": "string",
+      "description": "API key for the provider"
+    },
+    "api_keys": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Multiple API keys for round-robin rotation (takes precedence over api_key)"
+    },
+    "thinking_level": {
+      "type": "string",
+      "enum": ["off", "minimal", "low", "medium", "high", "xhigh", "adaptive"],
+      "description": "Reasoning depth for the model"
+    },
+    "temperature": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 2,
+      "description": "Sampling temperature"
+    },
+    "max_tokens": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Maximum output tokens"
+    },
+    "base_url": {
+      "type": "string",
+      "format": "uri",
+      "description": "API base URL (e.g. https://api.openai.com/v1)"
+    },
+    "api_type": {
+      "type": "string",
+      "enum": ["anthropic-messages", "openai-responses", "openai-completions", "google-generative-ai"],
+      "description": "API type for the provider endpoint"
+    }
+  },
+  "additionalProperties": false
+}

clawbench/_bundled/models/models.example.yaml

@@ -0,0 +1,16 @@
+# Copy to models/models.yaml and fill in your API keys.
+# Each top-level key is the model name (passed as MODEL_NAME to the container).
+
+qwen3.5-397b-a17b:
+  api_key: "sk-or-v1-..."
+  base_url: https://openrouter.ai/api/v1
+  api_type: openai-completions
+  thinking_level: medium             # optional
+
+# For multiple API keys (round-robin), use api_keys instead of api_key:
+# some-model:
+#   api_keys:
+#     - "key1"
+#     - "key2"
+#   base_url: https://api.openai.com/v1
+#   api_type: openai-completions

clawbench/_bundled/test-cases/task.schema.json

@@ -0,0 +1,132 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "task.schema.json",
+  "title": "ClawBench Test Case",
+  "description": "Schema for ClawBench test case task.json files",
+  "type": "object",
+  "properties": {
+    "$schema": true,
+    "metadata": {
+      "type": "object",
+      "description": "Human-readable metadata for documentation purposes (not read by the agent)",
+      "properties": {
+        "task_id": {
+          "type": "integer",
+          "description": "Unique numeric identifier for the test case"
+        },
+        "metaclass": {
+          "type": "string",
+          "description": "High-level category of the test case"
+        },
+        "class": {
+          "type": "string",
+          "description": "Granular sub-category of the test case"
+        },
+        "description": {
+          "type": "string",
+          "description": "Human-readable description of the test case"
+        },
+        "sites_involved": {
+          "type": "array",
+          "description": "Site domains involved in the test case (e.g., google.com, uber.com, github.com, etc.)",
+          "items": {
+            "type": "string"
+          }
+        },
+        "platform": {
+          "type": "string",
+          "description": "Platform involved in the test case (e.g., google, uber, github, etc.)"
+        },
+        "common_info": {
+          "type": "object",
+          "description": "Common information that is shared among all test cases",
+          "properties": {
+            "email_credentials": {
+              "const": "credentials to use the assigned disposable email account"
+            },
+            "user_info": {
+              "const": "alex_green_personal_info.json; the dummy user's personal information"
+            },
+            "user_resume": {
+              "const": "PDF resume with disposable email account injected"
+            }
+          },
+          "required": ["email_credentials", "user_info", "user_resume"],
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "task_id",
+        "metaclass",
+        "class",
+        "description",
+        "sites_involved",
+        "platform",
+        "common_info"
+      ]
+    },
+    "instruction": {
+      "type": "string",
+      "description": "Task prompt sent to the agent"
+    },
+    "eval_schema": {
+      "type": "object",
+      "description": "Configuration for the request interceptor. The interceptor blocks HTTP requests matching the URL pattern, method, and optional body/params filters, preventing irreversible actions (checkout, submission, etc.) from reaching the server.",
+      "properties": {
+        "url_pattern": {
+          "type": "string",
+          "description": "Regex pattern the request URL must match to be blocked by the interceptor"
+        },
+        "method": {
+          "type": "string",
+          "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"],
+          "description": "HTTP method the request must match to be blocked"
+        },
+        "body": {
+          "type": "object",
+          "description": "Key-value pairs that must match exactly in the request body. Used to disambiguate when URL + method alone isn't specific enough (e.g., same endpoint for login vs send)."
+        },
+        "params": {
+          "type": "object",
+          "description": "Key-value pairs that must match exactly in the URL query parameters. Used to disambiguate when URL + method alone isn't specific enough."
+        }
+      },
+      "required": ["url_pattern", "method"],
+      "additionalProperties": false
+    },
+    "time_limit": {
+      "type": "number",
+      "description": "Maximum time in minutes before the driver stops the container",
+      "minimum": 1
+    },
+    "extra_info": {
+      "type": "array",
+      "description": "Additional context injected into the agent prompt",
+      "items": {
+        "type": "object",
+        "properties": {
+          "path": {
+            "type": "string",
+            "description": "Relative path to a file in the test case directory (optional)"
+          },
+          "description": {
+            "type": "string",
+            "description": "Description text injected into the agent prompt"
+          }
+        },
+        "required": [
+          "path",
+          "description"
+        ],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": [
+    "instruction",
+    "eval_schema",
+    "time_limit"
+  ],
+  "additionalProperties": false
+}

clawbench/_bundled/test-cases/v1/001-daily-life-food-uber-eats/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 1,
+    "metaclass": "daily-life",
+    "class": "food",
+    "description": "On Uber Eats, order delivery: one Pad Thai, deliver to home address, note \"no peanuts\"",
+    "sites_involved": [
+      "ubereats.com"
+    ],
+    "platform": "uber-eats",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Uber Eats, order delivery: one Pad Thai, deliver to home address, note \"no peanuts\"",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/_bundled/test-cases/v1/002-daily-life-food-doordash/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 2,
+    "metaclass": "daily-life",
+    "class": "food",
+    "description": "On DoorDash, order delivery: one Big Mac + two 10 pc McNuggets, select fastest delivery",
+    "sites_involved": [
+      "doordash.com"
+    ],
+    "platform": "doordash",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On DoorDash, order delivery: one Big Mac + two 10 pc McNuggets, select fastest delivery",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/_bundled/test-cases/v1/004-daily-life-food-instacart/extra_info/grocery_list.json

@@ -0,0 +1,36 @@
+{
+  "items": [
+    {
+      "name": "Whole milk (1 gallon)",
+      "quantity": 1,
+      "category": "dairy"
+    },
+    {
+      "name": "Large eggs (dozen)",
+      "quantity": 1,
+      "category": "dairy"
+    },
+    {
+      "name": "Whole wheat bread",
+      "quantity": 1,
+      "category": "bakery"
+    },
+    {
+      "name": "Chicken breast (boneless, skinless)",
+      "quantity": "2 lbs",
+      "category": "meat"
+    },
+    {
+      "name": "Broccoli (fresh)",
+      "quantity": "1 bunch",
+      "category": "produce"
+    },
+    {
+      "name": "Bananas",
+      "quantity": "1 bunch (6-7)",
+      "category": "produce"
+    }
+  ],
+  "delivery_preference": "cheapest available window",
+  "store_preference": "nearest"
+}

clawbench/_bundled/test-cases/v1/004-daily-life-food-instacart/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 4,
+    "metaclass": "daily-life",
+    "class": "food",
+    "description": "Place an Instacart order for a week's groceries: milk, eggs, whole wheat bread, chicken breast, broccoli, bananas; select the cheapest delivery window",
+    "sites_involved": [
+      "instacart.com"
+    ],
+    "platform": "instacart",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Place an Instacart order for a week's groceries: milk, eggs, whole wheat bread, chicken breast, broccoli, bananas; select the cheapest delivery window",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/grocery_list.json",
+      "description": "Weekly grocery list with items, quantities, and categories"
+    }
+  ]
+}

clawbench/_bundled/test-cases/v1/006-daily-life-food-uber-eats/task.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 6,
+    "metaclass": "daily-life",
+    "class": "food",
+    "description": "On Uber Eats, order a vegan delivery (labeled vegan), deliver to office address",
+    "sites_involved": [
+      "ubereats.com"
+    ],
+    "platform": "uber-eats",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Uber Eats, order a vegan delivery (labeled vegan), deliver to office address",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30
+}

clawbench/_bundled/test-cases/v1/007-daily-life-food-instacart/extra_info/meal_plan.json

@@ -0,0 +1,21 @@
+{
+  "week_plan": [
+    {
+      "day": "Monday",
+      "breakfast": "Greek yogurt with granola",
+      "lunch": "Grilled chicken salad",
+      "dinner": "Salmon with steamed broccoli and brown rice"
+    },
+    {
+      "day": "Tuesday",
+      "breakfast": "Overnight oats with bananas",
+      "lunch": "Turkey wrap with veggies",
+      "dinner": "Pasta primavera with garlic bread"
+    }
+  ],
+  "dietary_restrictions": [
+    "no shellfish (allergy)",
+    "low sodium preferred"
+  ],
+  "servings_per_meal": 1
+}

clawbench/_bundled/test-cases/v1/007-daily-life-food-instacart/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 7,
+    "metaclass": "daily-life",
+    "class": "food",
+    "description": "On Instacart, purchase all ingredients for 2-day meal plan recipes, with the nearest supermarket auto-selected",
+    "sites_involved": [
+      "instacart.com"
+    ],
+    "platform": "instacart",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "On Instacart, purchase all ingredients for 2-day meal plan recipes, with the nearest supermarket auto-selected",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/meal_plan.json",
+      "description": "2-day meal plan with recipes and dietary restrictions"
+    }
+  ]
+}

clawbench/_bundled/test-cases/v1/011-daily-life-housing-zillow/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 11,
+    "metaclass": "daily-life",
+    "class": "housing",
+    "description": "Search Zillow for a one-bedroom apartment in Toronto downtown under $3500/month, select one and submit a rental application",
+    "sites_involved": [
+      "zillow.com"
+    ],
+    "platform": "zillow",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Search Zillow for a one-bedroom apartment in Toronto downtown under $3500/month, select one and submit a rental application",
+  "eval_schema": {
+    "url_pattern": "zillow\\.com/rentals/api/rcf/v1/rcf",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/_bundled/test-cases/v1/015-daily-life-housing-craigslist/extra_info/listing_details.json

@@ -0,0 +1,26 @@
+{
+  "listing_title": "Bright 1BR in Toronto Annex, Furnished, July-August Sublet",
+  "description": "Spacious furnished one-bedroom apartment in the heart of Toronto's Annex neighborhood. Walking distance to U of T campus, TTC subway, and Bloor Street shops and restaurants. The unit features hardwood floors, in-suite laundry, a fully equipped kitchen, and a sunny south-facing balcony. Perfect for visiting scholars or summer interns.",
+  "price": 2400,
+  "currency": "CAD",
+  "period": "monthly",
+  "available_from": "2026-07-01",
+  "available_to": "2026-08-31",
+  "furnished": true,
+  "bedrooms": 1,
+  "bathrooms": 1,
+  "amenities": [
+    "In-suite laundry",
+    "Dishwasher",
+    "Balcony",
+    "Air conditioning",
+    "WiFi included"
+  ],
+  "photos": [
+    "living_room.jpg",
+    "bedroom.jpg",
+    "kitchen.jpg",
+    "balcony.jpg"
+  ],
+  "contact_email": "alex.green.uoft@clawbench.cc"
+}

clawbench/_bundled/test-cases/v1/015-daily-life-housing-craigslist/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 15,
+    "metaclass": "daily-life",
+    "class": "housing",
+    "description": "Post a sublet listing on Craigslist: one-bedroom near Toronto Annex, $2400/month, July-August, furnished",
+    "sites_involved": [
+      "craigslist.org"
+    ],
+    "platform": "craigslist",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Post a sublet listing on Craigslist: one-bedroom near Toronto Annex, $2400/month, July-August, furnished",
+  "eval_schema": {
+    "url_pattern": "post\\.craigslist\\.org/k/",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/listing_details.json",
+      "description": "Sublet listing details including description, amenities, and photos"
+    }
+  ]
+}

clawbench/_bundled/test-cases/v1/035-daily-life-health-medical-betterhelp/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 35,
+    "metaclass": "daily-life",
+    "class": "health-medical",
+    "description": "Sign up on BetterHelp, book a counseling session, and complete the initial questionnaire",
+    "sites_involved": [
+      "betterhelp.com"
+    ],
+    "platform": "betterhelp",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Sign up on BetterHelp, book a counseling session, and complete the initial questionnaire",
+  "eval_schema": {
+    "url_pattern": "__PLACEHOLDER_WILL_NOT_MATCH__",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/_bundled/test-cases/v1/041-daily-life-pets-rover/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 41,
+    "metaclass": "daily-life",
+    "class": "pets",
+    "description": "Book daytime dog walking service on Rover for Monday through Friday next week, large dog (Golden Retriever)",
+    "sites_involved": [
+      "rover.com"
+    ],
+    "platform": "rover",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Book daytime dog walking service on Rover for Monday through Friday next week, large dog (Golden Retriever)",
+  "eval_schema": {
+    "url_pattern": "rover\\.com/api/v7/people/.+/services/dog-walking/contactability",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/_bundled/test-cases/v1/043-daily-life-pets-rover/extra_info/pet_info.json

@@ -0,0 +1,12 @@
+{
+  "name": "Maple",
+  "species": "Dog",
+  "breed": "Golden Retriever",
+  "age_years": 4,
+  "weight_lbs": 35,
+  "sex": "Female (spayed)",
+  "date_of_birth": "2022-03-15",
+  "vaccinations_up_to_date": true,
+  "microchip_id": "985121012345678",
+  "dietary_notes": "Grain-free kibble, sensitive stomach"
+}

clawbench/_bundled/test-cases/v1/043-daily-life-pets-rover/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 43,
+    "metaclass": "daily-life",
+    "class": "pets",
+    "description": "Book 5-day pet boarding next month on Rover (medium-sized dog), sitter rated 4.8+",
+    "sites_involved": [
+      "rover.com"
+    ],
+    "platform": "rover",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Book 5-day pet boarding next month on Rover (medium-sized dog), sitter rated 4.8+",
+  "eval_schema": {
+    "url_pattern": "rover\\.com/api/v\\d+/people/[^/]+/services/overnight-boarding/contactability",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/pet_info.json",
+      "description": "Pet information for Alex Green's dog Maple (Golden Retriever, 4 years, 35 lbs)"
+    }
+  ]
+}

clawbench/_bundled/test-cases/v1/045-daily-life-personal-care-booksy/task.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 45,
+    "metaclass": "daily-life",
+    "class": "personal-care",
+    "description": "Book a women's haircut on Booksy for the upcoming Saturday afternoon",
+    "sites_involved": [
+      "booksy.com"
+    ],
+    "platform": "booksy",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Book a women's haircut on Booksy for the upcoming Saturday afternoon",
+  "eval_schema": {
+    "url_pattern": "booksy\\.com/core/v\\d+/customer_api/me/appointments/business/\\d+/?$",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": []
+}

clawbench/_bundled/test-cases/v1/047-daily-life-personal-care-taskrabbit/extra_info/address_info.json

@@ -0,0 +1,7 @@
+{
+  "note": "Use home address from alex_green_personal_info.json",
+  "move out address": "Unit 1208, 664 Spadina Ave, Toronto, ON M5S 2H7, Canada",
+  "move in address": "450 Front St W, Toronto, ON M5V 0V7",
+  "Task size": "Large",
+  "Need Vehicle": "Yes, a Car"
+}

clawbench/_bundled/test-cases/v1/047-daily-life-personal-care-taskrabbit/task.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "../../task.schema.json",
+  "metadata": {
+    "task_id": 47,
+    "metaclass": "daily-life",
+    "class": "personal-care",
+    "description": "Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment",
+    "sites_involved": [
+      "taskrabbit.com"
+    ],
+    "platform": "taskrabbit",
+    "common_info": {
+      "email_credentials": "credentials to use the assigned disposable email account",
+      "user_info": "alex_green_personal_info.json; the dummy user's personal information",
+      "user_resume": "PDF resume with disposable email account injected"
+    }
+  },
+  "instruction": "Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment",
+  "eval_schema": {
+    "url_pattern": "taskrabbit\\.(com|ca)/(api/v\\d+/jobs|book/\\d+/confirm)",
+    "method": "POST"
+  },
+  "time_limit": 30,
+  "extra_info": [
+    {
+      "path": "extra_info/address_info.json",
+      "description": "Address information (references alex_green_personal_info.json)"
+    }
+  ]
+}

clawbench/_bundled/test-cases/v1/086-job-search-hr-cv-autofill-greenhouse-meta/extra_info/job_links.json

@@ -0,0 +1,5 @@
+{
+  "job_url": "https://boards.greenhouse.io/example/jobs/1234567",
+  "job_title": "Senior Software Engineer",
+  "company": "Example Corp"
+}