sys1bench 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. sys1bench/__init__.py +3 -0
  2. sys1bench/adapters/__init__.py +19 -0
  3. sys1bench/adapters/base.py +159 -0
  4. sys1bench/adapters/embed_knn.py +61 -0
  5. sys1bench/adapters/encoder_finetuned.py +114 -0
  6. sys1bench/adapters/generic_http.py +161 -0
  7. sys1bench/adapters/hybrid_router.py +49 -0
  8. sys1bench/adapters/jev_openrouter.py +172 -0
  9. sys1bench/adapters/jev_typesafe.py +165 -0
  10. sys1bench/adapters/laya_local.py +192 -0
  11. sys1bench/adapters/llm_constrained.py +104 -0
  12. sys1bench/adapters/majority_prior.py +53 -0
  13. sys1bench/adapters/mock.py +77 -0
  14. sys1bench/adapters/nli_zeroshot.py +50 -0
  15. sys1bench/adapters/regex_keyword.py +70 -0
  16. sys1bench/analysis/__init__.py +4 -0
  17. sys1bench/analysis/arms.py +46 -0
  18. sys1bench/analysis/decision_value.py +43 -0
  19. sys1bench/analysis/decomposition.py +47 -0
  20. sys1bench/analysis/meta_eval.py +67 -0
  21. sys1bench/analysis/stats.py +76 -0
  22. sys1bench/cli.py +434 -0
  23. sys1bench/data/__init__.py +37 -0
  24. sys1bench/data/canary/canary.jsonl +200 -0
  25. sys1bench/data/framings/guardrail_intent.yaml +21 -0
  26. sys1bench/data/framings/log_triage.yaml +20 -0
  27. sys1bench/data/framings/phishing_email.yaml +25 -0
  28. sys1bench/data/framings/policy_compliance.yaml +18 -0
  29. sys1bench/data/framings/support_tickets.yaml +41 -0
  30. sys1bench/framing/__init__.py +10 -0
  31. sys1bench/framing/expand.py +214 -0
  32. sys1bench/framing/perturb.py +67 -0
  33. sys1bench/generators/__init__.py +10 -0
  34. sys1bench/generators/base.py +105 -0
  35. sys1bench/generators/guardrail_intent.py +95 -0
  36. sys1bench/generators/log_triage.py +100 -0
  37. sys1bench/generators/multilingual_tickets.py +84 -0
  38. sys1bench/generators/phishing_email.py +127 -0
  39. sys1bench/generators/policy_compliance.py +90 -0
  40. sys1bench/generators/rag_relevance.py +122 -0
  41. sys1bench/generators/support_tickets.py +212 -0
  42. sys1bench/metrics/__init__.py +9 -0
  43. sys1bench/metrics/calibration.py +164 -0
  44. sys1bench/metrics/consistency.py +71 -0
  45. sys1bench/metrics/decision_value.py +43 -0
  46. sys1bench/metrics/efficiency.py +35 -0
  47. sys1bench/metrics/ordinal.py +80 -0
  48. sys1bench/metrics/robustness.py +46 -0
  49. sys1bench/metrics/selective.py +67 -0
  50. sys1bench/report/__init__.py +1 -0
  51. sys1bench/report/dashboard.py +177 -0
  52. sys1bench/report/latex.py +40 -0
  53. sys1bench/report/plots.py +143 -0
  54. sys1bench/report/results_doc.py +237 -0
  55. sys1bench/report/scorecard.py +170 -0
  56. sys1bench/runners/__init__.py +2 -0
  57. sys1bench/runners/benchmark_runner.py +149 -0
  58. sys1bench/runners/cache.py +53 -0
  59. sys1bench/runners/canary.py +43 -0
  60. sys1bench/runners/hybrid_sweep.py +37 -0
  61. sys1bench/runners/noul_consistency.py +121 -0
  62. sys1bench/runners/ordinal_probes.py +90 -0
  63. sys1bench/runners/robustness.py +112 -0
  64. sys1bench/runners/suite.py +183 -0
  65. sys1bench/runners/sweeps.py +190 -0
  66. sys1bench/schemas.py +274 -0
  67. sys1bench-0.3.0.dist-info/METADATA +195 -0
  68. sys1bench-0.3.0.dist-info/RECORD +72 -0
  69. sys1bench-0.3.0.dist-info/WHEEL +5 -0
  70. sys1bench-0.3.0.dist-info/entry_points.txt +2 -0
  71. sys1bench-0.3.0.dist-info/licenses/LICENSE +202 -0
  72. sys1bench-0.3.0.dist-info/top_level.txt +1 -0
sys1bench/data/canary/canary.jsonl ADDED
@@ -0,0 +1,200 @@
1
+ {"task_id":"tickets_9001_00000","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 12 seats. No rush, but I'd like to get this sorted.\"\nContext: customer tier enterprise; account age 39 months; open tickets 1.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-b28234687a0a","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"enterprise","angry":false}}
2
+ {"task_id":"tickets_9001_00001","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #41588? It's been 10 days. Appreciate you looking into it.\"\nContext: customer tier standard; account age 57 months; open tickets 0.","state_tokens":36,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-ba598f523b31","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"standard","angry":false}}
3
+ {"task_id":"tickets_9001_00002","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Export to CSV produces an empty file for reports over 14 days. Let me know if you need anything else from me.\"\nContext: customer tier free; account age 10 months; open tickets 0.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-0f602247cc8c","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"free","angry":false}}
4
+ {"task_id":"tickets_9001_00003","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Package #16864 arrived crushed and the contents are broken. I need this for an event on Sept 14. Let me know if you need anything else from me.\"\nContext: customer tier standard; account age 12 months; open tickets 2.","state_tokens":52,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-87b1f99e7642","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":2,"tier":"standard","angry":false}}
5
+ {"task_id":"tickets_9001_00004","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"My autopay failed on the 3rd and now there's a late fee of $45. I had funds available. No rush, but I'd like to get this sorted.\"\nContext: customer tier free; account age 53 months; open tickets 2.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-32a1fcc95534","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"free","angry":false}}
6
+ {"task_id":"tickets_9001_00005","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #67029? It's been 21 days. No rush, but I'd like to get this sorted.\"\nContext: customer tier gold; account age 11 months; open tickets 2.","state_tokens":41,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-f90c361f71ec","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"gold","angry":false}}
7
+ {"task_id":"tickets_9001_00006","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"If I cancel, do I keep access until the end of the billing period? Let me know if you need anything else from me.\"\nContext: customer tier gold; account age 43 months; open tickets 1.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-da0b271a02e6","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":0,"tier":"gold","angry":false}}
8
+ {"task_id":"tickets_9001_00007","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #81715? I need it for expenses. Let me know if you need anything else from me.\"\nContext: customer tier free; account age 17 months; open tickets 1.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-aa3574cbb7db","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"free","angry":false}}
9
+ {"task_id":"tickets_9001_00008","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Notifications stopped arriving on Monday. Settings look correct. Device: Galaxy S24. Appreciate you looking into it.\"\nContext: customer tier free; account age 54 months; open tickets 0.","state_tokens":36,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-caa9dd7fb1dd","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"free","angry":false}}
10
+ {"task_id":"tickets_9001_00009","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #51862 says delivered on Sept 14 but nothing arrived. Tracking shows a photo of a door that isn't mine. Thanks in advance for your help.\"\nContext: customer tier standard; account age 5 months; open tickets 3.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-c8c7b7ca8a56","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"standard","angry":false}}
11
+ {"task_id":"tickets_9001_00010","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #47505 says delivered on Monday but nothing arrived. Tracking shows a photo of a door that isn't mine. Thanks in advance for your help.\"\nContext: customer tier free; account age 45 months; open tickets 2.","state_tokens":48,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-26cb9afb9e80","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"free","angry":false}}
12
+ {"task_id":"tickets_9001_00011","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"My autopay failed on the 3rd and now there's a late fee of $1200. I had funds available. Appreciate you looking into it.\"\nContext: customer tier gold; account age 23 months; open tickets 3.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-6856f40375d0","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"gold","angry":false}}
13
+ {"task_id":"tickets_9001_00012","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Package #58052 arrived crushed and the contents are broken. I need this for an event on Sept 14. Unbelievable. Every single time. I want this escalated NOW.\"\nContext: customer tier enterprise; account age 21 months; open tickets 1.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":4,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-adb1f1eb0767","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":2,"tier":"enterprise","angry":true}}
14
+ {"task_id":"tickets_9001_00013","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"My autopay failed on the 3rd and now there's a late fee of $45. I had funds available. Unbelievable. Every single time. I want this escalated NOW.\"\nContext: customer tier standard; account age 4 months; open tickets 1.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-321608680728","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"standard","angry":true}}
15
+ {"task_id":"tickets_9001_00014","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"The app crashes on launch since the update on yesterday. I can't access anything. Error code E620. Let me know if you need anything else from me.\"\nContext: customer tier free; account age 27 months; open tickets 1.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-0eaa6f3e57a6","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":2,"tier":"free","angry":false}}
16
+ {"task_id":"tickets_9001_00015","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #92370? I need it for expenses. No rush, but I'd like to get this sorted.\"\nContext: customer tier standard; account age 39 months; open tickets 1.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-cb01cab1e563","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"standard","angry":false}}
17
+ {"task_id":"tickets_9001_00016","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"The app crashes on launch since the update on the 3rd. I can't access anything. Error code E179. No rush, but I'd like to get this sorted.\"\nContext: customer tier gold; account age 20 months; open tickets 3.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-e372b924d84f","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":2,"tier":"gold","angry":false}}
18
+ {"task_id":"tickets_9001_00017","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Appreciate you looking into it.\"\nContext: customer tier gold; account age 47 months; open tickets 0.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-47cb5f8550dc","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"gold","angry":false}}
19
+ {"task_id":"tickets_9001_00018","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"The app crashes on launch since the update on yesterday. I can't access anything. Error code E228. Thanks in advance for your help.\"\nContext: customer tier free; account age 2 months; open tickets 0.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-380bd484f3c8","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":2,"tier":"free","angry":false}}
20
+ {"task_id":"tickets_9001_00019","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #17709? It's been 11 days. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 20 months; open tickets 1.","state_tokens":37,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-3a8fd442bc75","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"enterprise","angry":false}}
21
+ {"task_id":"tickets_9001_00020","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 48 months; open tickets 2.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-2475f8e9623d","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"enterprise","angry":false}}
22
+ {"task_id":"tickets_9001_00021","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Please cancel my subscription before the renewal on yesterday. Reference #29842. Let me know if you need anything else from me.\"\nContext: customer tier gold; account age 2 months; open tickets 2.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-4612dc68dcc1","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":1,"tier":"gold","angry":false}}
23
+ {"task_id":"tickets_9001_00022","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Please cancel my subscription before the renewal on Sept 14. Reference #62210. This is completely unacceptable and I am furious.\"\nContext: customer tier free; account age 9 months; open tickets 1.","state_tokens":41,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-7539a6de78fe","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":1,"tier":"free","angry":true}}
24
+ {"task_id":"tickets_9001_00023","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 14 seats. Unbelievable. Every single time. I want this escalated NOW.\"\nContext: customer tier gold; account age 42 months; open tickets 3.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-7d92097c2130","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"gold","angry":true}}
25
+ {"task_id":"tickets_9001_00024","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"If I cancel, do I keep access until the end of the billing period? I've had it with this company. Absolutely appalling service.\"\nContext: customer tier enterprise; account age 21 months; open tickets 0.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-96c8d1576992","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":0,"tier":"enterprise","angry":true}}
26
+ {"task_id":"tickets_9001_00025","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Unbelievable. Every single time. I want this escalated NOW.\"\nContext: customer tier standard; account age 57 months; open tickets 0.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-924d0173d72a","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"standard","angry":true}}
27
+ {"task_id":"tickets_9001_00026","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"The app crashes on launch since the update on the 3rd. I can't access anything. Error code E537. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier free; account age 7 months; open tickets 2.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-4f1de74b2aff","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":2,"tier":"free","angry":true}}
28
+ {"task_id":"tickets_9001_00027","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #16096? I need it for expenses. Appreciate you looking into it.\"\nContext: customer tier gold; account age 59 months; open tickets 3.","state_tokens":40,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-368853743de6","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"gold","angry":false}}
29
+ {"task_id":"tickets_9001_00028","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"My autopay failed on Monday and now there's a late fee of $120. I had funds available. No rush, but I'd like to get this sorted.\"\nContext: customer tier gold; account age 33 months; open tickets 3.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-bb28666eda4f","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"gold","angry":false}}
30
+ {"task_id":"tickets_9001_00029","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Notifications stopped arriving on the 3rd. Settings look correct. Device: Galaxy S24. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier gold; account age 38 months; open tickets 2.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-31502da8e722","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"gold","angry":true}}
31
+ {"task_id":"tickets_9001_00030","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"The app crashes on launch since the update on yesterday. I can't access anything. Error code E555. Thanks in advance for your help.\"\nContext: customer tier standard; account age 34 months; open tickets 3.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-ddfe7b5a7c87","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":2,"tier":"standard","angry":false}}
32
+ {"task_id":"tickets_9001_00031","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I'm locked out. Two-factor codes go to a phone number I no longer have. Order #54189 is pending. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier gold; account age 55 months; open tickets 1.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":4,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-7ecdd10042a2","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":2,"tier":"gold","angry":true}}
33
+ {"task_id":"tickets_9001_00032","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Export to CSV produces an empty file for reports over 27 days. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier gold; account age 27 months; open tickets 2.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-b6f2abbfb265","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"gold","angry":true}}
34
+ {"task_id":"tickets_9001_00033","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. No rush, but I'd like to get this sorted.\"\nContext: customer tier enterprise; account age 4 months; open tickets 0.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-74fba001a560","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"enterprise","angry":false}}
35
+ {"task_id":"tickets_9001_00034","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Package #66808 arrived crushed and the contents are broken. I need this for an event on Monday. Unbelievable. Every single time. I want this escalated NOW.\"\nContext: customer tier gold; account age 37 months; open tickets 0.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":4,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-064bd5ef99d2","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":2,"tier":"gold","angry":true}}
36
+ {"task_id":"tickets_9001_00035","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"If I cancel, do I keep access until the end of the billing period? Appreciate you looking into it.\"\nContext: customer tier standard; account age 53 months; open tickets 1.","state_tokens":40,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-9f098ba28336","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":0,"tier":"standard","angry":false}}
37
+ {"task_id":"tickets_9001_00036","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #41389 says delivered on yesterday but nothing arrived. Tracking shows a photo of a door that isn't mine. Thanks in advance for your help.\"\nContext: customer tier gold; account age 27 months; open tickets 0.","state_tokens":48,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-7fe12c33db4a","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"gold","angry":false}}
38
+ {"task_id":"tickets_9001_00037","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #36455? I need it for expenses. Let me know if you need anything else from me.\"\nContext: customer tier gold; account age 15 months; open tickets 3.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-d24de2996613","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"gold","angry":false}}
39
+ {"task_id":"tickets_9001_00038","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 11 seats. Appreciate you looking into it.\"\nContext: customer tier gold; account age 12 months; open tickets 2.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-08f188ab062b","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"gold","angry":false}}
40
+ {"task_id":"tickets_9001_00039","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Please cancel my subscription before the renewal on yesterday. Reference #49474. Honestly, this is ridiculous. Fix it today or I'm done.\"\nContext: customer tier standard; account age 42 months; open tickets 0.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-4524082bd412","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":1,"tier":"standard","angry":true}}
41
+ {"task_id":"tickets_9001_00040","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"The app crashes on launch since the update on yesterday. I can't access anything. Error code E402. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier standard; account age 12 months; open tickets 2.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-6f785e92f632","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":2,"tier":"standard","angry":true}}
42
+ {"task_id":"tickets_9001_00041","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #87698 says delivered on Sept 14 but nothing arrived. Tracking shows a photo of a door that isn't mine. Thanks in advance for your help.\"\nContext: customer tier standard; account age 55 months; open tickets 2.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-7eb97203954f","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"standard","angry":false}}
43
+ {"task_id":"tickets_9001_00042","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Notifications stopped arriving on Monday. Settings look correct. Device: iPhone 15. Thanks in advance for your help.\"\nContext: customer tier free; account age 30 months; open tickets 1.","state_tokens":37,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-9c116d342c4c","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"free","angry":false}}
44
+ {"task_id":"tickets_9001_00043","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"My autopay failed on the 3rd and now there's a late fee of $45. I had funds available. Let me know if you need anything else from me.\"\nContext: customer tier standard; account age 16 months; open tickets 2.","state_tokens":52,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-befcaf91d251","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"standard","angry":false}}
45
+ {"task_id":"tickets_9001_00044","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #34646? It's been 18 days. Thanks in advance for your help.\"\nContext: customer tier standard; account age 26 months; open tickets 3.","state_tokens":37,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-c4ef7cf2f6be","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"standard","angry":false}}
46
+ {"task_id":"tickets_9001_00045","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Export to CSV produces an empty file for reports over 26 days. This is completely unacceptable and I am furious.\"\nContext: customer tier standard; account age 50 months; open tickets 3.","state_tokens":41,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-9b97efa13510","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"standard","angry":true}}
47
+ {"task_id":"tickets_9001_00046","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #85322? I need it for expenses. Honestly, this is ridiculous. Fix it today or I'm done.\"\nContext: customer tier free; account age 41 months; open tickets 2.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-efd102f44b7c","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"free","angry":true}}
48
+ {"task_id":"tickets_9001_00047","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I was charged $120 twice on the 3rd for order #91280. Only one charge should exist. No rush, but I'd like to get this sorted.\"\nContext: customer tier gold; account age 29 months; open tickets 2.","state_tokens":48,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-8c74d57f68db","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"gold","angry":false}}
49
+ {"task_id":"tickets_9001_00048","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 15 seats. Honestly, this is ridiculous. Fix it today or I'm done.\"\nContext: customer tier standard; account age 52 months; open tickets 2.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-677968f99132","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"standard","angry":true}}
50
+ {"task_id":"tickets_9001_00049","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Appreciate you looking into it.\"\nContext: customer tier gold; account age 47 months; open tickets 1.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-17e5092d1c36","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"gold","angry":false}}
51
+ {"task_id":"tickets_9001_00050","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I was charged $45 twice on Sept 14 for order #54540. Only one charge should exist. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 17 months; open tickets 0.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-e08ef0aba9a2","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"enterprise","angry":false}}
52
+ {"task_id":"tickets_9001_00051","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Honestly, this is ridiculous. Fix it today or I'm done.\"\nContext: customer tier enterprise; account age 59 months; open tickets 0.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-4b85f1a31df7","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"enterprise","angry":true}}
53
+ {"task_id":"tickets_9001_00052","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I'm locked out. Two-factor codes go to a phone number I no longer have. Order #26078 is pending. No rush, but I'd like to get this sorted.\"\nContext: customer tier enterprise; account age 42 months; open tickets 1.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-f60a9d05f3f4","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":2,"tier":"enterprise","angry":false}}
54
+ {"task_id":"tickets_9001_00053","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Package #18354 arrived crushed and the contents are broken. I need this for an event on the 3rd. This is completely unacceptable and I am furious.\"\nContext: customer tier gold; account age 51 months; open tickets 2.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":4,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-4bddf7896736","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":2,"tier":"gold","angry":true}}
55
+ {"task_id":"tickets_9001_00054","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #85004? It's been 10 days. Appreciate you looking into it.\"\nContext: customer tier gold; account age 50 months; open tickets 0.","state_tokens":36,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-8b52063d3d11","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"gold","angry":false}}
56
+ {"task_id":"tickets_9001_00055","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Export to CSV produces an empty file for reports over 2 days. Let me know if you need anything else from me.\"\nContext: customer tier standard; account age 38 months; open tickets 0.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-7fcc7402c3b8","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"standard","angry":false}}
57
+ {"task_id":"tickets_9001_00056","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #42909 says delivered on the 3rd but nothing arrived. Tracking shows a photo of a door that isn't mine. No rush, but I'd like to get this sorted.\"\nContext: customer tier gold; account age 20 months; open tickets 0.","state_tokens":53,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-157519a2efd6","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"gold","angry":false}}
58
+ {"task_id":"tickets_9001_00057","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Notifications stopped arriving on Sept 14. Settings look correct. Device: Pixel 8. No rush, but I'd like to get this sorted.\"\nContext: customer tier enterprise; account age 18 months; open tickets 3.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-33fff9e21f97","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"enterprise","angry":false}}
59
+ {"task_id":"tickets_9001_00058","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #93078 says delivered on yesterday but nothing arrived. Tracking shows a photo of a door that isn't mine. No rush, but I'd like to get this sorted.\"\nContext: customer tier free; account age 37 months; open tickets 1.","state_tokens":52,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-f65f50f4f539","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"free","angry":false}}
60
+ {"task_id":"tickets_9001_00059","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Please cancel my subscription before the renewal on the 3rd. Reference #57818. No rush, but I'd like to get this sorted.\"\nContext: customer tier gold; account age 58 months; open tickets 2.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-5c835c5043f7","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":1,"tier":"gold","angry":false}}
61
+ {"task_id":"tickets_9001_00060","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #41322? I need it for expenses. Appreciate you looking into it.\"\nContext: customer tier standard; account age 37 months; open tickets 2.","state_tokens":40,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-b92d55c1c151","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"standard","angry":false}}
62
+ {"task_id":"tickets_9001_00061","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #74509? It's been 7 days. Thanks in advance for your help.\"\nContext: customer tier gold; account age 46 months; open tickets 2.","state_tokens":37,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-d33f397b2a16","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"gold","angry":false}}
63
+ {"task_id":"tickets_9001_00062","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 26 seats. This is completely unacceptable and I am furious.\"\nContext: customer tier gold; account age 42 months; open tickets 1.","state_tokens":48,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-19efc680ca76","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"gold","angry":true}}
64
+ {"task_id":"tickets_9001_00063","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 2 seats. No rush, but I'd like to get this sorted.\"\nContext: customer tier gold; account age 24 months; open tickets 1.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-2361e4086382","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"gold","angry":false}}
65
+ {"task_id":"tickets_9001_00064","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 26 seats. Appreciate you looking into it.\"\nContext: customer tier free; account age 45 months; open tickets 0.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-194da068f855","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"free","angry":false}}
66
+ {"task_id":"tickets_9001_00065","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I was charged $249 twice on Monday for order #69148. Only one charge should exist. No rush, but I'd like to get this sorted.\"\nContext: customer tier enterprise; account age 35 months; open tickets 1.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-f2a5fdec7e38","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"enterprise","angry":false}}
67
+ {"task_id":"tickets_9001_00066","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #75635? I need it for expenses. Appreciate you looking into it.\"\nContext: customer tier standard; account age 41 months; open tickets 1.","state_tokens":40,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-8150c7b308b5","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"standard","angry":false}}
68
+ {"task_id":"tickets_9001_00067","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Let me know if you need anything else from me.\"\nContext: customer tier enterprise; account age 48 months; open tickets 0.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-e74d8b9ae485","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"enterprise","angry":false}}
69
+ {"task_id":"tickets_9001_00068","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #94821 says delivered on the 3rd but nothing arrived. Tracking shows a photo of a door that isn't mine. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier gold; account age 32 months; open tickets 2.","state_tokens":53,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-f12349fc1f09","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"gold","angry":true}}
70
+ {"task_id":"tickets_9001_00069","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #55973? It's been 26 days. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 53 months; open tickets 2.","state_tokens":37,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-35e8e936519e","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"enterprise","angry":false}}
71
+ {"task_id":"tickets_9001_00070","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #74215 says delivered on yesterday but nothing arrived. Tracking shows a photo of a door that isn't mine. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier free; account age 50 months; open tickets 1.","state_tokens":52,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-fb4ab66fcda7","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"free","angry":true}}
72
+ {"task_id":"tickets_9001_00071","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I'm locked out. Two-factor codes go to a phone number I no longer have. Order #74558 is pending. Let me know if you need anything else from me.\"\nContext: customer tier gold; account age 16 months; open tickets 1.","state_tokens":52,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-c40ff03e2d11","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":2,"tier":"gold","angry":false}}
73
+ {"task_id":"tickets_9001_00072","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Package #72544 arrived crushed and the contents are broken. I need this for an event on the 3rd. This is completely unacceptable and I am furious.\"\nContext: customer tier enterprise; account age 20 months; open tickets 3.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":4,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-04576b192ccc","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":2,"tier":"enterprise","angry":true}}
74
+ {"task_id":"tickets_9001_00073","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #66282 says delivered on the 3rd but nothing arrived. Tracking shows a photo of a door that isn't mine. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 23 months; open tickets 0.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-2601f877aae0","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"enterprise","angry":false}}
75
+ {"task_id":"tickets_9001_00074","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Order #61395 says delivered on Sept 14 but nothing arrived. Tracking shows a photo of a door that isn't mine. No rush, but I'd like to get this sorted.\"\nContext: customer tier standard; account age 11 months; open tickets 3.","state_tokens":53,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-d7673f936d57","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":1,"tier":"standard","angry":false}}
76
+ {"task_id":"tickets_9001_00075","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Please cancel my subscription before the renewal on Sept 14. Reference #15502. Let me know if you need anything else from me.\"\nContext: customer tier standard; account age 42 months; open tickets 1.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-921cc2b1f474","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":1,"tier":"standard","angry":false}}
77
+ {"task_id":"tickets_9001_00076","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I was charged $45 twice on yesterday for order #96425. Only one charge should exist. No rush, but I'd like to get this sorted.\"\nContext: customer tier free; account age 45 months; open tickets 2.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-6f52aa8bb279","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"free","angry":false}}
78
+ {"task_id":"tickets_9001_00077","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 18 months; open tickets 2.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-6aab7bc56c09","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"enterprise","angry":false}}
79
+ {"task_id":"tickets_9001_00078","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #74976? I need it for expenses. No rush, but I'd like to get this sorted.\"\nContext: customer tier free; account age 14 months; open tickets 0.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-6ee3ed7432b5","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"free","angry":false}}
80
+ {"task_id":"tickets_9001_00079","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Please cancel my subscription before the renewal on Monday. Reference #77869. Let me know if you need anything else from me.\"\nContext: customer tier enterprise; account age 23 months; open tickets 1.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-b52f8ef9ef77","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":1,"tier":"enterprise","angry":false}}
81
+ {"task_id":"tickets_9001_00080","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 5 seats. No rush, but I'd like to get this sorted.\"\nContext: customer tier free; account age 49 months; open tickets 3.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-e756276748a1","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"free","angry":false}}
82
+ {"task_id":"tickets_9001_00081","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I'm locked out. Two-factor codes go to a phone number I no longer have. Order #52392 is pending. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier free; account age 44 months; open tickets 3.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-d9f9b90f2d42","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":2,"tier":"free","angry":true}}
83
+ {"task_id":"tickets_9001_00082","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"If I cancel, do I keep access until the end of the billing period? Appreciate you looking into it.\"\nContext: customer tier standard; account age 28 months; open tickets 3.","state_tokens":40,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-26e450d6d94f","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":0,"tier":"standard","angry":false}}
84
+ {"task_id":"tickets_9001_00083","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. This is completely unacceptable and I am furious.\"\nContext: customer tier free; account age 4 months; open tickets 3.","state_tokens":48,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-bb60199a76c6","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"free","angry":true}}
85
+ {"task_id":"tickets_9001_00084","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #22814? It's been 12 days. Honestly, this is ridiculous. Fix it today or I'm done.\"\nContext: customer tier free; account age 26 months; open tickets 2.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-9a3fba2b844e","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"free","angry":true}}
86
+ {"task_id":"tickets_9001_00085","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"What's the estimated delivery for order #55391? It's been 18 days. Appreciate you looking into it.\"\nContext: customer tier standard; account age 39 months; open tickets 0.","state_tokens":36,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-4ea4ff776574","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":0,"tier":"standard","angry":false}}
87
+ {"task_id":"tickets_9001_00086","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"If I cancel, do I keep access until the end of the billing period? Let me know if you need anything else from me.\"\nContext: customer tier gold; account age 40 months; open tickets 1.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-623eee4693d9","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":0,"tier":"gold","angry":false}}
88
+ {"task_id":"tickets_9001_00087","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Package #81383 arrived crushed and the contents are broken. I need this for an event on the 3rd. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 47 months; open tickets 2.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"shipping","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-acbc1543bbfd","permutation_id":"p0","metadata":{"template_queue":"shipping","base_urgency":2,"tier":"enterprise","angry":false}}
89
+ {"task_id":"tickets_9001_00088","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I'm locked out. Two-factor codes go to a phone number I no longer have. Order #74585 is pending. Let me know if you need anything else from me.\"\nContext: customer tier enterprise; account age 39 months; open tickets 3.","state_tokens":52,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-267003fb9692","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":2,"tier":"enterprise","angry":false}}
90
+ {"task_id":"tickets_9001_00089","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Notifications stopped arriving on Sept 14. Settings look correct. Device: Galaxy S24. No rush, but I'd like to get this sorted.\"\nContext: customer tier standard; account age 55 months; open tickets 3.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-484b6ea99417","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"standard","angry":false}}
91
+ {"task_id":"tickets_9001_00090","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #21330? I need it for expenses. Let me know if you need anything else from me.\"\nContext: customer tier standard; account age 53 months; open tickets 2.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":0,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-3acece4751e0","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"standard","angry":false}}
92
+ {"task_id":"tickets_9001_00091","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 15 seats. I've had it with this company. Absolutely appalling service.\"\nContext: customer tier free; account age 26 months; open tickets 0.","state_tokens":49,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-e04a2dabc9dc","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"free","angry":true}}
93
+ {"task_id":"tickets_9001_00092","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"My autopay failed on the 3rd and now there's a late fee of $120. I had funds available. Appreciate you looking into it.\"\nContext: customer tier enterprise; account age 21 months; open tickets 0.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-a9ec7508e8b9","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"enterprise","angry":false}}
94
+ {"task_id":"tickets_9001_00093","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"If I cancel, do I keep access until the end of the billing period? Thanks in advance for your help.\"\nContext: customer tier gold; account age 1 months; open tickets 1.","state_tokens":41,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"cancellation","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-9a4b02c9f690","permutation_id":"p0","metadata":{"template_queue":"cancellation","base_urgency":0,"tier":"gold","angry":false}}
95
+ {"task_id":"tickets_9001_00094","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Export to CSV produces an empty file for reports over 22 days. Thanks in advance for your help.\"\nContext: customer tier gold; account age 44 months; open tickets 1.","state_tokens":39,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-47776bd7bded","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"gold","angry":false}}
96
+ {"task_id":"tickets_9001_00095","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Notifications stopped arriving on the 3rd. Settings look correct. Device: Pixel 8. Honestly, this is ridiculous. Fix it today or I'm done.\"\nContext: customer tier enterprise; account age 1 months; open tickets 3.","state_tokens":44,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"technical","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":true,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":3,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-0b95cb983ae9","permutation_id":"p0","metadata":{"template_queue":"technical","base_urgency":1,"tier":"enterprise","angry":true}}
97
+ {"task_id":"tickets_9001_00096","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"How do I change the email on my account? I want to switch to my work address. Let me know if you need anything else from me.\"\nContext: customer tier gold; account age 7 months; open tickets 3.","state_tokens":50,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"account","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-c9496c648ce3","permutation_id":"p0","metadata":{"template_queue":"account","base_urgency":0,"tier":"gold","angry":false}}
98
+ {"task_id":"tickets_9001_00097","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Do you offer a discount for annual billing on the team plan? We'd be about 18 seats. Thanks in advance for your help.\"\nContext: customer tier enterprise; account age 10 months; open tickets 1.","state_tokens":45,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"sales","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-0d9172374ab3","permutation_id":"p0","metadata":{"template_queue":"sales","base_urgency":0,"tier":"enterprise","angry":false}}
99
+ {"task_id":"tickets_9001_00098","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"Can you send me the invoice for order #77244? I need it for expenses. Let me know if you need anything else from me.\"\nContext: customer tier gold; account age 6 months; open tickets 3.","state_tokens":46,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":1,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-6f89374a7d18","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":0,"tier":"gold","angry":false}}
100
+ {"task_id":"tickets_9001_00099","tier":"G","domain":"support_triage","language":"en","state":"Customer: \"I was charged $89 twice on yesterday for order #21709. Only one charge should exist. Thanks in advance for your help.\"\nContext: customer tier gold; account age 47 months; open tickets 1.","state_tokens":42,"questions":{"queue":{"type":"choice","instructions":"Which queue should handle this ticket?","criteria":[{"key":"billing","description":"Charges, refunds, invoices, duplicate payments, autopay problems"},{"key":"shipping","description":"Delivery delays, tracking, lost or damaged packages, wrong address"},{"key":"account","description":"Login, password, two-factor, email change, account lockout"},{"key":"technical","description":"App crashes, errors, features not working, connectivity"},{"key":"cancellation","description":"Ending a subscription or closing an account"},{"key":"sales","description":"Pre-purchase questions, upgrades, quotes, discounts"}],"ground_truth":"billing","framing_group":"tickets.queue","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"is_angry":{"type":"noul","instructions":"Is the customer expressing anger or hostility?","criteria":null,"ground_truth":false,"framing_group":"tickets.is_angry","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"priority":{"type":"score","instructions":"Assign a priority from 0 (lowest) to 4 (highest). Policy: start from the urgency of the issue (informational 0, routine 1, blocked or money at stake 2). Add 1 if the customer is angry. Add 1 if the customer tier is gold or enterprise. Cap at 4.","criteria":[{"level":0,"description":"No action needed or informational."},{"level":1,"description":"Routine request; respond within 2 business days."},{"level":2,"description":"Customer blocked or money at stake; respond within 1 business day."},{"level":3,"description":"Urgent: blocked plus angry or high-value customer; respond within 4 hours."},{"level":4,"description":"Critical: blocked, high-value and angry, or active fraud; respond within 1 hour."}],"ground_truth":2,"framing_group":"tickets.priority","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"support_tickets@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"queue":{"billing":{"billing":0.0,"shipping":3.0,"account":3.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"shipping":{"billing":1.0,"shipping":0.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"account":{"billing":3.0,"shipping":3.0,"account":0.0,"technical":3.0,"cancellation":3.0,"sales":3.0,"privacy":3.0,"returns":3.0,"fraud":3.0,"feedback":3.0,"accessibility":3.0,"partnership":3.0},"technical":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":0.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"cancellation":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":0.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"sales":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":0.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"privacy":{"billing":4.0,"shipping":4.0,"account":4.0,"technical":4.0,"cancellation":4.0,"sales":4.0,"privacy":0.0,"returns":4.0,"fraud":4.0,"feedback":4.0,"accessibility":4.0,"partnership":4.0},"returns":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":0.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":1.0},"fraud":{"billing":8.0,"shipping":8.0,"account":8.0,"technical":8.0,"cancellation":8.0,"sales":8.0,"privacy":8.0,"returns":8.0,"fraud":0.0,"feedback":8.0,"accessibility":8.0,"partnership":8.0},"feedback":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":0.0,"accessibility":1.0,"partnership":1.0},"accessibility":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":0.0,"partnership":1.0},"partnership":{"billing":1.0,"shipping":1.0,"account":1.0,"technical":1.0,"cancellation":1.0,"sales":1.0,"privacy":1.0,"returns":1.0,"fraud":1.0,"feedback":1.0,"accessibility":1.0,"partnership":0.0}},"priority":{"0":{"0":0.0,"1":1.0,"2":2.0,"3":3.0,"4":4.0},"1":{"0":2.0,"1":0.0,"2":1.0,"3":2.0,"4":3.0},"2":{"0":4.0,"1":2.0,"2":0.0,"3":1.0,"4":2.0},"3":{"0":6.0,"1":4.0,"2":2.0,"3":0.0,"4":1.0},"4":{"0":8.0,"1":6.0,"2":4.0,"3":2.0,"4":0.0}},"is_angry":{"true":{"true":0.0,"false":3.0},"false":{"true":0.5,"false":0.0}}},"canary":"s1b-aba97f9ba50a","permutation_id":"p0","metadata":{"template_queue":"billing","base_urgency":1,"tier":"gold","angry":false}}
101
+ {"task_id":"phish_9001_00000","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso-cloud-login.net>","to":"luca@example-corp.com","subject":"Urgent: verify your Contoso Cloud account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://contoso-cloud-login.net/verify or your account will be suspended.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-8d917af9c2e9","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"contoso-cloud-login.net","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
102
+ {"task_id":"phish_9001_00001","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"priya@example-corp.com","subject":"Delivery update for order 35892","body":"Hi Priya, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-1b20531b5f08","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
103
+ {"task_id":"phish_9001_00002","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"jordan@example-corp.com","subject":"Delivery update for order 31275","body":"Hi Jordan, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-66183ae9f461","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
104
+ {"task_id":"phish_9001_00003","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"mei@example-corp.com","subject":"Delivery update for order 58686","body":"Hi Mei, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-1b9ccc5da5d9","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
105
+ {"task_id":"phish_9001_00004","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"jordan@example-corp.com","subject":"Invoice 45827 attached","body":"Hi Jordan, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_45827.pdf"],"known_brand_domain":"fabrikam.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-1d27eda09ca9","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
106
+ {"task_id":"phish_9001_00005","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"jordan@example-corp.com","subject":"Updated banking details for invoice 15827","body":"Hi Jordan, please note our bank details have changed. Remit invoice 15827 to the new account in the attached PDF by end of week.","attachments":["file_15827.pdf"],"known_brand_domain":"northwindbank.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-be6ce9a84171","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
107
+ {"task_id":"phish_9001_00006","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"alex@example-corp.com","subject":"Team lunch Thursday","body":"Hi Alex, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-86c381352b2d","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
108
+ {"task_id":"phish_9001_00007","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove-hr.org>","to":"luca@example-corp.com","subject":"Urgent: verify your Woodgrove HR account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://woodgrove-hr.org/verify or your account will be suspended.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-e8b895c3bb75","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"woodgrove-hr.org","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
109
+ {"task_id":"phish_9001_00008","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"noor@example-corp.com","subject":"Updated banking details for invoice 44123","body":"Hi Noor, please note our bank details have changed. Remit invoice 44123 to the new account in the attached PDF by end of week.","attachments":["file_44123.pdf"],"known_brand_domain":"northwindbank.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-9f7f770c2614","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
110
+ {"task_id":"phish_9001_00009","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"priya@example-corp.com","subject":"Your Contoso Cloud statement is ready","body":"Hi Priya, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-d085e8ccb304","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
111
+ {"task_id":"phish_9001_00010","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"mei@example-corp.com","subject":"Password expires today","body":"Your Northwind Bank password expires today. Reset now: https://northwind-bank-secure.com/reset","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-0ae9f0fd927d","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
112
+ {"task_id":"phish_9001_00011","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"alex@example-corp.com","subject":"Urgent: verify your Fabrikam Shipping account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://fabrikam-delivery.co/verify or your account will be suspended.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-dab9610fc45e","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
113
+ {"task_id":"phish_9001_00012","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove-hr.org>","to":"mei@example-corp.com","subject":"Updated banking details for invoice 61403","body":"Hi Mei, please note our bank details have changed. Remit invoice 61403 to the new account in the attached PDF by end of week.","attachments":["file_61403.pdf"],"known_brand_domain":"woodgrove.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-58b9b6ebc8c4","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"woodgrove-hr.org","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
114
+ {"task_id":"phish_9001_00013","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"mei@example-corp.com","subject":"Your Woodgrove HR statement is ready","body":"Hi Mei, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-0dfff253f5c2","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
115
+ {"task_id":"phish_9001_00014","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"jordan@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_61736.zip). Open to review.","attachments":["file_61736.zip"],"known_brand_domain":"fabrikam.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-347ca69e41ec","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
116
+ {"task_id":"phish_9001_00015","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"alex@example-corp.com","subject":"Team lunch Thursday","body":"Hi Alex, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-16c4cb16af38","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
117
+ {"task_id":"phish_9001_00016","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove-hr.org>","to":"jordan@example-corp.com","subject":"Password expires today","body":"Your Woodgrove HR password expires today. Reset now: https://woodgrove-hr.org/reset","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-ee0b8b9ff986","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"woodgrove-hr.org","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
118
+ {"task_id":"phish_9001_00017","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"noor@example-corp.com","subject":"Quick favour","body":"Noor, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-59b17c1ba99b","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
119
+ {"task_id":"phish_9001_00018","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"mei@example-corp.com","subject":"Delivery update for order 34969","body":"Hi Mei, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-b75f051f4dd0","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
120
+ {"task_id":"phish_9001_00019","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"noor@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Noor, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-f1e25e80441d","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
121
+ {"task_id":"phish_9001_00020","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"jordan@example-corp.com","subject":"Team lunch Thursday","body":"Hi Jordan, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-6184e141ef4e","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
122
+ {"task_id":"phish_9001_00021","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"jordan@example-corp.com","subject":"Team lunch Thursday","body":"Hi Jordan, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-dbfc3bcd1a19","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
123
+ {"task_id":"phish_9001_00022","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"priya@example-corp.com","subject":"Delivery update for order 26313","body":"Hi Priya, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-0fcf86429ca2","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
124
+ {"task_id":"phish_9001_00023","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin-payroll.info>","to":"jordan@example-corp.com","subject":"Quick favour","body":"Jordan, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-f5a5a1f72b25","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"tailspin-payroll.info","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
125
+ {"task_id":"phish_9001_00024","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"fatima@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_76433.zip). Open to review.","attachments":["file_76433.zip"],"known_brand_domain":"northwindbank.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-0bc643c7c05b","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
126
+ {"task_id":"phish_9001_00025","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"luca@example-corp.com","subject":"Team lunch Thursday","body":"Hi Luca, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-ebb100b2533e","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
127
+ {"task_id":"phish_9001_00026","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin-payroll.info>","to":"sam@example-corp.com","subject":"Quick favour","body":"Sam, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-a49cba8cdfd0","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"tailspin-payroll.info","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
128
+ {"task_id":"phish_9001_00027","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"fatima@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Fatima, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-ce6c3c710489","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
129
+ {"task_id":"phish_9001_00028","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"priya@example-corp.com","subject":"Your Contoso Cloud statement is ready","body":"Hi Priya, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-ee31709b1826","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
130
+ {"task_id":"phish_9001_00029","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"fatima@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Fatima, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-50eaf207dc03","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
131
+ {"task_id":"phish_9001_00030","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"alex@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_46405.zip). Open to review.","attachments":["file_46405.zip"],"known_brand_domain":"northwindbank.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-8a8e34001d48","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
132
+ {"task_id":"phish_9001_00031","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"fatima@example-corp.com","subject":"Urgent: verify your Fabrikam Shipping account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://fabrikam-delivery.co/verify or your account will be suspended.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-5a4e34758139","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
133
+ {"task_id":"phish_9001_00032","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"luca@example-corp.com","subject":"Password expires today","body":"Your Fabrikam Shipping password expires today. Reset now: https://fabrikam-delivery.co/reset","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-4e6e2da5c987","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
134
+ {"task_id":"phish_9001_00033","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"alex@example-corp.com","subject":"Team lunch Thursday","body":"Hi Alex, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-4b9f5db7ae52","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
135
+ {"task_id":"phish_9001_00034","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"alex@example-corp.com","subject":"Team lunch Thursday","body":"Hi Alex, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-b3c591dd97d4","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
136
+ {"task_id":"phish_9001_00035","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin-payroll.info>","to":"luca@example-corp.com","subject":"Updated banking details for invoice 35293","body":"Hi Luca, please note our bank details have changed. Remit invoice 35293 to the new account in the attached PDF by end of week.","attachments":["file_35293.pdf"],"known_brand_domain":"tailspin.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-951c3ff4826d","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"tailspin-payroll.info","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
137
+ {"task_id":"phish_9001_00036","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin-payroll.info>","to":"noor@example-corp.com","subject":"Urgent: verify your Tailspin Payroll account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://tailspin-payroll.info/verify or your account will be suspended.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-daa8d53955a1","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"tailspin-payroll.info","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
138
+ {"task_id":"phish_9001_00037","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"sam@example-corp.com","subject":"Delivery update for order 30147","body":"Hi Sam, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-fb691ea04bea","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
139
+ {"task_id":"phish_9001_00038","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"priya@example-corp.com","subject":"Team lunch Thursday","body":"Hi Priya, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-7c4c838a1492","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
140
+ {"task_id":"phish_9001_00039","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"fatima@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_60771.zip). Open to review.","attachments":["file_60771.zip"],"known_brand_domain":"fabrikam.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-f844afeb8ce3","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
141
+ {"task_id":"phish_9001_00040","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"sam@example-corp.com","subject":"Invoice 50023 attached","body":"Hi Sam, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_50023.pdf"],"known_brand_domain":"northwindbank.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-ed7a53621680","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
142
+ {"task_id":"phish_9001_00041","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"mei@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Mei, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-864ffc48e8af","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
143
+ {"task_id":"phish_9001_00042","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin-payroll.info>","to":"jordan@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_96044.zip). Open to review.","attachments":["file_96044.zip"],"known_brand_domain":"tailspin.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-8c7473139fd3","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"tailspin-payroll.info","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
144
+ {"task_id":"phish_9001_00043","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"mei@example-corp.com","subject":"Urgent: verify your Tailspin Payroll account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://tailspin-payroll.info/verify or your account will be suspended.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-08bc6fb9d146","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
145
+ {"task_id":"phish_9001_00044","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"jordan@example-corp.com","subject":"Your Contoso Cloud statement is ready","body":"Hi Jordan, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-b4efa9699fd9","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
146
+ {"task_id":"phish_9001_00045","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"alex@example-corp.com","subject":"Team lunch Thursday","body":"Hi Alex, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-a926d47a09b6","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
147
+ {"task_id":"phish_9001_00046","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"fatima@example-corp.com","subject":"Your Contoso Cloud statement is ready","body":"Hi Fatima, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-5ad294102d5b","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
148
+ {"task_id":"phish_9001_00047","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"mei@example-corp.com","subject":"Delivery update for order 52309","body":"Hi Mei, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-8e896fd9b145","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
149
+ {"task_id":"phish_9001_00048","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"jordan@example-corp.com","subject":"Team lunch Thursday","body":"Hi Jordan, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-17a51840aa18","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
150
+ {"task_id":"phish_9001_00049","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove-hr.org>","to":"jordan@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_11245.zip). Open to review.","attachments":["file_11245.zip"],"known_brand_domain":"woodgrove.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-6814d363c2f9","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"woodgrove-hr.org","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
151
+ {"task_id":"phish_9001_00050","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"fatima@example-corp.com","subject":"Your Tailspin Payroll statement is ready","body":"Hi Fatima, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-0f8ee85112ee","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
152
+ {"task_id":"phish_9001_00051","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"mei@example-corp.com","subject":"Delivery update for order 42788","body":"Hi Mei, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-745c18bf64d8","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
153
+ {"task_id":"phish_9001_00052","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"mei@example-corp.com","subject":"Updated banking details for invoice 58108","body":"Hi Mei, please note our bank details have changed. Remit invoice 58108 to the new account in the attached PDF by end of week.","attachments":["file_58108.pdf"],"known_brand_domain":"northwindbank.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-e6eb4f243f41","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
154
+ {"task_id":"phish_9001_00053","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove-hr.org>","to":"priya@example-corp.com","subject":"Urgent: verify your Woodgrove HR account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://woodgrove-hr.org/verify or your account will be suspended.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-af308e715438","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"woodgrove-hr.org","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
155
+ {"task_id":"phish_9001_00054","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso-cloud-login.net>","to":"mei@example-corp.com","subject":"Updated banking details for invoice 72064","body":"Hi Mei, please note our bank details have changed. Remit invoice 72064 to the new account in the attached PDF by end of week.","attachments":["file_72064.pdf"],"known_brand_domain":"contoso.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-c6e0f0ccc281","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"contoso-cloud-login.net","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
156
+ {"task_id":"phish_9001_00055","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"sam@example-corp.com","subject":"Quick favour","body":"Sam, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-07e3090d6848","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
157
+ {"task_id":"phish_9001_00056","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"jordan@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_87399.zip). Open to review.","attachments":["file_87399.zip"],"known_brand_domain":"northwindbank.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-16abe327b34d","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
158
+ {"task_id":"phish_9001_00057","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"priya@example-corp.com","subject":"Urgent: verify your Fabrikam Shipping account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://fabrikam-delivery.co/verify or your account will be suspended.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-3e40fec7446d","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
159
+ {"task_id":"phish_9001_00058","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"sam@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Sam, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-9c906c0b0a3d","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
160
+ {"task_id":"phish_9001_00059","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"jordan@example-corp.com","subject":"Delivery update for order 96663","body":"Hi Jordan, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-5df5515c9ed0","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
161
+ {"task_id":"phish_9001_00060","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"fatima@example-corp.com","subject":"Invoice 51582 attached","body":"Hi Fatima, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_51582.pdf"],"known_brand_domain":"fabrikam.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-d31692d97fc5","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
162
+ {"task_id":"phish_9001_00061","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"fatima@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_10345.zip). Open to review.","attachments":["file_10345.zip"],"known_brand_domain":"northwindbank.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-e88f86994047","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
163
+ {"task_id":"phish_9001_00062","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"fatima@example-corp.com","subject":"Team lunch Thursday","body":"Hi Fatima, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-da6bd9ddcc60","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
164
+ {"task_id":"phish_9001_00063","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"alex@example-corp.com","subject":"Your Northwind Bank statement is ready","body":"Hi Alex, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-ea9aa500503c","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
165
+ {"task_id":"phish_9001_00064","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"sam@example-corp.com","subject":"Invoice 64433 attached","body":"Hi Sam, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_64433.pdf"],"known_brand_domain":"contoso.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-8ed0dfac7a57","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
166
+ {"task_id":"phish_9001_00065","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"noor@example-corp.com","subject":"Invoice 90796 attached","body":"Hi Noor, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_90796.pdf"],"known_brand_domain":"tailspin.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-53f3d6c86b13","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
167
+ {"task_id":"phish_9001_00066","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"luca@example-corp.com","subject":"Invoice 28766 attached","body":"Hi Luca, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_28766.pdf"],"known_brand_domain":"contoso.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-438ff2853dc1","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
168
+ {"task_id":"phish_9001_00067","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"mei@example-corp.com","subject":"Invoice 83033 attached","body":"Hi Mei, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_83033.pdf"],"known_brand_domain":"northwindbank.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-bdb87b9f7d70","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
169
+ {"task_id":"phish_9001_00068","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"jordan@example-corp.com","subject":"Urgent: verify your Fabrikam Shipping account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://fabrikam-delivery.co/verify or your account will be suspended.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-3e9963bcf809","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
170
+ {"task_id":"phish_9001_00069","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"luca@example-corp.com","subject":"Team lunch Thursday","body":"Hi Luca, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-a8d5ca311aa1","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
171
+ {"task_id":"phish_9001_00070","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"mei@example-corp.com","subject":"Team lunch Thursday","body":"Hi Mei, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-0f10d261affb","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
172
+ {"task_id":"phish_9001_00071","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"sam@example-corp.com","subject":"Team lunch Thursday","body":"Hi Sam, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-b8ff48aa4897","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
173
+ {"task_id":"phish_9001_00072","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"luca@example-corp.com","subject":"Quick favour","body":"Luca, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-c9c372fcb919","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
174
+ {"task_id":"phish_9001_00073","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove-hr.org>","to":"noor@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_49334.zip). Open to review.","attachments":["file_49334.zip"],"known_brand_domain":"woodgrove.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-3a558d92d74d","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"woodgrove-hr.org","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
175
+ {"task_id":"phish_9001_00074","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso-cloud-login.net>","to":"luca@example-corp.com","subject":"Quick favour","body":"Luca, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-fed2d7c5ffd4","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"contoso-cloud-login.net","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
176
+ {"task_id":"phish_9001_00075","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove-hr.org>","to":"sam@example-corp.com","subject":"Quick favour","body":"Sam, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-1a714c5d1ff5","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"woodgrove-hr.org","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
177
+ {"task_id":"phish_9001_00076","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"luca@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Luca, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-4c70b495c65e","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
178
+ {"task_id":"phish_9001_00077","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin-payroll.info>","to":"fatima@example-corp.com","subject":"Urgent: verify your Tailspin Payroll account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://tailspin-payroll.info/verify or your account will be suspended.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-970b79045421","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"tailspin-payroll.info","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
179
+ {"task_id":"phish_9001_00078","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"sam@example-corp.com","subject":"Urgent: verify your Fabrikam Shipping account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://fabrikam-delivery.co/verify or your account will be suspended.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-52e413927d92","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
180
+ {"task_id":"phish_9001_00079","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"fatima@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Fatima, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-9c869822f96e","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
181
+ {"task_id":"phish_9001_00080","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"mei@example-corp.com","subject":"Delivery update for order 56502","body":"Hi Mei, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-01fac7e10f38","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
182
+ {"task_id":"phish_9001_00081","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"sam@example-corp.com","subject":"Your Woodgrove HR statement is ready","body":"Hi Sam, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-b13e44a35631","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
183
+ {"task_id":"phish_9001_00082","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin-payroll.info>","to":"priya@example-corp.com","subject":"Updated banking details for invoice 44621","body":"Hi Priya, please note our bank details have changed. Remit invoice 44621 to the new account in the attached PDF by end of week.","attachments":["file_44621.pdf"],"known_brand_domain":"tailspin.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-9f465fb51ef8","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"tailspin-payroll.info","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
184
+ {"task_id":"phish_9001_00083","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"fatima@example-corp.com","subject":"Your Northwind Bank statement is ready","body":"Hi Fatima, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-bb89a7757102","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
185
+ {"task_id":"phish_9001_00084","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"fatima@example-corp.com","subject":"Quick favour","body":"Fatima, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-ce390698147b","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
186
+ {"task_id":"phish_9001_00085","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"sam@example-corp.com","subject":"Team lunch Thursday","body":"Hi Sam, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-2727f16d977c","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
187
+ {"task_id":"phish_9001_00086","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"jordan@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Jordan, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-e2d953bacda6","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
188
+ {"task_id":"phish_9001_00087","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"mei@example-corp.com","subject":"Invoice 38950 attached","body":"Hi Mei, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_38950.pdf"],"known_brand_domain":"contoso.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-b250116e70bd","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
189
+ {"task_id":"phish_9001_00088","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwindbank.com>","to":"luca@example-corp.com","subject":"Your Northwind Bank statement is ready","body":"Hi Luca, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"northwindbank.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-257883117edb","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"northwindbank.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
190
+ {"task_id":"phish_9001_00089","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam.com>","to":"jordan@example-corp.com","subject":"Team lunch Thursday","body":"Hi Jordan, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"fabrikam.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-7aad17414285","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"fabrikam.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
191
+ {"task_id":"phish_9001_00090","tier":"G","domain":"email_security","language":"en","state":{"from":"Northwind Bank <no-reply@northwind-bank-secure.com>","to":"luca@example-corp.com","subject":"Scanned document","body":"Please see the attached scan (Document_32824.zip). Open to review.","attachments":["file_32824.zip"],"known_brand_domain":"northwindbank.com"},"state_tokens":51,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"malware_attachment","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-1509ea1f021c","permutation_id":"p0","metadata":{"class":"malware_attachment","sender_domain":"northwind-bank-secure.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
192
+ {"task_id":"phish_9001_00091","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"luca@example-corp.com","subject":"Invoice 94325 attached","body":"Hi Luca, attached is the invoice we discussed on the call. Let me know if anything looks off.","attachments":["file_94325.pdf"],"known_brand_domain":"contoso.com"},"state_tokens":63,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-64abbc809a92","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
193
+ {"task_id":"phish_9001_00092","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"luca@example-corp.com","subject":"Your Woodgrove HR statement is ready","body":"Hi Luca, your monthly statement is available in the app under Documents. No action is required.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-a9b92942a04c","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
194
+ {"task_id":"phish_9001_00093","tier":"G","domain":"email_security","language":"en","state":{"from":"Fabrikam Shipping <no-reply@fabrikam-delivery.co>","to":"fatima@example-corp.com","subject":"Updated banking details for invoice 83433","body":"Hi Fatima, please note our bank details have changed. Remit invoice 83433 to the new account in the attached PDF by end of week.","attachments":["file_83433.pdf"],"known_brand_domain":"fabrikam.com"},"state_tokens":71,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"invoice_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":2,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-c2ffa557ce78","permutation_id":"p0","metadata":{"class":"invoice_fraud","sender_domain":"fabrikam-delivery.co","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
195
+ {"task_id":"phish_9001_00094","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"fatima@example-corp.com","subject":"Delivery update for order 47836","body":"Hi Fatima, your parcel is out for delivery today. Track it in your account.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":0,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-e4962a0deb4f","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
196
+ {"task_id":"phish_9001_00095","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso.com>","to":"jordan@example-corp.com","subject":"Q3 timesheet reminder","body":"Hi Jordan, timesheets are due Friday. Submit through the usual portal when you get a chance.","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":60,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-7b4d8c0642ed","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"contoso.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
197
+ {"task_id":"phish_9001_00096","tier":"G","domain":"email_security","language":"en","state":{"from":"Contoso Cloud <no-reply@contoso-cloud-login.net>","to":"sam@example-corp.com","subject":"Quick favour","body":"Sam, I'm in a meeting and need you to buy 5 gift cards for a client today. Send the codes here. Keep it confidential. Sent from my iPhone","attachments":[],"known_brand_domain":"contoso.com"},"state_tokens":76,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"ceo_fraud","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-3a20036353ca","permutation_id":"p0","metadata":{"class":"ceo_fraud","sender_domain":"contoso-cloud-login.net","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
198
+ {"task_id":"phish_9001_00097","tier":"G","domain":"email_security","language":"en","state":{"from":"Woodgrove HR <no-reply@woodgrove.com>","to":"fatima@example-corp.com","subject":"Urgent: verify your Woodgrove HR account","body":"Dear customer, unusual sign-in detected. Verify your identity within 24 hours at https://woodgrove-hr.org/verify or your account will be suspended.","attachments":[],"known_brand_domain":"woodgrove.com"},"state_tokens":64,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":true,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"credential_harvest","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":3,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":true,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-aac8fbdf6f44","permutation_id":"p0","metadata":{"class":"credential_harvest","sender_domain":"woodgrove.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
199
+ {"task_id":"phish_9001_00098","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"jordan@example-corp.com","subject":"Team lunch Thursday","body":"Hi Jordan, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-cf99f2e9cfe5","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}
200
+ {"task_id":"phish_9001_00099","tier":"G","domain":"email_security","language":"en","state":{"from":"Tailspin Payroll <no-reply@tailspin.com>","to":"jordan@example-corp.com","subject":"Team lunch Thursday","body":"Hi Jordan, we're doing lunch Thursday at noon. Reply if you have dietary needs.","attachments":[],"known_brand_domain":"tailspin.com"},"state_tokens":58,"questions":{"is_phishing":{"type":"noul","instructions":"Is this email a phishing or social-engineering attack?","criteria":null,"ground_truth":false,"framing_group":"phish.is_phishing","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"attack_class":{"type":"choice","instructions":"Classify the email.","criteria":[{"key":"credential_harvest","description":"Asks the recipient to log in or verify credentials via a link"},{"key":"invoice_fraud","description":"Fake invoice or payment redirection"},{"key":"malware_attachment","description":"Unexpected attachment the recipient is urged to open"},{"key":"ceo_fraud","description":"Impersonates an executive requesting an urgent transfer or gift cards"},{"key":"benign","description":"Legitimate message with no attack"}],"ground_truth":"benign","framing_group":"phish.attack_class","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"urgency":{"type":"score","instructions":"Rate the time pressure the message applies, 0 (none) to 3 (within 24 hours or threats).","criteria":[{"level":0,"description":"No time pressure"},{"level":1,"description":"Mild: 'when you get a chance'"},{"level":2,"description":"Moderate: action requested within days"},{"level":3,"description":"High: within 24 hours or account consequences"}],"ground_truth":1,"framing_group":"phish.urgency","framing_id":"f0","decomposition_of":null,"allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_sender_mismatch":{"type":"noul","instructions":"Does the sender domain differ from the known brand domain?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.sender_mismatch","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_asks_credentials":{"type":"noul","instructions":"Does the message ask the recipient to log in, verify, or reset credentials via a link?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.asks_credentials","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_urgent_language":{"type":"noul","instructions":"Does the message pressure the recipient to act within 24 hours or threaten consequences?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.urgent_language","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_payment_redirect":{"type":"noul","instructions":"Does the message ask to change payment details or send money or gift cards?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.payment_redirect","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"},"sub_unexpected_attachment":{"type":"noul","instructions":"Does the message urge opening an attachment the recipient did not request?","criteria":null,"ground_truth":false,"framing_group":"phish.sub.unexpected_attachment","framing_id":"f0","decomposition_of":"is_phishing","allow_abstain":false,"abstain_key":"none_of_the_above"}},"label_provenance":{"source":"generator","generator":"phishing_email@1.0.0","seed":9001,"noise_injected":false,"annotators":null,"agreement":null},"controls":{"unknowable":false,"none_correct":false,"distractor_density":0.0,"perturbation":null,"prior_shift":null},"cost_matrix":{"is_phishing":{"true":{"true":0.0,"false":20.0},"false":{"true":1.0,"false":0.0}}},"canary":"s1b-b0ed30f9d2a2","permutation_id":"p0","metadata":{"class":"benign","sender_domain":"tailspin.com","decomposition_rule":"phishing if sender_mismatch or any of the other four"}}