pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
@@ -0,0 +1,360 @@
1
+ {"item": "demo00", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 65, "completion_tokens": 311, "reasoning_tokens": null, "latency": 3.8633601402444997, "model": "synthetic"}
2
+ {"item": "demo00", "arm": "sample", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 65, "completion_tokens": 277, "reasoning_tokens": null, "latency": 2.7355575860387646, "model": "synthetic"}
3
+ {"item": "demo00", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 65, "completion_tokens": 253, "reasoning_tokens": null, "latency": 5.056120353753874, "model": "synthetic"}
4
+ {"item": "demo00", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 65, "completion_tokens": 222, "reasoning_tokens": null, "latency": 5.6534194461241825, "model": "synthetic"}
5
+ {"item": "demo00", "arm": "sample", "idx": 3, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 65, "completion_tokens": 302, "reasoning_tokens": null, "latency": 4.09321661190071, "model": "synthetic"}
6
+ {"item": "demo00", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 65, "completion_tokens": 213, "reasoning_tokens": null, "latency": 2.0808622934245875, "model": "synthetic"}
7
+ {"item": "demo00", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 65, "completion_tokens": 236, "reasoning_tokens": null, "latency": 2.7500133374238662, "model": "synthetic"}
8
+ {"item": "demo00", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 65, "completion_tokens": 214, "reasoning_tokens": null, "latency": 4.361551471339407, "model": "synthetic"}
9
+ {"item": "demo00", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 65, "completion_tokens": 281, "reasoning_tokens": null, "latency": 2.0843014177755643, "model": "synthetic"}
10
+ {"item": "demo01", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 69, "completion_tokens": 174, "reasoning_tokens": null, "latency": 4.294564717994092, "model": "synthetic"}
11
+ {"item": "demo01", "arm": "sample", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 69, "completion_tokens": 206, "reasoning_tokens": null, "latency": 2.384982710512174, "model": "synthetic"}
12
+ {"item": "demo01", "arm": "sample", "idx": 1, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 69, "completion_tokens": 223, "reasoning_tokens": null, "latency": 5.214050796521313, "model": "synthetic"}
13
+ {"item": "demo01", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 69, "completion_tokens": 215, "reasoning_tokens": null, "latency": 5.469093550289424, "model": "synthetic"}
14
+ {"item": "demo01", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 69, "completion_tokens": 171, "reasoning_tokens": null, "latency": 2.1068094003451385, "model": "synthetic"}
15
+ {"item": "demo01", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 69, "completion_tokens": 203, "reasoning_tokens": null, "latency": 5.779218119227364, "model": "synthetic"}
16
+ {"item": "demo01", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 69, "completion_tokens": 178, "reasoning_tokens": null, "latency": 3.8260401998474776, "model": "synthetic"}
17
+ {"item": "demo01", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 69, "completion_tokens": 166, "reasoning_tokens": null, "latency": 3.522339054002171, "model": "synthetic"}
18
+ {"item": "demo01", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 69, "completion_tokens": 210, "reasoning_tokens": null, "latency": 5.322210129052736, "model": "synthetic"}
19
+ {"item": "demo02", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 259, "reasoning_tokens": null, "latency": 5.535689858888587, "model": "synthetic"}
20
+ {"item": "demo02", "arm": "sample", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 223, "reasoning_tokens": null, "latency": 4.909299106954176, "model": "synthetic"}
21
+ {"item": "demo02", "arm": "sample", "idx": 1, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 300, "reasoning_tokens": null, "latency": 4.696803961740844, "model": "synthetic"}
22
+ {"item": "demo02", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 267, "reasoning_tokens": null, "latency": 5.029839625528808, "model": "synthetic"}
23
+ {"item": "demo02", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 241, "reasoning_tokens": null, "latency": 4.144480169517489, "model": "synthetic"}
24
+ {"item": "demo02", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 308, "reasoning_tokens": null, "latency": 2.5029660701366767, "model": "synthetic"}
25
+ {"item": "demo02", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 297, "reasoning_tokens": null, "latency": 4.578086492093899, "model": "synthetic"}
26
+ {"item": "demo02", "arm": "sample", "idx": 6, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 74, "completion_tokens": 312, "reasoning_tokens": null, "latency": 5.372100267366345, "model": "synthetic"}
27
+ {"item": "demo02", "arm": "sample", "idx": 7, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 254, "reasoning_tokens": null, "latency": 4.5649164424392605, "model": "synthetic"}
28
+ {"item": "demo03", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 70, "completion_tokens": 236, "reasoning_tokens": null, "latency": 2.367239652606438, "model": "synthetic"}
29
+ {"item": "demo03", "arm": "sample", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 70, "completion_tokens": 236, "reasoning_tokens": null, "latency": 2.707230236547584, "model": "synthetic"}
30
+ {"item": "demo03", "arm": "sample", "idx": 1, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 70, "completion_tokens": 306, "reasoning_tokens": null, "latency": 2.7622289926324535, "model": "synthetic"}
31
+ {"item": "demo03", "arm": "sample", "idx": 2, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 70, "completion_tokens": 247, "reasoning_tokens": null, "latency": 5.076443960746559, "model": "synthetic"}
32
+ {"item": "demo03", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 70, "completion_tokens": 276, "reasoning_tokens": null, "latency": 2.130233306580772, "model": "synthetic"}
33
+ {"item": "demo03", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 70, "completion_tokens": 258, "reasoning_tokens": null, "latency": 4.878993756554302, "model": "synthetic"}
34
+ {"item": "demo03", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 70, "completion_tokens": 229, "reasoning_tokens": null, "latency": 5.981446637715988, "model": "synthetic"}
35
+ {"item": "demo03", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 70, "completion_tokens": 250, "reasoning_tokens": null, "latency": 3.576441018899872, "model": "synthetic"}
36
+ {"item": "demo03", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 70, "completion_tokens": 237, "reasoning_tokens": null, "latency": 2.1320956986629427, "model": "synthetic"}
37
+ {"item": "demo04", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 87, "completion_tokens": 188, "reasoning_tokens": null, "latency": 4.025904064752523, "model": "synthetic"}
38
+ {"item": "demo04", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 87, "completion_tokens": 249, "reasoning_tokens": null, "latency": 2.162115439579106, "model": "synthetic"}
39
+ {"item": "demo04", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 87, "completion_tokens": 222, "reasoning_tokens": null, "latency": 2.265592874013116, "model": "synthetic"}
40
+ {"item": "demo04", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 87, "completion_tokens": 211, "reasoning_tokens": null, "latency": 5.523428107476911, "model": "synthetic"}
41
+ {"item": "demo04", "arm": "sample", "idx": 3, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 87, "completion_tokens": 236, "reasoning_tokens": null, "latency": 4.830883104199132, "model": "synthetic"}
42
+ {"item": "demo04", "arm": "sample", "idx": 4, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 87, "completion_tokens": 229, "reasoning_tokens": null, "latency": 4.942732237254404, "model": "synthetic"}
43
+ {"item": "demo04", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 87, "completion_tokens": 185, "reasoning_tokens": null, "latency": 5.026099916623154, "model": "synthetic"}
44
+ {"item": "demo04", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 87, "completion_tokens": 249, "reasoning_tokens": null, "latency": 4.386571349424486, "model": "synthetic"}
45
+ {"item": "demo04", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 87, "completion_tokens": 251, "reasoning_tokens": null, "latency": 2.620521087751379, "model": "synthetic"}
46
+ {"item": "demo05", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 83, "completion_tokens": 175, "reasoning_tokens": null, "latency": 4.881778603470876, "model": "synthetic"}
47
+ {"item": "demo05", "arm": "sample", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 83, "completion_tokens": 198, "reasoning_tokens": null, "latency": 2.3390315789348426, "model": "synthetic"}
48
+ {"item": "demo05", "arm": "sample", "idx": 1, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 83, "completion_tokens": 213, "reasoning_tokens": null, "latency": 2.1584223123545274, "model": "synthetic"}
49
+ {"item": "demo05", "arm": "sample", "idx": 2, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 83, "completion_tokens": 191, "reasoning_tokens": null, "latency": 4.202987805073006, "model": "synthetic"}
50
+ {"item": "demo05", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 83, "completion_tokens": 188, "reasoning_tokens": null, "latency": 2.8887875409090573, "model": "synthetic"}
51
+ {"item": "demo05", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 83, "completion_tokens": 209, "reasoning_tokens": null, "latency": 2.791368430958746, "model": "synthetic"}
52
+ {"item": "demo05", "arm": "sample", "idx": 5, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 83, "completion_tokens": 197, "reasoning_tokens": null, "latency": 4.213836023685194, "model": "synthetic"}
53
+ {"item": "demo05", "arm": "sample", "idx": 6, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 83, "completion_tokens": 179, "reasoning_tokens": null, "latency": 4.481541734957725, "model": "synthetic"}
54
+ {"item": "demo05", "arm": "sample", "idx": 7, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 83, "completion_tokens": 192, "reasoning_tokens": null, "latency": 3.6245148949484642, "model": "synthetic"}
55
+ {"item": "demo06", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 80, "completion_tokens": 187, "reasoning_tokens": null, "latency": 3.170568140543465, "model": "synthetic"}
56
+ {"item": "demo06", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 80, "completion_tokens": 209, "reasoning_tokens": null, "latency": 5.996235021919263, "model": "synthetic"}
57
+ {"item": "demo06", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 80, "completion_tokens": 210, "reasoning_tokens": null, "latency": 3.486510255305811, "model": "synthetic"}
58
+ {"item": "demo06", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 80, "completion_tokens": 253, "reasoning_tokens": null, "latency": 5.556291574874076, "model": "synthetic"}
59
+ {"item": "demo06", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 80, "completion_tokens": 194, "reasoning_tokens": null, "latency": 5.5375944174055975, "model": "synthetic"}
60
+ {"item": "demo06", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 80, "completion_tokens": 230, "reasoning_tokens": null, "latency": 2.293165691318612, "model": "synthetic"}
61
+ {"item": "demo06", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 80, "completion_tokens": 198, "reasoning_tokens": null, "latency": 3.9781078291498764, "model": "synthetic"}
62
+ {"item": "demo06", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 80, "completion_tokens": 208, "reasoning_tokens": null, "latency": 5.312195669187155, "model": "synthetic"}
63
+ {"item": "demo06", "arm": "sample", "idx": 7, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 80, "completion_tokens": 238, "reasoning_tokens": null, "latency": 3.8322674629062923, "model": "synthetic"}
64
+ {"item": "demo07", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 175, "reasoning_tokens": null, "latency": 3.0252495326405837, "model": "synthetic"}
65
+ {"item": "demo07", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 184, "reasoning_tokens": null, "latency": 5.08734524526686, "model": "synthetic"}
66
+ {"item": "demo07", "arm": "sample", "idx": 1, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 166, "reasoning_tokens": null, "latency": 2.5477276101333306, "model": "synthetic"}
67
+ {"item": "demo07", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 210, "reasoning_tokens": null, "latency": 3.9523125434089015, "model": "synthetic"}
68
+ {"item": "demo07", "arm": "sample", "idx": 3, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 199, "reasoning_tokens": null, "latency": 5.070764504189288, "model": "synthetic"}
69
+ {"item": "demo07", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 74, "completion_tokens": 178, "reasoning_tokens": null, "latency": 3.0280336069107205, "model": "synthetic"}
70
+ {"item": "demo07", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 189, "reasoning_tokens": null, "latency": 2.5668910816956765, "model": "synthetic"}
71
+ {"item": "demo07", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 200, "reasoning_tokens": null, "latency": 2.542858019933919, "model": "synthetic"}
72
+ {"item": "demo07", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 165, "reasoning_tokens": null, "latency": 3.1333256214330083, "model": "synthetic"}
73
+ {"item": "demo08", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 89, "completion_tokens": 311, "reasoning_tokens": null, "latency": 5.54478859905633, "model": "synthetic"}
74
+ {"item": "demo08", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 244, "reasoning_tokens": null, "latency": 2.1297767556558402, "model": "synthetic"}
75
+ {"item": "demo08", "arm": "sample", "idx": 1, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 89, "completion_tokens": 307, "reasoning_tokens": null, "latency": 5.959237005503607, "model": "synthetic"}
76
+ {"item": "demo08", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 89, "completion_tokens": 282, "reasoning_tokens": null, "latency": 4.2805937392517315, "model": "synthetic"}
77
+ {"item": "demo08", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 89, "completion_tokens": 275, "reasoning_tokens": null, "latency": 3.3429041567885607, "model": "synthetic"}
78
+ {"item": "demo08", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 89, "completion_tokens": 287, "reasoning_tokens": null, "latency": 4.775684304778134, "model": "synthetic"}
79
+ {"item": "demo08", "arm": "sample", "idx": 5, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 89, "completion_tokens": 301, "reasoning_tokens": null, "latency": 5.022104383401198, "model": "synthetic"}
80
+ {"item": "demo08", "arm": "sample", "idx": 6, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 89, "completion_tokens": 225, "reasoning_tokens": null, "latency": 2.737088917339227, "model": "synthetic"}
81
+ {"item": "demo08", "arm": "sample", "idx": 7, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 89, "completion_tokens": 258, "reasoning_tokens": null, "latency": 5.7119863131603505, "model": "synthetic"}
82
+ {"item": "demo09", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 77, "completion_tokens": 233, "reasoning_tokens": null, "latency": 5.058772253570925, "model": "synthetic"}
83
+ {"item": "demo09", "arm": "sample", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 77, "completion_tokens": 237, "reasoning_tokens": null, "latency": 4.51561551781646, "model": "synthetic"}
84
+ {"item": "demo09", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 294, "reasoning_tokens": null, "latency": 3.171759609991503, "model": "synthetic"}
85
+ {"item": "demo09", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 77, "completion_tokens": 281, "reasoning_tokens": null, "latency": 3.5130464547971467, "model": "synthetic"}
86
+ {"item": "demo09", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 77, "completion_tokens": 277, "reasoning_tokens": null, "latency": 3.549955568410714, "model": "synthetic"}
87
+ {"item": "demo09", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 304, "reasoning_tokens": null, "latency": 4.854401584305874, "model": "synthetic"}
88
+ {"item": "demo09", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 280, "reasoning_tokens": null, "latency": 3.114733454295224, "model": "synthetic"}
89
+ {"item": "demo09", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 77, "completion_tokens": 251, "reasoning_tokens": null, "latency": 3.1520262913609822, "model": "synthetic"}
90
+ {"item": "demo09", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 213, "reasoning_tokens": null, "latency": 2.8780223185508227, "model": "synthetic"}
91
+ {"item": "demo10", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 175, "reasoning_tokens": null, "latency": 3.0597062486997393, "model": "synthetic"}
92
+ {"item": "demo10", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 209, "reasoning_tokens": null, "latency": 3.8126012859668306, "model": "synthetic"}
93
+ {"item": "demo10", "arm": "sample", "idx": 1, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 85, "completion_tokens": 196, "reasoning_tokens": null, "latency": 2.166888234581479, "model": "synthetic"}
94
+ {"item": "demo10", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 182, "reasoning_tokens": null, "latency": 2.327567424790733, "model": "synthetic"}
95
+ {"item": "demo10", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 250, "reasoning_tokens": null, "latency": 4.317630297105515, "model": "synthetic"}
96
+ {"item": "demo10", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 197, "reasoning_tokens": null, "latency": 4.284491427814235, "model": "synthetic"}
97
+ {"item": "demo10", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 219, "reasoning_tokens": null, "latency": 5.211113496749956, "model": "synthetic"}
98
+ {"item": "demo10", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 186, "reasoning_tokens": null, "latency": 4.330905016939177, "model": "synthetic"}
99
+ {"item": "demo10", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 85, "completion_tokens": 183, "reasoning_tokens": null, "latency": 4.476663404641917, "model": "synthetic"}
100
+ {"item": "demo11", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 212, "reasoning_tokens": null, "latency": 3.948117125925486, "model": "synthetic"}
101
+ {"item": "demo11", "arm": "sample", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 269, "reasoning_tokens": null, "latency": 3.750744154190115, "model": "synthetic"}
102
+ {"item": "demo11", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 230, "reasoning_tokens": null, "latency": 5.928372473072792, "model": "synthetic"}
103
+ {"item": "demo11", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 275, "reasoning_tokens": null, "latency": 3.243705855925934, "model": "synthetic"}
104
+ {"item": "demo11", "arm": "sample", "idx": 3, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 197, "reasoning_tokens": null, "latency": 5.485080616067011, "model": "synthetic"}
105
+ {"item": "demo11", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 240, "reasoning_tokens": null, "latency": 4.647022393163377, "model": "synthetic"}
106
+ {"item": "demo11", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 74, "completion_tokens": 209, "reasoning_tokens": null, "latency": 3.1381375716958524, "model": "synthetic"}
107
+ {"item": "demo11", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 217, "reasoning_tokens": null, "latency": 2.520420065101018, "model": "synthetic"}
108
+ {"item": "demo11", "arm": "sample", "idx": 7, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 230, "reasoning_tokens": null, "latency": 4.3473289948018845, "model": "synthetic"}
109
+ {"item": "demo12", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 70, "completion_tokens": 367, "reasoning_tokens": null, "latency": 2.4123766422089385, "model": "synthetic"}
110
+ {"item": "demo12", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 70, "completion_tokens": 294, "reasoning_tokens": null, "latency": 5.751836640804127, "model": "synthetic"}
111
+ {"item": "demo12", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 70, "completion_tokens": 368, "reasoning_tokens": null, "latency": 4.446446314780506, "model": "synthetic"}
112
+ {"item": "demo12", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 70, "completion_tokens": 324, "reasoning_tokens": null, "latency": 3.8351628900730956, "model": "synthetic"}
113
+ {"item": "demo12", "arm": "sample", "idx": 3, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 70, "completion_tokens": 345, "reasoning_tokens": null, "latency": 4.714476388736251, "model": "synthetic"}
114
+ {"item": "demo12", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 70, "completion_tokens": 355, "reasoning_tokens": null, "latency": 3.6443719476182923, "model": "synthetic"}
115
+ {"item": "demo12", "arm": "sample", "idx": 5, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 70, "completion_tokens": 365, "reasoning_tokens": null, "latency": 5.967992751924811, "model": "synthetic"}
116
+ {"item": "demo12", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 70, "completion_tokens": 371, "reasoning_tokens": null, "latency": 5.782953519837548, "model": "synthetic"}
117
+ {"item": "demo12", "arm": "sample", "idx": 7, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 70, "completion_tokens": 371, "reasoning_tokens": null, "latency": 3.6235646211151455, "model": "synthetic"}
118
+ {"item": "demo13", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 67, "completion_tokens": 178, "reasoning_tokens": null, "latency": 4.544164707407859, "model": "synthetic"}
119
+ {"item": "demo13", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 67, "completion_tokens": 186, "reasoning_tokens": null, "latency": 4.311833451423507, "model": "synthetic"}
120
+ {"item": "demo13", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 67, "completion_tokens": 182, "reasoning_tokens": null, "latency": 4.438198255243458, "model": "synthetic"}
121
+ {"item": "demo13", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 67, "completion_tokens": 167, "reasoning_tokens": null, "latency": 4.826327544849711, "model": "synthetic"}
122
+ {"item": "demo13", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 67, "completion_tokens": 230, "reasoning_tokens": null, "latency": 2.8094478326944547, "model": "synthetic"}
123
+ {"item": "demo13", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 67, "completion_tokens": 170, "reasoning_tokens": null, "latency": 5.965995838110324, "model": "synthetic"}
124
+ {"item": "demo13", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 67, "completion_tokens": 202, "reasoning_tokens": null, "latency": 5.962536609774659, "model": "synthetic"}
125
+ {"item": "demo13", "arm": "sample", "idx": 6, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 67, "completion_tokens": 195, "reasoning_tokens": null, "latency": 4.405218705425032, "model": "synthetic"}
126
+ {"item": "demo13", "arm": "sample", "idx": 7, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 67, "completion_tokens": 181, "reasoning_tokens": null, "latency": 3.3116344405232248, "model": "synthetic"}
127
+ {"item": "demo14", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 192, "reasoning_tokens": null, "latency": 2.4017713770317872, "model": "synthetic"}
128
+ {"item": "demo14", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 212, "reasoning_tokens": null, "latency": 3.341809694122881, "model": "synthetic"}
129
+ {"item": "demo14", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 152, "reasoning_tokens": null, "latency": 5.183722087788599, "model": "synthetic"}
130
+ {"item": "demo14", "arm": "sample", "idx": 2, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 89, "completion_tokens": 191, "reasoning_tokens": null, "latency": 2.4015227696157013, "model": "synthetic"}
131
+ {"item": "demo14", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 189, "reasoning_tokens": null, "latency": 5.808927753691734, "model": "synthetic"}
132
+ {"item": "demo14", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 89, "completion_tokens": 193, "reasoning_tokens": null, "latency": 4.075274315295208, "model": "synthetic"}
133
+ {"item": "demo14", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 211, "reasoning_tokens": null, "latency": 2.267416381562293, "model": "synthetic"}
134
+ {"item": "demo14", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 197, "reasoning_tokens": null, "latency": 5.24857592378355, "model": "synthetic"}
135
+ {"item": "demo14", "arm": "sample", "idx": 7, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 89, "completion_tokens": 159, "reasoning_tokens": null, "latency": 2.1412088663196216, "model": "synthetic"}
136
+ {"item": "demo15", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 279, "reasoning_tokens": null, "latency": 4.016418011965586, "model": "synthetic"}
137
+ {"item": "demo15", "arm": "sample", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 328, "reasoning_tokens": null, "latency": 5.795088122935353, "model": "synthetic"}
138
+ {"item": "demo15", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 239, "reasoning_tokens": null, "latency": 3.629954423121935, "model": "synthetic"}
139
+ {"item": "demo15", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 73, "completion_tokens": 303, "reasoning_tokens": null, "latency": 4.304233529487493, "model": "synthetic"}
140
+ {"item": "demo15", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 313, "reasoning_tokens": null, "latency": 2.056083928248398, "model": "synthetic"}
141
+ {"item": "demo15", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 244, "reasoning_tokens": null, "latency": 3.8843659193348747, "model": "synthetic"}
142
+ {"item": "demo15", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 228, "reasoning_tokens": null, "latency": 3.715119319402149, "model": "synthetic"}
143
+ {"item": "demo15", "arm": "sample", "idx": 6, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 228, "reasoning_tokens": null, "latency": 3.010614906004839, "model": "synthetic"}
144
+ {"item": "demo15", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 282, "reasoning_tokens": null, "latency": 3.5259103167154415, "model": "synthetic"}
145
+ {"item": "demo16", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 198, "reasoning_tokens": null, "latency": 2.3903598376246187, "model": "synthetic"}
146
+ {"item": "demo16", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 223, "reasoning_tokens": null, "latency": 5.923104414192764, "model": "synthetic"}
147
+ {"item": "demo16", "arm": "sample", "idx": 1, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 84, "completion_tokens": 164, "reasoning_tokens": null, "latency": 5.880623814806377, "model": "synthetic"}
148
+ {"item": "demo16", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 216, "reasoning_tokens": null, "latency": 5.511494673112157, "model": "synthetic"}
149
+ {"item": "demo16", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 190, "reasoning_tokens": null, "latency": 2.801700745342322, "model": "synthetic"}
150
+ {"item": "demo16", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 173, "reasoning_tokens": null, "latency": 2.503522835261712, "model": "synthetic"}
151
+ {"item": "demo16", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 211, "reasoning_tokens": null, "latency": 5.689613520078053, "model": "synthetic"}
152
+ {"item": "demo16", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 152, "reasoning_tokens": null, "latency": 3.0545308191979292, "model": "synthetic"}
153
+ {"item": "demo16", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 165, "reasoning_tokens": null, "latency": 4.333307597657751, "model": "synthetic"}
154
+ {"item": "demo17", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 205, "reasoning_tokens": null, "latency": 2.8534571550060233, "model": "synthetic"}
155
+ {"item": "demo17", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 84, "completion_tokens": 235, "reasoning_tokens": null, "latency": 4.369751708164699, "model": "synthetic"}
156
+ {"item": "demo17", "arm": "sample", "idx": 1, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 84, "completion_tokens": 200, "reasoning_tokens": null, "latency": 5.243824310423831, "model": "synthetic"}
157
+ {"item": "demo17", "arm": "sample", "idx": 2, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 84, "completion_tokens": 203, "reasoning_tokens": null, "latency": 4.167645241297446, "model": "synthetic"}
158
+ {"item": "demo17", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 84, "completion_tokens": 231, "reasoning_tokens": null, "latency": 5.455453550347517, "model": "synthetic"}
159
+ {"item": "demo17", "arm": "sample", "idx": 4, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 84, "completion_tokens": 201, "reasoning_tokens": null, "latency": 4.25420319820895, "model": "synthetic"}
160
+ {"item": "demo17", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 84, "completion_tokens": 260, "reasoning_tokens": null, "latency": 3.0279233398180083, "model": "synthetic"}
161
+ {"item": "demo17", "arm": "sample", "idx": 6, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 84, "completion_tokens": 230, "reasoning_tokens": null, "latency": 3.6596341028857386, "model": "synthetic"}
162
+ {"item": "demo17", "arm": "sample", "idx": 7, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 84, "completion_tokens": 207, "reasoning_tokens": null, "latency": 4.5585096504795874, "model": "synthetic"}
163
+ {"item": "demo18", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 250, "reasoning_tokens": null, "latency": 5.746146505367481, "model": "synthetic"}
164
+ {"item": "demo18", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 228, "reasoning_tokens": null, "latency": 2.0235966783334973, "model": "synthetic"}
165
+ {"item": "demo18", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 262, "reasoning_tokens": null, "latency": 2.4753542033959794, "model": "synthetic"}
166
+ {"item": "demo18", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 79, "completion_tokens": 203, "reasoning_tokens": null, "latency": 5.112643785087821, "model": "synthetic"}
167
+ {"item": "demo18", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 257, "reasoning_tokens": null, "latency": 2.331036628200375, "model": "synthetic"}
168
+ {"item": "demo18", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 244, "reasoning_tokens": null, "latency": 3.7967132102574683, "model": "synthetic"}
169
+ {"item": "demo18", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 223, "reasoning_tokens": null, "latency": 3.1878764946205522, "model": "synthetic"}
170
+ {"item": "demo18", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 243, "reasoning_tokens": null, "latency": 5.427357438124086, "model": "synthetic"}
171
+ {"item": "demo18", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 230, "reasoning_tokens": null, "latency": 5.934663327632752, "model": "synthetic"}
172
+ {"item": "demo19", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 79, "completion_tokens": 248, "reasoning_tokens": null, "latency": 3.1594232486290945, "model": "synthetic"}
173
+ {"item": "demo19", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 230, "reasoning_tokens": null, "latency": 5.5206655901047155, "model": "synthetic"}
174
+ {"item": "demo19", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 192, "reasoning_tokens": null, "latency": 3.760308191356607, "model": "synthetic"}
175
+ {"item": "demo19", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 241, "reasoning_tokens": null, "latency": 4.348506920403214, "model": "synthetic"}
176
+ {"item": "demo19", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 79, "completion_tokens": 225, "reasoning_tokens": null, "latency": 4.454055643906763, "model": "synthetic"}
177
+ {"item": "demo19", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 79, "completion_tokens": 253, "reasoning_tokens": null, "latency": 3.88354175514931, "model": "synthetic"}
178
+ {"item": "demo19", "arm": "sample", "idx": 5, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 79, "completion_tokens": 277, "reasoning_tokens": null, "latency": 3.129997791283027, "model": "synthetic"}
179
+ {"item": "demo19", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 264, "reasoning_tokens": null, "latency": 3.1891960566695725, "model": "synthetic"}
180
+ {"item": "demo19", "arm": "sample", "idx": 7, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 79, "completion_tokens": 244, "reasoning_tokens": null, "latency": 4.035837249457677, "model": "synthetic"}
181
+ {"item": "demo20", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 89, "completion_tokens": 227, "reasoning_tokens": null, "latency": 4.223733909367619, "model": "synthetic"}
182
+ {"item": "demo20", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 293, "reasoning_tokens": null, "latency": 2.339195075448495, "model": "synthetic"}
183
+ {"item": "demo20", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 209, "reasoning_tokens": null, "latency": 4.847556802582134, "model": "synthetic"}
184
+ {"item": "demo20", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 291, "reasoning_tokens": null, "latency": 5.5745257060003635, "model": "synthetic"}
185
+ {"item": "demo20", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 89, "completion_tokens": 226, "reasoning_tokens": null, "latency": 2.0026270334233653, "model": "synthetic"}
186
+ {"item": "demo20", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 297, "reasoning_tokens": null, "latency": 5.054967701753563, "model": "synthetic"}
187
+ {"item": "demo20", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 271, "reasoning_tokens": null, "latency": 5.668254131177292, "model": "synthetic"}
188
+ {"item": "demo20", "arm": "sample", "idx": 6, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 89, "completion_tokens": 287, "reasoning_tokens": null, "latency": 5.366808411599722, "model": "synthetic"}
189
+ {"item": "demo20", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 89, "completion_tokens": 210, "reasoning_tokens": null, "latency": 4.909856669015719, "model": "synthetic"}
190
+ {"item": "demo21", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 81, "completion_tokens": 250, "reasoning_tokens": null, "latency": 4.945354726267845, "model": "synthetic"}
191
+ {"item": "demo21", "arm": "sample", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 81, "completion_tokens": 261, "reasoning_tokens": null, "latency": 2.185622860391285, "model": "synthetic"}
192
+ {"item": "demo21", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 81, "completion_tokens": 289, "reasoning_tokens": null, "latency": 2.4928820047778535, "model": "synthetic"}
193
+ {"item": "demo21", "arm": "sample", "idx": 2, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 81, "completion_tokens": 293, "reasoning_tokens": null, "latency": 2.6862538315052373, "model": "synthetic"}
194
+ {"item": "demo21", "arm": "sample", "idx": 3, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 81, "completion_tokens": 254, "reasoning_tokens": null, "latency": 4.874606803725363, "model": "synthetic"}
195
+ {"item": "demo21", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 81, "completion_tokens": 234, "reasoning_tokens": null, "latency": 2.3013584426209217, "model": "synthetic"}
196
+ {"item": "demo21", "arm": "sample", "idx": 5, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 81, "completion_tokens": 297, "reasoning_tokens": null, "latency": 2.0680641880461685, "model": "synthetic"}
197
+ {"item": "demo21", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 81, "completion_tokens": 245, "reasoning_tokens": null, "latency": 4.090546700579566, "model": "synthetic"}
198
+ {"item": "demo21", "arm": "sample", "idx": 7, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 81, "completion_tokens": 298, "reasoning_tokens": null, "latency": 4.749608437055091, "model": "synthetic"}
199
+ {"item": "demo22", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 82, "completion_tokens": 222, "reasoning_tokens": null, "latency": 4.540487242964195, "model": "synthetic"}
200
+ {"item": "demo22", "arm": "sample", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 82, "completion_tokens": 185, "reasoning_tokens": null, "latency": 4.832190602822619, "model": "synthetic"}
201
+ {"item": "demo22", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 82, "completion_tokens": 199, "reasoning_tokens": null, "latency": 4.725961849351187, "model": "synthetic"}
202
+ {"item": "demo22", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 82, "completion_tokens": 164, "reasoning_tokens": null, "latency": 5.163792448052435, "model": "synthetic"}
203
+ {"item": "demo22", "arm": "sample", "idx": 3, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 82, "completion_tokens": 184, "reasoning_tokens": null, "latency": 4.632692022262855, "model": "synthetic"}
204
+ {"item": "demo22", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 82, "completion_tokens": 211, "reasoning_tokens": null, "latency": 4.304794200508085, "model": "synthetic"}
205
+ {"item": "demo22", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 82, "completion_tokens": 194, "reasoning_tokens": null, "latency": 2.90063327595514, "model": "synthetic"}
206
+ {"item": "demo22", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 82, "completion_tokens": 218, "reasoning_tokens": null, "latency": 3.456720369295267, "model": "synthetic"}
207
+ {"item": "demo22", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 82, "completion_tokens": 202, "reasoning_tokens": null, "latency": 4.491045437497597, "model": "synthetic"}
208
+ {"item": "demo23", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 246, "reasoning_tokens": null, "latency": 4.62760441743564, "model": "synthetic"}
209
+ {"item": "demo23", "arm": "sample", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 73, "completion_tokens": 279, "reasoning_tokens": null, "latency": 2.420271147974388, "model": "synthetic"}
210
+ {"item": "demo23", "arm": "sample", "idx": 1, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 343, "reasoning_tokens": null, "latency": 2.4898272653043123, "model": "synthetic"}
211
+ {"item": "demo23", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 296, "reasoning_tokens": null, "latency": 2.47149122423152, "model": "synthetic"}
212
+ {"item": "demo23", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 270, "reasoning_tokens": null, "latency": 4.8532016336488635, "model": "synthetic"}
213
+ {"item": "demo23", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 73, "completion_tokens": 280, "reasoning_tokens": null, "latency": 5.162877919870899, "model": "synthetic"}
214
+ {"item": "demo23", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 306, "reasoning_tokens": null, "latency": 3.730852999137529, "model": "synthetic"}
215
+ {"item": "demo23", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 335, "reasoning_tokens": null, "latency": 4.985687033613763, "model": "synthetic"}
216
+ {"item": "demo23", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 272, "reasoning_tokens": null, "latency": 5.27801952613229, "model": "synthetic"}
217
+ {"item": "demo24", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 66, "completion_tokens": 197, "reasoning_tokens": null, "latency": 3.851129745256541, "model": "synthetic"}
218
+ {"item": "demo24", "arm": "sample", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 66, "completion_tokens": 264, "reasoning_tokens": null, "latency": 5.2689841320732915, "model": "synthetic"}
219
+ {"item": "demo24", "arm": "sample", "idx": 1, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 66, "completion_tokens": 249, "reasoning_tokens": null, "latency": 2.7482077260883746, "model": "synthetic"}
220
+ {"item": "demo24", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 66, "completion_tokens": 257, "reasoning_tokens": null, "latency": 4.987466343843749, "model": "synthetic"}
221
+ {"item": "demo24", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 66, "completion_tokens": 216, "reasoning_tokens": null, "latency": 4.689802752492958, "model": "synthetic"}
222
+ {"item": "demo24", "arm": "sample", "idx": 4, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 66, "completion_tokens": 220, "reasoning_tokens": null, "latency": 3.307357621538, "model": "synthetic"}
223
+ {"item": "demo24", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 66, "completion_tokens": 235, "reasoning_tokens": null, "latency": 3.474869992392624, "model": "synthetic"}
224
+ {"item": "demo24", "arm": "sample", "idx": 6, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 66, "completion_tokens": 253, "reasoning_tokens": null, "latency": 4.7209931708429735, "model": "synthetic"}
225
+ {"item": "demo24", "arm": "sample", "idx": 7, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 66, "completion_tokens": 209, "reasoning_tokens": null, "latency": 4.031581714879189, "model": "synthetic"}
226
+ {"item": "demo25", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 237, "reasoning_tokens": null, "latency": 2.4248125752853027, "model": "synthetic"}
227
+ {"item": "demo25", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 253, "reasoning_tokens": null, "latency": 3.6285935267752056, "model": "synthetic"}
228
+ {"item": "demo25", "arm": "sample", "idx": 1, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 77, "completion_tokens": 322, "reasoning_tokens": null, "latency": 4.883552632318613, "model": "synthetic"}
229
+ {"item": "demo25", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 275, "reasoning_tokens": null, "latency": 2.6374820963609475, "model": "synthetic"}
230
+ {"item": "demo25", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 317, "reasoning_tokens": null, "latency": 3.351416043829167, "model": "synthetic"}
231
+ {"item": "demo25", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 226, "reasoning_tokens": null, "latency": 4.60856293024013, "model": "synthetic"}
232
+ {"item": "demo25", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 217, "reasoning_tokens": null, "latency": 4.8607253348943935, "model": "synthetic"}
233
+ {"item": "demo25", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 300, "reasoning_tokens": null, "latency": 3.954423091075537, "model": "synthetic"}
234
+ {"item": "demo25", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 265, "reasoning_tokens": null, "latency": 4.703927644754726, "model": "synthetic"}
235
+ {"item": "demo26", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 76, "completion_tokens": 276, "reasoning_tokens": null, "latency": 2.4210484138249337, "model": "synthetic"}
236
+ {"item": "demo26", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 76, "completion_tokens": 234, "reasoning_tokens": null, "latency": 3.126750821042609, "model": "synthetic"}
237
+ {"item": "demo26", "arm": "sample", "idx": 1, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 76, "completion_tokens": 272, "reasoning_tokens": null, "latency": 2.132185814778222, "model": "synthetic"}
238
+ {"item": "demo26", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 76, "completion_tokens": 284, "reasoning_tokens": null, "latency": 5.697360307102267, "model": "synthetic"}
239
+ {"item": "demo26", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 76, "completion_tokens": 238, "reasoning_tokens": null, "latency": 3.4204733137644876, "model": "synthetic"}
240
+ {"item": "demo26", "arm": "sample", "idx": 4, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 76, "completion_tokens": 291, "reasoning_tokens": null, "latency": 3.3633254620039064, "model": "synthetic"}
241
+ {"item": "demo26", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 76, "completion_tokens": 203, "reasoning_tokens": null, "latency": 2.549883389853859, "model": "synthetic"}
242
+ {"item": "demo26", "arm": "sample", "idx": 6, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 76, "completion_tokens": 303, "reasoning_tokens": null, "latency": 2.277847163331542, "model": "synthetic"}
243
+ {"item": "demo26", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 76, "completion_tokens": 284, "reasoning_tokens": null, "latency": 4.644825459031685, "model": "synthetic"}
244
+ {"item": "demo27", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 261, "reasoning_tokens": null, "latency": 3.0692690973032875, "model": "synthetic"}
245
+ {"item": "demo27", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 238, "reasoning_tokens": null, "latency": 5.433442848672744, "model": "synthetic"}
246
+ {"item": "demo27", "arm": "sample", "idx": 1, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 78, "completion_tokens": 248, "reasoning_tokens": null, "latency": 2.0979989570130613, "model": "synthetic"}
247
+ {"item": "demo27", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 78, "completion_tokens": 284, "reasoning_tokens": null, "latency": 5.432622100452635, "model": "synthetic"}
248
+ {"item": "demo27", "arm": "sample", "idx": 3, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 78, "completion_tokens": 306, "reasoning_tokens": null, "latency": 3.956537814630016, "model": "synthetic"}
249
+ {"item": "demo27", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 78, "completion_tokens": 242, "reasoning_tokens": null, "latency": 3.9238661002503203, "model": "synthetic"}
250
+ {"item": "demo27", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 296, "reasoning_tokens": null, "latency": 5.32683539831839, "model": "synthetic"}
251
+ {"item": "demo27", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 232, "reasoning_tokens": null, "latency": 3.534645241338613, "model": "synthetic"}
252
+ {"item": "demo27", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 316, "reasoning_tokens": null, "latency": 5.766768398348519, "model": "synthetic"}
253
+ {"item": "demo28", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 235, "reasoning_tokens": null, "latency": 3.51717409645332, "model": "synthetic"}
254
+ {"item": "demo28", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 219, "reasoning_tokens": null, "latency": 3.625505462076282, "model": "synthetic"}
255
+ {"item": "demo28", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 243, "reasoning_tokens": null, "latency": 3.5372115806301863, "model": "synthetic"}
256
+ {"item": "demo28", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 252, "reasoning_tokens": null, "latency": 4.476123891915574, "model": "synthetic"}
257
+ {"item": "demo28", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 233, "reasoning_tokens": null, "latency": 5.1810386247419284, "model": "synthetic"}
258
+ {"item": "demo28", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 288, "reasoning_tokens": null, "latency": 5.008551815880162, "model": "synthetic"}
259
+ {"item": "demo28", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 299, "reasoning_tokens": null, "latency": 4.676060919900207, "model": "synthetic"}
260
+ {"item": "demo28", "arm": "sample", "idx": 6, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 73, "completion_tokens": 287, "reasoning_tokens": null, "latency": 3.0178312643965524, "model": "synthetic"}
261
+ {"item": "demo28", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 73, "completion_tokens": 225, "reasoning_tokens": null, "latency": 3.5876361051530865, "model": "synthetic"}
262
+ {"item": "demo29", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 64, "completion_tokens": 250, "reasoning_tokens": null, "latency": 3.089276308911594, "model": "synthetic"}
263
+ {"item": "demo29", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 64, "completion_tokens": 244, "reasoning_tokens": null, "latency": 4.071078674189911, "model": "synthetic"}
264
+ {"item": "demo29", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 64, "completion_tokens": 220, "reasoning_tokens": null, "latency": 3.4543517190392845, "model": "synthetic"}
265
+ {"item": "demo29", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 64, "completion_tokens": 198, "reasoning_tokens": null, "latency": 3.8088394211688335, "model": "synthetic"}
266
+ {"item": "demo29", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 64, "completion_tokens": 195, "reasoning_tokens": null, "latency": 5.908120145068382, "model": "synthetic"}
267
+ {"item": "demo29", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 64, "completion_tokens": 246, "reasoning_tokens": null, "latency": 3.4327990744906436, "model": "synthetic"}
268
+ {"item": "demo29", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 64, "completion_tokens": 281, "reasoning_tokens": null, "latency": 5.742629685683411, "model": "synthetic"}
269
+ {"item": "demo29", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 64, "completion_tokens": 235, "reasoning_tokens": null, "latency": 3.494562987882459, "model": "synthetic"}
270
+ {"item": "demo29", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 64, "completion_tokens": 258, "reasoning_tokens": null, "latency": 5.127984244609549, "model": "synthetic"}
271
+ {"item": "demo30", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 73, "completion_tokens": 257, "reasoning_tokens": null, "latency": 4.92543554958014, "model": "synthetic"}
272
+ {"item": "demo30", "arm": "sample", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 245, "reasoning_tokens": null, "latency": 4.344565040747115, "model": "synthetic"}
273
+ {"item": "demo30", "arm": "sample", "idx": 1, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 73, "completion_tokens": 290, "reasoning_tokens": null, "latency": 5.1253376055972435, "model": "synthetic"}
274
+ {"item": "demo30", "arm": "sample", "idx": 2, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 73, "completion_tokens": 291, "reasoning_tokens": null, "latency": 3.818352514063319, "model": "synthetic"}
275
+ {"item": "demo30", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 253, "reasoning_tokens": null, "latency": 3.181242109395375, "model": "synthetic"}
276
+ {"item": "demo30", "arm": "sample", "idx": 4, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 73, "completion_tokens": 326, "reasoning_tokens": null, "latency": 2.456151435262535, "model": "synthetic"}
277
+ {"item": "demo30", "arm": "sample", "idx": 5, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 73, "completion_tokens": 296, "reasoning_tokens": null, "latency": 3.250165847795019, "model": "synthetic"}
278
+ {"item": "demo30", "arm": "sample", "idx": 6, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 73, "completion_tokens": 293, "reasoning_tokens": null, "latency": 4.697340430499844, "model": "synthetic"}
279
+ {"item": "demo30", "arm": "sample", "idx": 7, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 73, "completion_tokens": 334, "reasoning_tokens": null, "latency": 3.719969602138407, "model": "synthetic"}
280
+ {"item": "demo31", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 65, "completion_tokens": 151, "reasoning_tokens": null, "latency": 5.842136895530267, "model": "synthetic"}
281
+ {"item": "demo31", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 65, "completion_tokens": 171, "reasoning_tokens": null, "latency": 2.8131692066773444, "model": "synthetic"}
282
+ {"item": "demo31", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 65, "completion_tokens": 183, "reasoning_tokens": null, "latency": 2.3041992200233508, "model": "synthetic"}
283
+ {"item": "demo31", "arm": "sample", "idx": 2, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 65, "completion_tokens": 174, "reasoning_tokens": null, "latency": 5.366256296938438, "model": "synthetic"}
284
+ {"item": "demo31", "arm": "sample", "idx": 3, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 65, "completion_tokens": 171, "reasoning_tokens": null, "latency": 5.388912671442141, "model": "synthetic"}
285
+ {"item": "demo31", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 65, "completion_tokens": 173, "reasoning_tokens": null, "latency": 5.602344192943541, "model": "synthetic"}
286
+ {"item": "demo31", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 65, "completion_tokens": 172, "reasoning_tokens": null, "latency": 2.345167689831044, "model": "synthetic"}
287
+ {"item": "demo31", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 65, "completion_tokens": 164, "reasoning_tokens": null, "latency": 4.267708853536691, "model": "synthetic"}
288
+ {"item": "demo31", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 65, "completion_tokens": 214, "reasoning_tokens": null, "latency": 3.3684588643757802, "model": "synthetic"}
289
+ {"item": "demo32", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 79, "completion_tokens": 271, "reasoning_tokens": null, "latency": 3.4400663793377086, "model": "synthetic"}
290
+ {"item": "demo32", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 234, "reasoning_tokens": null, "latency": 3.1231309064081407, "model": "synthetic"}
291
+ {"item": "demo32", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 278, "reasoning_tokens": null, "latency": 3.365845842424471, "model": "synthetic"}
292
+ {"item": "demo32", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 276, "reasoning_tokens": null, "latency": 3.190375483136129, "model": "synthetic"}
293
+ {"item": "demo32", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 198, "reasoning_tokens": null, "latency": 4.933322072086202, "model": "synthetic"}
294
+ {"item": "demo32", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 266, "reasoning_tokens": null, "latency": 5.90021205414538, "model": "synthetic"}
295
+ {"item": "demo32", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 279, "reasoning_tokens": null, "latency": 3.1134133389142606, "model": "synthetic"}
296
+ {"item": "demo32", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 208, "reasoning_tokens": null, "latency": 2.3341149178925846, "model": "synthetic"}
297
+ {"item": "demo32", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 255, "reasoning_tokens": null, "latency": 5.113787457472665, "model": "synthetic"}
298
+ {"item": "demo33", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 305, "reasoning_tokens": null, "latency": 5.86641836655787, "model": "synthetic"}
299
+ {"item": "demo33", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 300, "reasoning_tokens": null, "latency": 4.12595538389253, "model": "synthetic"}
300
+ {"item": "demo33", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 287, "reasoning_tokens": null, "latency": 3.7318603536670514, "model": "synthetic"}
301
+ {"item": "demo33", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 288, "reasoning_tokens": null, "latency": 4.1521585532704135, "model": "synthetic"}
302
+ {"item": "demo33", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 287, "reasoning_tokens": null, "latency": 4.776153298866621, "model": "synthetic"}
303
+ {"item": "demo33", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 78, "completion_tokens": 284, "reasoning_tokens": null, "latency": 5.49247051154671, "model": "synthetic"}
304
+ {"item": "demo33", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 245, "reasoning_tokens": null, "latency": 3.335949874310595, "model": "synthetic"}
305
+ {"item": "demo33", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 227, "reasoning_tokens": null, "latency": 3.3261453765368425, "model": "synthetic"}
306
+ {"item": "demo33", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 78, "completion_tokens": 282, "reasoning_tokens": null, "latency": 3.7078104676767842, "model": "synthetic"}
307
+ {"item": "demo34", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 68, "completion_tokens": 346, "reasoning_tokens": null, "latency": 4.565787127895659, "model": "synthetic"}
308
+ {"item": "demo34", "arm": "sample", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 68, "completion_tokens": 305, "reasoning_tokens": null, "latency": 4.219187588626049, "model": "synthetic"}
309
+ {"item": "demo34", "arm": "sample", "idx": 1, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 68, "completion_tokens": 335, "reasoning_tokens": null, "latency": 4.948781961794102, "model": "synthetic"}
310
+ {"item": "demo34", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 68, "completion_tokens": 329, "reasoning_tokens": null, "latency": 3.8544666080798113, "model": "synthetic"}
311
+ {"item": "demo34", "arm": "sample", "idx": 3, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 68, "completion_tokens": 272, "reasoning_tokens": null, "latency": 3.869353114432884, "model": "synthetic"}
312
+ {"item": "demo34", "arm": "sample", "idx": 4, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 68, "completion_tokens": 272, "reasoning_tokens": null, "latency": 3.9538471794314063, "model": "synthetic"}
313
+ {"item": "demo34", "arm": "sample", "idx": 5, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 68, "completion_tokens": 269, "reasoning_tokens": null, "latency": 5.2922976174810055, "model": "synthetic"}
314
+ {"item": "demo34", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 68, "completion_tokens": 273, "reasoning_tokens": null, "latency": 4.215272364837169, "model": "synthetic"}
315
+ {"item": "demo34", "arm": "sample", "idx": 7, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 68, "completion_tokens": 342, "reasoning_tokens": null, "latency": 3.0006470006488173, "model": "synthetic"}
316
+ {"item": "demo35", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 86, "completion_tokens": 276, "reasoning_tokens": null, "latency": 2.705553978549754, "model": "synthetic"}
317
+ {"item": "demo35", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 86, "completion_tokens": 252, "reasoning_tokens": null, "latency": 2.64779856667057, "model": "synthetic"}
318
+ {"item": "demo35", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 86, "completion_tokens": 233, "reasoning_tokens": null, "latency": 3.8814165552968816, "model": "synthetic"}
319
+ {"item": "demo35", "arm": "sample", "idx": 2, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 86, "completion_tokens": 300, "reasoning_tokens": null, "latency": 5.901034946655157, "model": "synthetic"}
320
+ {"item": "demo35", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 86, "completion_tokens": 328, "reasoning_tokens": null, "latency": 3.777786759465124, "model": "synthetic"}
321
+ {"item": "demo35", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 86, "completion_tokens": 258, "reasoning_tokens": null, "latency": 4.106811351273927, "model": "synthetic"}
322
+ {"item": "demo35", "arm": "sample", "idx": 5, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 86, "completion_tokens": 283, "reasoning_tokens": null, "latency": 5.149230367900309, "model": "synthetic"}
323
+ {"item": "demo35", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 86, "completion_tokens": 320, "reasoning_tokens": null, "latency": 2.074182980500665, "model": "synthetic"}
324
+ {"item": "demo35", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 86, "completion_tokens": 246, "reasoning_tokens": null, "latency": 3.029401756039459, "model": "synthetic"}
325
+ {"item": "demo36", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 77, "completion_tokens": 277, "reasoning_tokens": null, "latency": 5.188710379552245, "model": "synthetic"}
326
+ {"item": "demo36", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 305, "reasoning_tokens": null, "latency": 3.5290065887127477, "model": "synthetic"}
327
+ {"item": "demo36", "arm": "sample", "idx": 1, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 77, "completion_tokens": 267, "reasoning_tokens": null, "latency": 5.27230887685041, "model": "synthetic"}
328
+ {"item": "demo36", "arm": "sample", "idx": 2, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 77, "completion_tokens": 281, "reasoning_tokens": null, "latency": 2.630405792122826, "model": "synthetic"}
329
+ {"item": "demo36", "arm": "sample", "idx": 3, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 77, "completion_tokens": 218, "reasoning_tokens": null, "latency": 2.532764077854469, "model": "synthetic"}
330
+ {"item": "demo36", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 77, "completion_tokens": 238, "reasoning_tokens": null, "latency": 5.0612914833287, "model": "synthetic"}
331
+ {"item": "demo36", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 211, "reasoning_tokens": null, "latency": 5.9879185938081605, "model": "synthetic"}
332
+ {"item": "demo36", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 264, "reasoning_tokens": null, "latency": 2.8401884974511415, "model": "synthetic"}
333
+ {"item": "demo36", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 77, "completion_tokens": 252, "reasoning_tokens": null, "latency": 5.159196283334508, "model": "synthetic"}
334
+ {"item": "demo37", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 235, "reasoning_tokens": null, "latency": 5.065401102583835, "model": "synthetic"}
335
+ {"item": "demo37", "arm": "sample", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 79, "completion_tokens": 295, "reasoning_tokens": null, "latency": 4.4262035861793345, "model": "synthetic"}
336
+ {"item": "demo37", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 291, "reasoning_tokens": null, "latency": 2.4637514287257822, "model": "synthetic"}
337
+ {"item": "demo37", "arm": "sample", "idx": 2, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 264, "reasoning_tokens": null, "latency": 2.4577640692466294, "model": "synthetic"}
338
+ {"item": "demo37", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 286, "reasoning_tokens": null, "latency": 4.967829965875483, "model": "synthetic"}
339
+ {"item": "demo37", "arm": "sample", "idx": 4, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 258, "reasoning_tokens": null, "latency": 2.830404004309891, "model": "synthetic"}
340
+ {"item": "demo37", "arm": "sample", "idx": 5, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 79, "completion_tokens": 278, "reasoning_tokens": null, "latency": 5.734426050638996, "model": "synthetic"}
341
+ {"item": "demo37", "arm": "sample", "idx": 6, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 314, "reasoning_tokens": null, "latency": 5.962590682516947, "model": "synthetic"}
342
+ {"item": "demo37", "arm": "sample", "idx": 7, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 79, "completion_tokens": 294, "reasoning_tokens": null, "latency": 5.299378589603273, "model": "synthetic"}
343
+ {"item": "demo38", "arm": "greedy", "idx": 0, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 71, "completion_tokens": 222, "reasoning_tokens": null, "latency": 4.312342186678515, "model": "synthetic"}
344
+ {"item": "demo38", "arm": "sample", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 71, "completion_tokens": 198, "reasoning_tokens": null, "latency": 4.940326638770896, "model": "synthetic"}
345
+ {"item": "demo38", "arm": "sample", "idx": 1, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 71, "completion_tokens": 195, "reasoning_tokens": null, "latency": 3.8790712477659506, "model": "synthetic"}
346
+ {"item": "demo38", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 71, "completion_tokens": 173, "reasoning_tokens": null, "latency": 2.100297110966435, "model": "synthetic"}
347
+ {"item": "demo38", "arm": "sample", "idx": 3, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 71, "completion_tokens": 170, "reasoning_tokens": null, "latency": 5.428978823267192, "model": "synthetic"}
348
+ {"item": "demo38", "arm": "sample", "idx": 4, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 71, "completion_tokens": 191, "reasoning_tokens": null, "latency": 2.41108465645758, "model": "synthetic"}
349
+ {"item": "demo38", "arm": "sample", "idx": 5, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 71, "completion_tokens": 224, "reasoning_tokens": null, "latency": 4.036619754200008, "model": "synthetic"}
350
+ {"item": "demo38", "arm": "sample", "idx": 6, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 71, "completion_tokens": 186, "reasoning_tokens": null, "latency": 2.60034681125615, "model": "synthetic"}
351
+ {"item": "demo38", "arm": "sample", "idx": 7, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 71, "completion_tokens": 183, "reasoning_tokens": null, "latency": 3.034078310063446, "model": "synthetic"}
352
+ {"item": "demo39", "arm": "greedy", "idx": 0, "gold": "G", "answer": "G", "correct": true, "prompt_tokens": 74, "completion_tokens": 193, "reasoning_tokens": null, "latency": 4.299888285789019, "model": "synthetic"}
353
+ {"item": "demo39", "arm": "sample", "idx": 0, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 185, "reasoning_tokens": null, "latency": 5.842550647026637, "model": "synthetic"}
354
+ {"item": "demo39", "arm": "sample", "idx": 1, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 74, "completion_tokens": 237, "reasoning_tokens": null, "latency": 3.9420873534707632, "model": "synthetic"}
355
+ {"item": "demo39", "arm": "sample", "idx": 2, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 181, "reasoning_tokens": null, "latency": 2.3516019146214524, "model": "synthetic"}
356
+ {"item": "demo39", "arm": "sample", "idx": 3, "gold": "G", "answer": "w1", "correct": false, "prompt_tokens": 74, "completion_tokens": 251, "reasoning_tokens": null, "latency": 3.459957270250817, "model": "synthetic"}
357
+ {"item": "demo39", "arm": "sample", "idx": 4, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 198, "reasoning_tokens": null, "latency": 5.759530879774289, "model": "synthetic"}
358
+ {"item": "demo39", "arm": "sample", "idx": 5, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 247, "reasoning_tokens": null, "latency": 4.814764284626622, "model": "synthetic"}
359
+ {"item": "demo39", "arm": "sample", "idx": 6, "gold": "G", "answer": "w0", "correct": false, "prompt_tokens": 74, "completion_tokens": 245, "reasoning_tokens": null, "latency": 4.146656373152341, "model": "synthetic"}
360
+ {"item": "demo39", "arm": "sample", "idx": 7, "gold": "G", "answer": "w2", "correct": false, "prompt_tokens": 74, "completion_tokens": 247, "reasoning_tokens": null, "latency": 5.604392857788792, "model": "synthetic"}