skilldrop-cli 0.16.36 → 0.16.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +1 -1
  2. package/catalogue.json +3 -1
  3. package/model-routing.json +8 -0
  4. package/package.json +1 -1
  5. package/packs/ai-engineering/skills/judge-audit/SKILL.md +157 -0
  6. package/packs/ai-engineering/skills/judge-audit/evals/eval_queries.json +11 -0
  7. package/packs/ai-engineering/skills/judge-audit/evals/evals.json +136 -0
  8. package/packs/ai-engineering/skills/judge-audit/evals/files/1/judgements.csv +321 -0
  9. package/packs/ai-engineering/skills/judge-audit/evals/files/1/notes.md +28 -0
  10. package/packs/ai-engineering/skills/judge-audit/evals/files/2/pairwise.csv +101 -0
  11. package/packs/ai-engineering/skills/judge-audit/evals/files/2/run-notes.md +16 -0
  12. package/packs/ai-engineering/skills/judge-audit/evals/files/4/grades.csv +97 -0
  13. package/packs/ai-engineering/skills/judge-audit/evals/files/5/pairwise_judgements.jsonl +120 -0
  14. package/packs/ai-engineering/skills/judge-audit/evals/files/6/rag-judge/README.md +36 -0
  15. package/packs/ai-engineering/skills/judge-audit/evals/files/6/rag-judge/judgments.csv +301 -0
  16. package/packs/ai-engineering/skills/judge-audit/evals/files/7/moderation/gold_labels.csv +101 -0
  17. package/packs/ai-engineering/skills/judge-audit/evals/files/7/moderation/run_1.jsonl +100 -0
  18. package/packs/ai-engineering/skills/judge-audit/evals/files/7/moderation/run_2.jsonl +100 -0
  19. package/packs/ai-engineering/skills/judge-audit/evals/files/8/patches.csv +241 -0
  20. package/packs/ai-engineering/skills/judge-audit/evals/files/9/adequacy_ratings.csv +151 -0
  21. package/packs/ai-engineering/skills/judge-audit/examples/summary-panel.md +70 -0
  22. package/packs/ai-engineering/skills/judge-audit/manifest.json +44 -0
  23. package/packs/ai-engineering/skills/judge-audit/reference.md +176 -0
  24. package/packs/ai-engineering/skills/judge-audit/scripts/judge_stats.py +1735 -0
  25. package/packs/frontend/pack.json +9 -3
  26. package/packs/frontend/skills/bundle-size-review/SKILL.md +114 -0
  27. package/packs/frontend/skills/bundle-size-review/evals/eval_queries.json +12 -0
  28. package/packs/frontend/skills/bundle-size-review/evals/evals.json +108 -0
  29. package/packs/frontend/skills/bundle-size-review/evals/files/1/package.json +28 -0
  30. package/packs/frontend/skills/bundle-size-review/evals/files/1/src/utils/format.js +26 -0
  31. package/packs/frontend/skills/bundle-size-review/evals/files/1/stats.json +1334 -0
  32. package/packs/frontend/skills/bundle-size-review/evals/files/2/PR.md +21 -0
  33. package/packs/frontend/skills/bundle-size-review/evals/files/2/meta-after.json +440 -0
  34. package/packs/frontend/skills/bundle-size-review/evals/files/2/meta-before.json +360 -0
  35. package/packs/frontend/skills/bundle-size-review/evals/files/3/package.json +29 -0
  36. package/packs/frontend/skills/bundle-size-review/evals/files/3/stats.json +1583 -0
  37. package/packs/frontend/skills/bundle-size-review/evals/files/4/src/components/CommentBox.jsx +24 -0
  38. package/packs/frontend/skills/bundle-size-review/evals/files/4/stats/after.json +299 -0
  39. package/packs/frontend/skills/bundle-size-review/evals/files/4/stats/before.json +275 -0
  40. package/packs/frontend/skills/bundle-size-review/evals/files/5/.browserslistrc +3 -0
  41. package/packs/frontend/skills/bundle-size-review/evals/files/5/build/meta.json +371 -0
  42. package/packs/frontend/skills/bundle-size-review/evals/files/5/build.mjs +18 -0
  43. package/packs/frontend/skills/bundle-size-review/evals/files/5/package-lock.json +39 -0
  44. package/packs/frontend/skills/bundle-size-review/evals/files/5/package.json +22 -0
  45. package/packs/frontend/skills/bundle-size-review/evals/files/5/src/components/Icon.jsx +8 -0
  46. package/packs/frontend/skills/bundle-size-review/evals/files/5/src/polyfills.js +7 -0
  47. package/packs/frontend/skills/bundle-size-review/evals/files/5/src/utils/groupRows.js +4 -0
  48. package/packs/frontend/skills/bundle-size-review/evals/files/6/.browserslistrc +3 -0
  49. package/packs/frontend/skills/bundle-size-review/evals/files/6/package.json +25 -0
  50. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/App.tsx +8 -0
  51. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/chartLocales.ts +16 -0
  52. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/index.ts +8 -0
  53. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/messages/de.ts +5 -0
  54. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/messages/en.ts +5 -0
  55. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/useLocale.ts +14 -0
  56. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/main.tsx +10 -0
  57. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/pages/Home.tsx +14 -0
  58. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/pages/Reports.tsx +31 -0
  59. package/packs/frontend/skills/bundle-size-review/evals/files/6/src/routes.tsx +17 -0
  60. package/packs/frontend/skills/bundle-size-review/evals/files/6/stats.json +1869 -0
  61. package/packs/frontend/skills/bundle-size-review/evals/files/6/vite.config.ts +13 -0
  62. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/package.json +26 -0
  63. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/src/App.tsx +27 -0
  64. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/src/pages/Analytics.tsx +10 -0
  65. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/src/pages/Users.tsx +13 -0
  66. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/stats.json +1421 -0
  67. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/webpack.config.js +25 -0
  68. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/package.json +27 -0
  69. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/App.tsx +18 -0
  70. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/index.tsx +4 -0
  71. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/pages/Home.tsx +10 -0
  72. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/pages/Product.tsx +13 -0
  73. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/stats.json +1496 -0
  74. package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/webpack.config.js +26 -0
  75. package/packs/frontend/skills/bundle-size-review/evals/files/7/package.json +9 -0
  76. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/Button.css +3 -0
  77. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/Button.js +14 -0
  78. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/Chart.js +21 -0
  79. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/index.js +22 -0
  80. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/package.json +20 -0
  81. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/Button.css +3 -0
  82. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/Button.tsx +9 -0
  83. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/Chart.tsx +14 -0
  84. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/index.ts +6 -0
  85. package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/tsconfig.json +14 -0
  86. package/packs/frontend/skills/bundle-size-review/examples/inputs/stats-after.json +1358 -0
  87. package/packs/frontend/skills/bundle-size-review/examples/inputs/stats-before.json +1334 -0
  88. package/packs/frontend/skills/bundle-size-review/examples/larkspur-sparkline.md +52 -0
  89. package/packs/frontend/skills/bundle-size-review/manifest.json +52 -0
  90. package/packs/frontend/skills/bundle-size-review/reference.md +147 -0
  91. package/packs/frontend/skills/bundle-size-review/scripts/bundle_stats.py +1633 -0
  92. package/packs/frontend/skills/web-performance-review/SKILL.md +3 -3
  93. package/packs/frontend/skills/web-performance-review/manifest.json +2 -1
@@ -0,0 +1,321 @@
1
+ item,judge,trial,label,expert,length
2
+ R01,judge-a,1,pass,pass,132
3
+ R01,judge-a,2,pass,pass,132
4
+ R01,judge-b,1,pass,pass,132
5
+ R01,judge-b,2,pass,pass,132
6
+ R02,judge-a,1,pass,pass,169
7
+ R02,judge-a,2,pass,pass,169
8
+ R02,judge-b,1,pass,pass,169
9
+ R02,judge-b,2,pass,pass,169
10
+ R03,judge-a,1,pass,pass,96
11
+ R03,judge-a,2,pass,pass,96
12
+ R03,judge-b,1,pass,pass,96
13
+ R03,judge-b,2,fail,pass,96
14
+ R04,judge-a,1,pass,pass,133
15
+ R04,judge-a,2,pass,pass,133
16
+ R04,judge-b,1,pass,pass,133
17
+ R04,judge-b,2,pass,pass,133
18
+ R05,judge-a,1,fail,pass,170
19
+ R05,judge-a,2,fail,pass,170
20
+ R05,judge-b,1,fail,pass,170
21
+ R05,judge-b,2,fail,pass,170
22
+ R06,judge-a,1,pass,pass,97
23
+ R06,judge-a,2,pass,pass,97
24
+ R06,judge-b,1,pass,pass,97
25
+ R06,judge-b,2,pass,pass,97
26
+ R07,judge-a,1,pass,pass,134
27
+ R07,judge-a,2,pass,pass,134
28
+ R07,judge-b,1,pass,pass,134
29
+ R07,judge-b,2,pass,pass,134
30
+ R08,judge-a,1,pass,pass,171
31
+ R08,judge-a,2,pass,pass,171
32
+ R08,judge-b,1,pass,pass,171
33
+ R08,judge-b,2,pass,pass,171
34
+ R09,judge-a,1,pass,pass,98
35
+ R09,judge-a,2,pass,pass,98
36
+ R09,judge-b,1,pass,pass,98
37
+ R09,judge-b,2,fail,pass,98
38
+ R10,judge-a,1,pass,pass,135
39
+ R10,judge-a,2,pass,pass,135
40
+ R10,judge-b,1,pass,pass,135
41
+ R10,judge-b,2,pass,pass,135
42
+ R11,judge-a,1,pass,pass,172
43
+ R11,judge-a,2,pass,pass,172
44
+ R11,judge-b,1,pass,pass,172
45
+ R11,judge-b,2,pass,pass,172
46
+ R12,judge-a,1,pass,pass,99
47
+ R12,judge-a,2,fail,pass,99
48
+ R12,judge-b,1,pass,pass,99
49
+ R12,judge-b,2,pass,pass,99
50
+ R13,judge-a,1,pass,pass,136
51
+ R13,judge-a,2,pass,pass,136
52
+ R13,judge-b,1,pass,pass,136
53
+ R13,judge-b,2,pass,pass,136
54
+ R14,judge-a,1,pass,pass,173
55
+ R14,judge-a,2,pass,pass,173
56
+ R14,judge-b,1,pass,pass,173
57
+ R14,judge-b,2,pass,pass,173
58
+ R15,judge-a,1,pass,pass,100
59
+ R15,judge-a,2,pass,pass,100
60
+ R15,judge-b,1,pass,pass,100
61
+ R15,judge-b,2,pass,pass,100
62
+ R16,judge-a,1,pass,pass,137
63
+ R16,judge-a,2,pass,pass,137
64
+ R16,judge-b,1,pass,pass,137
65
+ R16,judge-b,2,pass,pass,137
66
+ R17,judge-a,1,fail,pass,174
67
+ R17,judge-a,2,fail,pass,174
68
+ R17,judge-b,1,pass,pass,174
69
+ R17,judge-b,2,pass,pass,174
70
+ R18,judge-a,1,pass,pass,101
71
+ R18,judge-a,2,pass,pass,101
72
+ R18,judge-b,1,pass,pass,101
73
+ R18,judge-b,2,pass,pass,101
74
+ R19,judge-a,1,pass,pass,138
75
+ R19,judge-a,2,pass,pass,138
76
+ R19,judge-b,1,pass,pass,138
77
+ R19,judge-b,2,pass,pass,138
78
+ R20,judge-a,1,pass,pass,175
79
+ R20,judge-a,2,pass,pass,175
80
+ R20,judge-b,1,pass,pass,175
81
+ R20,judge-b,2,pass,pass,175
82
+ R21,judge-a,1,pass,pass,102
83
+ R21,judge-a,2,pass,pass,102
84
+ R21,judge-b,1,pass,pass,102
85
+ R21,judge-b,2,pass,pass,102
86
+ R22,judge-a,1,pass,pass,139
87
+ R22,judge-a,2,pass,pass,139
88
+ R22,judge-b,1,fail,pass,139
89
+ R22,judge-b,2,pass,pass,139
90
+ R23,judge-a,1,pass,pass,176
91
+ R23,judge-a,2,pass,pass,176
92
+ R23,judge-b,1,pass,pass,176
93
+ R23,judge-b,2,pass,pass,176
94
+ R24,judge-a,1,pass,pass,103
95
+ R24,judge-a,2,pass,pass,103
96
+ R24,judge-b,1,pass,pass,103
97
+ R24,judge-b,2,pass,pass,103
98
+ R25,judge-a,1,pass,pass,140
99
+ R25,judge-a,2,pass,pass,140
100
+ R25,judge-b,1,pass,pass,140
101
+ R25,judge-b,2,pass,pass,140
102
+ R26,judge-a,1,pass,pass,177
103
+ R26,judge-a,2,pass,pass,177
104
+ R26,judge-b,1,fail,pass,177
105
+ R26,judge-b,2,fail,pass,177
106
+ R27,judge-a,1,pass,pass,104
107
+ R27,judge-a,2,pass,pass,104
108
+ R27,judge-b,1,pass,pass,104
109
+ R27,judge-b,2,pass,pass,104
110
+ R28,judge-a,1,pass,pass,141
111
+ R28,judge-a,2,pass,pass,141
112
+ R28,judge-b,1,pass,pass,141
113
+ R28,judge-b,2,pass,pass,141
114
+ R29,judge-a,1,pass,fail,215
115
+ R29,judge-a,2,pass,fail,215
116
+ R29,judge-b,1,pass,fail,215
117
+ R29,judge-b,2,pass,fail,215
118
+ R30,judge-a,1,pass,fail,228
119
+ R30,judge-a,2,pass,fail,228
120
+ R30,judge-b,1,pass,fail,228
121
+ R30,judge-b,2,fail,fail,228
122
+ R31,judge-a,1,pass,fail,241
123
+ R31,judge-a,2,pass,fail,241
124
+ R31,judge-b,1,pass,fail,241
125
+ R31,judge-b,2,fail,fail,241
126
+ R32,judge-a,1,pass,fail,254
127
+ R32,judge-a,2,pass,fail,254
128
+ R32,judge-b,1,pass,fail,254
129
+ R32,judge-b,2,pass,fail,254
130
+ R33,judge-a,1,pass,fail,267
131
+ R33,judge-a,2,fail,fail,267
132
+ R33,judge-b,1,pass,fail,267
133
+ R33,judge-b,2,pass,fail,267
134
+ R34,judge-a,1,pass,fail,280
135
+ R34,judge-a,2,pass,fail,280
136
+ R34,judge-b,1,pass,fail,280
137
+ R34,judge-b,2,pass,fail,280
138
+ R35,judge-a,1,pass,fail,293
139
+ R35,judge-a,2,pass,fail,293
140
+ R35,judge-b,1,pass,fail,293
141
+ R35,judge-b,2,fail,fail,293
142
+ R36,judge-a,1,pass,fail,306
143
+ R36,judge-a,2,pass,fail,306
144
+ R36,judge-b,1,pass,fail,306
145
+ R36,judge-b,2,pass,fail,306
146
+ R37,judge-a,1,pass,fail,319
147
+ R37,judge-a,2,pass,fail,319
148
+ R37,judge-b,1,pass,fail,319
149
+ R37,judge-b,2,pass,fail,319
150
+ R38,judge-a,1,fail,fail,44
151
+ R38,judge-a,2,fail,fail,44
152
+ R38,judge-b,1,pass,fail,44
153
+ R38,judge-b,2,pass,fail,44
154
+ R39,judge-a,1,fail,fail,56
155
+ R39,judge-a,2,fail,fail,56
156
+ R39,judge-b,1,fail,fail,56
157
+ R39,judge-b,2,fail,fail,56
158
+ R40,judge-a,1,fail,fail,68
159
+ R40,judge-a,2,fail,fail,68
160
+ R40,judge-b,1,fail,fail,68
161
+ R40,judge-b,2,fail,fail,68
162
+ R41,judge-a,1,pass,,163
163
+ R41,judge-a,2,pass,,163
164
+ R41,judge-b,1,pass,,163
165
+ R41,judge-b,2,pass,,163
166
+ R42,judge-a,1,pass,,216
167
+ R42,judge-a,2,pass,,216
168
+ R42,judge-b,1,pass,,216
169
+ R42,judge-b,2,pass,,216
170
+ R43,judge-a,1,pass,,119
171
+ R43,judge-a,2,pass,,119
172
+ R43,judge-b,1,pass,,119
173
+ R43,judge-b,2,pass,,119
174
+ R44,judge-a,1,fail,,52
175
+ R44,judge-a,2,fail,,52
176
+ R44,judge-b,1,fail,,52
177
+ R44,judge-b,2,fail,,52
178
+ R45,judge-a,1,pass,,225
179
+ R45,judge-a,2,pass,,225
180
+ R45,judge-b,1,pass,,225
181
+ R45,judge-b,2,pass,,225
182
+ R46,judge-a,1,pass,,128
183
+ R46,judge-a,2,pass,,128
184
+ R46,judge-b,1,pass,,128
185
+ R46,judge-b,2,pass,,128
186
+ R47,judge-a,1,pass,,181
187
+ R47,judge-a,2,pass,,181
188
+ R47,judge-b,1,pass,,181
189
+ R47,judge-b,2,pass,,181
190
+ R48,judge-a,1,pass,,234
191
+ R48,judge-a,2,pass,,234
192
+ R48,judge-b,1,pass,,234
193
+ R48,judge-b,2,fail,,234
194
+ R49,judge-a,1,pass,,137
195
+ R49,judge-a,2,pass,,137
196
+ R49,judge-b,1,pass,,137
197
+ R49,judge-b,2,pass,,137
198
+ R50,judge-a,1,pass,,190
199
+ R50,judge-a,2,pass,,190
200
+ R50,judge-b,1,pass,,190
201
+ R50,judge-b,2,pass,,190
202
+ R51,judge-a,1,pass,,93
203
+ R51,judge-a,2,pass,,93
204
+ R51,judge-b,1,pass,,93
205
+ R51,judge-b,2,pass,,93
206
+ R52,judge-a,1,pass,,146
207
+ R52,judge-a,2,pass,,146
208
+ R52,judge-b,1,pass,,146
209
+ R52,judge-b,2,fail,,146
210
+ R53,judge-a,1,pass,,199
211
+ R53,judge-a,2,pass,,199
212
+ R53,judge-b,1,pass,,199
213
+ R53,judge-b,2,pass,,199
214
+ R54,judge-a,1,pass,,102
215
+ R54,judge-a,2,pass,,102
216
+ R54,judge-b,1,pass,,102
217
+ R54,judge-b,2,pass,,102
218
+ R55,judge-a,1,pass,,155
219
+ R55,judge-a,2,pass,,155
220
+ R55,judge-b,1,pass,,155
221
+ R55,judge-b,2,pass,,155
222
+ R56,judge-a,1,pass,,208
223
+ R56,judge-a,2,pass,,208
224
+ R56,judge-b,1,pass,,208
225
+ R56,judge-b,2,pass,,208
226
+ R57,judge-a,1,pass,,111
227
+ R57,judge-a,2,pass,,111
228
+ R57,judge-b,1,pass,,111
229
+ R57,judge-b,2,pass,,111
230
+ R58,judge-a,1,pass,,164
231
+ R58,judge-a,2,pass,,164
232
+ R58,judge-b,1,pass,,164
233
+ R58,judge-b,2,pass,,164
234
+ R59,judge-a,1,fail,,61
235
+ R59,judge-a,2,fail,,61
236
+ R59,judge-b,1,fail,,61
237
+ R59,judge-b,2,fail,,61
238
+ R60,judge-a,1,pass,,120
239
+ R60,judge-a,2,pass,,120
240
+ R60,judge-b,1,pass,,120
241
+ R60,judge-b,2,pass,,120
242
+ R61,judge-a,1,pass,,173
243
+ R61,judge-a,2,pass,,173
244
+ R61,judge-b,1,pass,,173
245
+ R61,judge-b,2,fail,,173
246
+ R62,judge-a,1,pass,,226
247
+ R62,judge-a,2,pass,,226
248
+ R62,judge-b,1,pass,,226
249
+ R62,judge-b,2,pass,,226
250
+ R63,judge-a,1,pass,,129
251
+ R63,judge-a,2,pass,,129
252
+ R63,judge-b,1,pass,,129
253
+ R63,judge-b,2,pass,,129
254
+ R64,judge-a,1,pass,,182
255
+ R64,judge-a,2,pass,,182
256
+ R64,judge-b,1,pass,,182
257
+ R64,judge-b,2,pass,,182
258
+ R65,judge-a,1,pass,,235
259
+ R65,judge-a,2,pass,,235
260
+ R65,judge-b,1,pass,,235
261
+ R65,judge-b,2,pass,,235
262
+ R66,judge-a,1,fail,,70
263
+ R66,judge-a,2,fail,,70
264
+ R66,judge-b,1,pass,,70
265
+ R66,judge-b,2,fail,,70
266
+ R67,judge-a,1,pass,,191
267
+ R67,judge-a,2,pass,,191
268
+ R67,judge-b,1,pass,,191
269
+ R67,judge-b,2,pass,,191
270
+ R68,judge-a,1,pass,,94
271
+ R68,judge-a,2,pass,,94
272
+ R68,judge-b,1,pass,,94
273
+ R68,judge-b,2,pass,,94
274
+ R69,judge-a,1,pass,,147
275
+ R69,judge-a,2,pass,,147
276
+ R69,judge-b,1,pass,,147
277
+ R69,judge-b,2,pass,,147
278
+ R70,judge-a,1,pass,,200
279
+ R70,judge-a,2,fail,,200
280
+ R70,judge-b,1,pass,,200
281
+ R70,judge-b,2,pass,,200
282
+ R71,judge-a,1,pass,,58
283
+ R71,judge-a,2,pass,,58
284
+ R71,judge-b,1,fail,,58
285
+ R71,judge-b,2,fail,,58
286
+ R72,judge-a,1,pass,,156
287
+ R72,judge-a,2,pass,,156
288
+ R72,judge-b,1,pass,,156
289
+ R72,judge-b,2,pass,,156
290
+ R73,judge-a,1,pass,,209
291
+ R73,judge-a,2,pass,,209
292
+ R73,judge-b,1,pass,,209
293
+ R73,judge-b,2,fail,,209
294
+ R74,judge-a,1,pass,,112
295
+ R74,judge-a,2,pass,,112
296
+ R74,judge-b,1,pass,,112
297
+ R74,judge-b,2,pass,,112
298
+ R75,judge-a,1,pass,,165
299
+ R75,judge-a,2,pass,,165
300
+ R75,judge-b,1,pass,,165
301
+ R75,judge-b,2,pass,,165
302
+ R76,judge-a,1,pass,,218
303
+ R76,judge-a,2,pass,,218
304
+ R76,judge-b,1,pass,,218
305
+ R76,judge-b,2,pass,,218
306
+ R77,judge-a,1,pass,,66
307
+ R77,judge-a,2,pass,,66
308
+ R77,judge-b,1,fail,,66
309
+ R77,judge-b,2,fail,,66
310
+ R78,judge-a,1,pass,,174
311
+ R78,judge-a,2,pass,,174
312
+ R78,judge-b,1,pass,,174
313
+ R78,judge-b,2,pass,,174
314
+ R79,judge-a,1,pass,,227
315
+ R79,judge-a,2,pass,,227
316
+ R79,judge-b,1,pass,,227
317
+ R79,judge-b,2,fail,,227
318
+ R80,judge-a,1,pass,,130
319
+ R80,judge-a,2,pass,,130
320
+ R80,judge-b,1,pass,,130
321
+ R80,judge-b,2,pass,,130
@@ -0,0 +1,28 @@
1
+ # Reply-drafter judge — notes for the release gate
2
+
3
+ **Team:** Customer Support Tooling, Larkspur Telecom (fictional)
4
+ **Feature:** the reply drafter suggests an answer to each incoming support ticket; agents edit and send.
5
+
6
+ ## What we did
7
+
8
+ - Sampled 80 drafted replies (R01–R80) from last week's tickets.
9
+ - Two LLM judges graded every reply pass/fail against one rubric: "Is this a good reply the agent
10
+ could send as-is?" Each judge graded each reply twice (trial 1 and 2), same prompt, same settings.
11
+ - `judge-a` is the larger model.
12
+ - `judge-b` is the cheaper model, about a fifth of the cost per call.
13
+ - Our senior support lead, Maren Holt, graded R01–R40 by hand against the refund and contract
14
+ policy. She did not see the judges' grades.
15
+ - `length` is the reply's word count.
16
+
17
+ All of it is in `judgements.csv`.
18
+
19
+ ## What we want to decide
20
+
21
+ From next sprint, every prompt change to the reply drafter ships only if **`judge-b` passes at
22
+ least 85% of a fresh 80-reply sample**. `judge-b`, because it is cheaper; `judge-a` stays as a
23
+ spot check.
24
+
25
+ ## Why we think it's fine
26
+
27
+ Tomás (ML engineer): "The two judges agree on 91% of replies, so the grading is reliable. Both
28
+ pass about 89% today, so we're comfortably over the 85% bar."
@@ -0,0 +1,101 @@
1
+ item,judge,trial,winner,position,length,other_length
2
+ E01,grader-1,1,v2,A,191,149
3
+ E01,grader-1,2,v1,A,149,191
4
+ E02,grader-1,1,v2,B,152,178
5
+ E02,grader-1,2,v1,B,178,152
6
+ E03,grader-1,1,v2,A,193,137
7
+ E03,grader-1,2,v2,A,193,137
8
+ E04,grader-1,1,v1,A,166,154
9
+ E04,grader-1,2,v1,A,166,154
10
+ E05,grader-1,1,v1,A,125,195
11
+ E05,grader-1,2,v1,A,125,195
12
+ E06,grader-1,1,v2,B,156,154
13
+ E06,grader-1,2,v2,A,156,154
14
+ E07,grader-1,1,v1,B,183,197
15
+ E07,grader-1,2,tie,,,
16
+ E08,grader-1,1,v1,A,142,158
17
+ E08,grader-1,2,v1,B,142,158
18
+ E09,grader-1,1,v1,A,171,199
19
+ E09,grader-1,2,v2,A,199,171
20
+ E10,grader-1,1,v1,A,130,160
21
+ E10,grader-1,2,v2,A,160,130
22
+ E11,grader-1,1,v1,A,159,201
23
+ E11,grader-1,2,v2,A,201,159
24
+ E12,grader-1,1,v2,B,162,188
25
+ E12,grader-1,2,v2,B,162,188
26
+ E13,grader-1,1,v2,A,203,147
27
+ E13,grader-1,2,v1,B,147,203
28
+ E14,grader-1,1,v1,A,176,164
29
+ E14,grader-1,2,v1,A,176,164
30
+ E15,grader-1,1,v2,A,205,135
31
+ E15,grader-1,2,v1,A,135,205
32
+ E16,grader-1,1,v1,A,164,166
33
+ E16,grader-1,2,v2,A,166,164
34
+ E17,grader-1,1,v2,A,207,123
35
+ E17,grader-1,2,v2,A,207,123
36
+ E18,grader-1,1,v2,A,168,152
37
+ E18,grader-1,2,v1,B,152,168
38
+ E19,grader-1,1,tie,,,
39
+ E19,grader-1,2,v2,A,209,181
40
+ E20,grader-1,1,v1,A,140,170
41
+ E20,grader-1,2,v2,A,170,140
42
+ E21,grader-1,1,v2,B,211,169
43
+ E21,grader-1,2,v1,A,169,211
44
+ E22,grader-1,1,v1,A,128,172
45
+ E22,grader-1,2,v2,A,172,128
46
+ E23,grader-1,1,v2,B,213,157
47
+ E23,grader-1,2,v1,B,157,213
48
+ E24,grader-1,1,v1,B,186,174
49
+ E24,grader-1,2,v2,A,174,186
50
+ E25,grader-1,1,v1,A,145,215
51
+ E25,grader-1,2,v2,A,215,145
52
+ E26,grader-1,1,v1,A,174,176
53
+ E26,grader-1,2,v2,A,176,174
54
+ E27,grader-1,1,v2,B,217,133
55
+ E27,grader-1,2,v2,A,217,133
56
+ E28,grader-1,1,v2,A,178,162
57
+ E28,grader-1,2,v2,A,178,162
58
+ E29,grader-1,1,v2,A,219,121
59
+ E29,grader-1,2,v1,B,121,219
60
+ E30,grader-1,1,v1,B,150,180
61
+ E30,grader-1,2,v1,B,150,180
62
+ E31,grader-1,1,v1,A,179,221
63
+ E31,grader-1,2,v2,A,221,179
64
+ E32,grader-1,1,v1,B,138,182
65
+ E32,grader-1,2,v1,A,138,182
66
+ E33,grader-1,1,v1,A,167,223
67
+ E33,grader-1,2,tie,,,
68
+ E34,grader-1,1,v2,A,184,126
69
+ E34,grader-1,2,v1,B,126,184
70
+ E35,grader-1,1,v2,A,225,155
71
+ E35,grader-1,2,v2,A,225,155
72
+ E36,grader-1,1,v2,A,186,184
73
+ E36,grader-1,2,v2,A,186,184
74
+ E37,grader-1,1,v2,A,227,143
75
+ E37,grader-1,2,v2,B,227,143
76
+ E38,grader-1,1,v2,A,188,172
77
+ E38,grader-1,2,v1,A,172,188
78
+ E39,grader-1,1,v2,A,229,131
79
+ E39,grader-1,2,v2,A,229,131
80
+ E40,grader-1,1,v1,A,160,190
81
+ E40,grader-1,2,v1,B,160,190
82
+ E41,grader-1,1,v1,A,189,151
83
+ E41,grader-1,2,v2,B,151,189
84
+ E42,grader-1,1,v1,A,148,192
85
+ E42,grader-1,2,v1,B,148,192
86
+ E43,grader-1,1,v2,A,153,177
87
+ E43,grader-1,2,v2,B,153,177
88
+ E44,grader-1,1,v2,A,194,136
89
+ E44,grader-1,2,v2,B,194,136
90
+ E45,grader-1,1,tie,,,
91
+ E45,grader-1,2,v1,B,165,155
92
+ E46,grader-1,1,v1,B,124,196
93
+ E46,grader-1,2,v2,A,196,124
94
+ E47,grader-1,1,v1,B,153,157
95
+ E47,grader-1,2,v1,B,153,157
96
+ E48,grader-1,1,v2,A,198,182
97
+ E48,grader-1,2,v2,A,198,182
98
+ E49,grader-1,1,v2,A,159,141
99
+ E49,grader-1,2,v2,A,159,141
100
+ E50,grader-1,1,v2,A,200,170
101
+ E50,grader-1,2,v2,A,200,170
@@ -0,0 +1,16 @@
1
+ # Onboarding email prompt — v1 vs v2
2
+
3
+ **Team:** Growth, Quillmark (fictional)
4
+
5
+ - 50 sign-up profiles (E01–E50). For each, prompt v1 and prompt v2 each wrote a welcome email.
6
+ - One LLM grader (`grader-1`) saw both emails for a profile and picked the better one, or `tie`.
7
+ It ran twice per profile (trial 1 and 2). Which email was shown as A was randomised on every run.
8
+ - `winner` is the prompt whose email the grader picked. `position` is where that email was shown
9
+ (A first, B second). `length` is the picked email's word count; `other_length` is the other's.
10
+
11
+ Data: `pairwise.csv`.
12
+
13
+ ## Result so far
14
+
15
+ v2 won 53 comparisons, v1 won 43, 4 ties. Plan: switch production to v2 this sprint and use the
16
+ same grader setup for every future prompt change.
@@ -0,0 +1,97 @@
1
+ ticket_id,prompt_version,reply_words,judge_score,human_score
2
+ TCK-4101,v1,97,3,
3
+ TCK-4102,v1,104,2,
4
+ TCK-4103,v1,91,5,
5
+ TCK-4104,v1,104,4,3
6
+ TCK-4105,v1,56,5,5
7
+ TCK-4106,v1,57,4,4
8
+ TCK-4107,v1,74,3,
9
+ TCK-4108,v1,76,3,
10
+ TCK-4109,v1,62,3,
11
+ TCK-4110,v1,59,5,5
12
+ TCK-4111,v1,82,2,
13
+ TCK-4112,v1,101,5,
14
+ TCK-4113,v1,71,4,4
15
+ TCK-4114,v1,57,3,
16
+ TCK-4115,v1,73,3,
17
+ TCK-4116,v1,61,2,
18
+ TCK-4117,v1,83,3,3
19
+ TCK-4118,v1,78,1,1
20
+ TCK-4119,v1,62,5,5
21
+ TCK-4120,v1,73,2,3
22
+ TCK-4121,v1,66,4,4
23
+ TCK-4122,v1,118,1,
24
+ TCK-4123,v1,57,2,
25
+ TCK-4124,v1,64,2,
26
+ TCK-4125,v1,106,5,
27
+ TCK-4126,v1,90,4,
28
+ TCK-4127,v1,56,2,
29
+ TCK-4128,v1,96,4,
30
+ TCK-4129,v1,63,3,
31
+ TCK-4130,v1,68,2,
32
+ TCK-4131,v1,71,5,
33
+ TCK-4132,v1,65,4,
34
+ TCK-4133,v1,108,4,
35
+ TCK-4134,v1,85,2,2
36
+ TCK-4135,v1,101,4,
37
+ TCK-4136,v1,107,4,
38
+ TCK-4137,v1,90,2,
39
+ TCK-4138,v1,86,3,
40
+ TCK-4139,v1,96,3,
41
+ TCK-4140,v1,92,3,3
42
+ TCK-4141,v1,65,5,5
43
+ TCK-4142,v1,76,4,4
44
+ TCK-4143,v1,83,1,1
45
+ TCK-4144,v1,85,3,
46
+ TCK-4145,v1,125,3,3
47
+ TCK-4146,v1,74,4,
48
+ TCK-4147,v1,70,3,
49
+ TCK-4148,v1,70,3,
50
+ TCK-4149,v2,229,5,4
51
+ TCK-4150,v2,198,5,
52
+ TCK-4151,v2,240,4,3
53
+ TCK-4152,v2,225,5,4
54
+ TCK-4153,v2,201,5,
55
+ TCK-4154,v2,228,3,
56
+ TCK-4155,v2,211,5,
57
+ TCK-4156,v2,186,5,
58
+ TCK-4157,v2,244,4,
59
+ TCK-4158,v2,246,4,
60
+ TCK-4159,v2,188,5,
61
+ TCK-4160,v2,223,5,
62
+ TCK-4161,v2,211,5,4
63
+ TCK-4162,v2,250,3,
64
+ TCK-4163,v2,198,3,3
65
+ TCK-4164,v2,141,3,
66
+ TCK-4165,v2,239,5,
67
+ TCK-4166,v2,232,5,
68
+ TCK-4167,v2,198,5,
69
+ TCK-4168,v2,157,4,
70
+ TCK-4169,v2,156,4,
71
+ TCK-4170,v2,138,3,
72
+ TCK-4171,v2,202,5,
73
+ TCK-4172,v2,214,3,3
74
+ TCK-4173,v2,228,5,
75
+ TCK-4174,v2,140,2,2
76
+ TCK-4175,v2,210,4,3
77
+ TCK-4176,v2,143,4,
78
+ TCK-4177,v2,227,4,4
79
+ TCK-4178,v2,170,3,
80
+ TCK-4179,v2,237,5,
81
+ TCK-4180,v2,148,3,
82
+ TCK-4181,v2,238,3,
83
+ TCK-4182,v2,220,3,
84
+ TCK-4183,v2,200,4,3
85
+ TCK-4184,v2,184,4,3
86
+ TCK-4185,v2,157,5,
87
+ TCK-4186,v2,143,1,
88
+ TCK-4187,v2,174,4,3
89
+ TCK-4188,v2,159,1,1
90
+ TCK-4189,v2,150,4,
91
+ TCK-4190,v2,258,4,2
92
+ TCK-4191,v2,170,5,5
93
+ TCK-4192,v2,250,5,5
94
+ TCK-4193,v2,248,5,
95
+ TCK-4194,v2,220,5,
96
+ TCK-4195,v2,169,5,
97
+ TCK-4196,v2,237,2,