skilldrop-cli 0.16.36 → 0.16.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/catalogue.json +3 -1
- package/model-routing.json +8 -0
- package/package.json +1 -1
- package/packs/ai-engineering/skills/judge-audit/SKILL.md +157 -0
- package/packs/ai-engineering/skills/judge-audit/evals/eval_queries.json +11 -0
- package/packs/ai-engineering/skills/judge-audit/evals/evals.json +136 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/1/judgements.csv +321 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/1/notes.md +28 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/2/pairwise.csv +101 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/2/run-notes.md +16 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/4/grades.csv +97 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/5/pairwise_judgements.jsonl +120 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/6/rag-judge/README.md +36 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/6/rag-judge/judgments.csv +301 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/7/moderation/gold_labels.csv +101 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/7/moderation/run_1.jsonl +100 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/7/moderation/run_2.jsonl +100 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/8/patches.csv +241 -0
- package/packs/ai-engineering/skills/judge-audit/evals/files/9/adequacy_ratings.csv +151 -0
- package/packs/ai-engineering/skills/judge-audit/examples/summary-panel.md +70 -0
- package/packs/ai-engineering/skills/judge-audit/manifest.json +44 -0
- package/packs/ai-engineering/skills/judge-audit/reference.md +176 -0
- package/packs/ai-engineering/skills/judge-audit/scripts/judge_stats.py +1735 -0
- package/packs/frontend/pack.json +9 -3
- package/packs/frontend/skills/bundle-size-review/SKILL.md +114 -0
- package/packs/frontend/skills/bundle-size-review/evals/eval_queries.json +12 -0
- package/packs/frontend/skills/bundle-size-review/evals/evals.json +108 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/1/package.json +28 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/1/src/utils/format.js +26 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/1/stats.json +1334 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/2/PR.md +21 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/2/meta-after.json +440 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/2/meta-before.json +360 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/3/package.json +29 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/3/stats.json +1583 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/4/src/components/CommentBox.jsx +24 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/4/stats/after.json +299 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/4/stats/before.json +275 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/.browserslistrc +3 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/build/meta.json +371 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/build.mjs +18 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/package-lock.json +39 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/package.json +22 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/src/components/Icon.jsx +8 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/src/polyfills.js +7 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/5/src/utils/groupRows.js +4 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/.browserslistrc +3 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/package.json +25 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/App.tsx +8 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/chartLocales.ts +16 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/index.ts +8 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/messages/de.ts +5 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/messages/en.ts +5 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/i18n/useLocale.ts +14 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/main.tsx +10 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/pages/Home.tsx +14 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/pages/Reports.tsx +31 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/src/routes.tsx +17 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/stats.json +1869 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/6/vite.config.ts +13 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/package.json +26 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/src/App.tsx +27 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/src/pages/Analytics.tsx +10 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/src/pages/Users.tsx +13 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/stats.json +1421 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/backoffice/webpack.config.js +25 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/package.json +27 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/App.tsx +18 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/index.tsx +4 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/pages/Home.tsx +10 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/src/pages/Product.tsx +13 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/stats.json +1496 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/apps/storefront/webpack.config.js +26 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/package.json +9 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/Button.css +3 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/Button.js +14 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/Chart.js +21 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/dist/index.js +22 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/package.json +20 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/Button.css +3 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/Button.tsx +9 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/Chart.tsx +14 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/src/index.ts +6 -0
- package/packs/frontend/skills/bundle-size-review/evals/files/7/packages/ui/tsconfig.json +14 -0
- package/packs/frontend/skills/bundle-size-review/examples/inputs/stats-after.json +1358 -0
- package/packs/frontend/skills/bundle-size-review/examples/inputs/stats-before.json +1334 -0
- package/packs/frontend/skills/bundle-size-review/examples/larkspur-sparkline.md +52 -0
- package/packs/frontend/skills/bundle-size-review/manifest.json +52 -0
- package/packs/frontend/skills/bundle-size-review/reference.md +147 -0
- package/packs/frontend/skills/bundle-size-review/scripts/bundle_stats.py +1633 -0
- package/packs/frontend/skills/web-performance-review/SKILL.md +3 -3
- package/packs/frontend/skills/web-performance-review/manifest.json +2 -1
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
item,judge,trial,label,expert,length
|
|
2
|
+
R01,judge-a,1,pass,pass,132
|
|
3
|
+
R01,judge-a,2,pass,pass,132
|
|
4
|
+
R01,judge-b,1,pass,pass,132
|
|
5
|
+
R01,judge-b,2,pass,pass,132
|
|
6
|
+
R02,judge-a,1,pass,pass,169
|
|
7
|
+
R02,judge-a,2,pass,pass,169
|
|
8
|
+
R02,judge-b,1,pass,pass,169
|
|
9
|
+
R02,judge-b,2,pass,pass,169
|
|
10
|
+
R03,judge-a,1,pass,pass,96
|
|
11
|
+
R03,judge-a,2,pass,pass,96
|
|
12
|
+
R03,judge-b,1,pass,pass,96
|
|
13
|
+
R03,judge-b,2,fail,pass,96
|
|
14
|
+
R04,judge-a,1,pass,pass,133
|
|
15
|
+
R04,judge-a,2,pass,pass,133
|
|
16
|
+
R04,judge-b,1,pass,pass,133
|
|
17
|
+
R04,judge-b,2,pass,pass,133
|
|
18
|
+
R05,judge-a,1,fail,pass,170
|
|
19
|
+
R05,judge-a,2,fail,pass,170
|
|
20
|
+
R05,judge-b,1,fail,pass,170
|
|
21
|
+
R05,judge-b,2,fail,pass,170
|
|
22
|
+
R06,judge-a,1,pass,pass,97
|
|
23
|
+
R06,judge-a,2,pass,pass,97
|
|
24
|
+
R06,judge-b,1,pass,pass,97
|
|
25
|
+
R06,judge-b,2,pass,pass,97
|
|
26
|
+
R07,judge-a,1,pass,pass,134
|
|
27
|
+
R07,judge-a,2,pass,pass,134
|
|
28
|
+
R07,judge-b,1,pass,pass,134
|
|
29
|
+
R07,judge-b,2,pass,pass,134
|
|
30
|
+
R08,judge-a,1,pass,pass,171
|
|
31
|
+
R08,judge-a,2,pass,pass,171
|
|
32
|
+
R08,judge-b,1,pass,pass,171
|
|
33
|
+
R08,judge-b,2,pass,pass,171
|
|
34
|
+
R09,judge-a,1,pass,pass,98
|
|
35
|
+
R09,judge-a,2,pass,pass,98
|
|
36
|
+
R09,judge-b,1,pass,pass,98
|
|
37
|
+
R09,judge-b,2,fail,pass,98
|
|
38
|
+
R10,judge-a,1,pass,pass,135
|
|
39
|
+
R10,judge-a,2,pass,pass,135
|
|
40
|
+
R10,judge-b,1,pass,pass,135
|
|
41
|
+
R10,judge-b,2,pass,pass,135
|
|
42
|
+
R11,judge-a,1,pass,pass,172
|
|
43
|
+
R11,judge-a,2,pass,pass,172
|
|
44
|
+
R11,judge-b,1,pass,pass,172
|
|
45
|
+
R11,judge-b,2,pass,pass,172
|
|
46
|
+
R12,judge-a,1,pass,pass,99
|
|
47
|
+
R12,judge-a,2,fail,pass,99
|
|
48
|
+
R12,judge-b,1,pass,pass,99
|
|
49
|
+
R12,judge-b,2,pass,pass,99
|
|
50
|
+
R13,judge-a,1,pass,pass,136
|
|
51
|
+
R13,judge-a,2,pass,pass,136
|
|
52
|
+
R13,judge-b,1,pass,pass,136
|
|
53
|
+
R13,judge-b,2,pass,pass,136
|
|
54
|
+
R14,judge-a,1,pass,pass,173
|
|
55
|
+
R14,judge-a,2,pass,pass,173
|
|
56
|
+
R14,judge-b,1,pass,pass,173
|
|
57
|
+
R14,judge-b,2,pass,pass,173
|
|
58
|
+
R15,judge-a,1,pass,pass,100
|
|
59
|
+
R15,judge-a,2,pass,pass,100
|
|
60
|
+
R15,judge-b,1,pass,pass,100
|
|
61
|
+
R15,judge-b,2,pass,pass,100
|
|
62
|
+
R16,judge-a,1,pass,pass,137
|
|
63
|
+
R16,judge-a,2,pass,pass,137
|
|
64
|
+
R16,judge-b,1,pass,pass,137
|
|
65
|
+
R16,judge-b,2,pass,pass,137
|
|
66
|
+
R17,judge-a,1,fail,pass,174
|
|
67
|
+
R17,judge-a,2,fail,pass,174
|
|
68
|
+
R17,judge-b,1,pass,pass,174
|
|
69
|
+
R17,judge-b,2,pass,pass,174
|
|
70
|
+
R18,judge-a,1,pass,pass,101
|
|
71
|
+
R18,judge-a,2,pass,pass,101
|
|
72
|
+
R18,judge-b,1,pass,pass,101
|
|
73
|
+
R18,judge-b,2,pass,pass,101
|
|
74
|
+
R19,judge-a,1,pass,pass,138
|
|
75
|
+
R19,judge-a,2,pass,pass,138
|
|
76
|
+
R19,judge-b,1,pass,pass,138
|
|
77
|
+
R19,judge-b,2,pass,pass,138
|
|
78
|
+
R20,judge-a,1,pass,pass,175
|
|
79
|
+
R20,judge-a,2,pass,pass,175
|
|
80
|
+
R20,judge-b,1,pass,pass,175
|
|
81
|
+
R20,judge-b,2,pass,pass,175
|
|
82
|
+
R21,judge-a,1,pass,pass,102
|
|
83
|
+
R21,judge-a,2,pass,pass,102
|
|
84
|
+
R21,judge-b,1,pass,pass,102
|
|
85
|
+
R21,judge-b,2,pass,pass,102
|
|
86
|
+
R22,judge-a,1,pass,pass,139
|
|
87
|
+
R22,judge-a,2,pass,pass,139
|
|
88
|
+
R22,judge-b,1,fail,pass,139
|
|
89
|
+
R22,judge-b,2,pass,pass,139
|
|
90
|
+
R23,judge-a,1,pass,pass,176
|
|
91
|
+
R23,judge-a,2,pass,pass,176
|
|
92
|
+
R23,judge-b,1,pass,pass,176
|
|
93
|
+
R23,judge-b,2,pass,pass,176
|
|
94
|
+
R24,judge-a,1,pass,pass,103
|
|
95
|
+
R24,judge-a,2,pass,pass,103
|
|
96
|
+
R24,judge-b,1,pass,pass,103
|
|
97
|
+
R24,judge-b,2,pass,pass,103
|
|
98
|
+
R25,judge-a,1,pass,pass,140
|
|
99
|
+
R25,judge-a,2,pass,pass,140
|
|
100
|
+
R25,judge-b,1,pass,pass,140
|
|
101
|
+
R25,judge-b,2,pass,pass,140
|
|
102
|
+
R26,judge-a,1,pass,pass,177
|
|
103
|
+
R26,judge-a,2,pass,pass,177
|
|
104
|
+
R26,judge-b,1,fail,pass,177
|
|
105
|
+
R26,judge-b,2,fail,pass,177
|
|
106
|
+
R27,judge-a,1,pass,pass,104
|
|
107
|
+
R27,judge-a,2,pass,pass,104
|
|
108
|
+
R27,judge-b,1,pass,pass,104
|
|
109
|
+
R27,judge-b,2,pass,pass,104
|
|
110
|
+
R28,judge-a,1,pass,pass,141
|
|
111
|
+
R28,judge-a,2,pass,pass,141
|
|
112
|
+
R28,judge-b,1,pass,pass,141
|
|
113
|
+
R28,judge-b,2,pass,pass,141
|
|
114
|
+
R29,judge-a,1,pass,fail,215
|
|
115
|
+
R29,judge-a,2,pass,fail,215
|
|
116
|
+
R29,judge-b,1,pass,fail,215
|
|
117
|
+
R29,judge-b,2,pass,fail,215
|
|
118
|
+
R30,judge-a,1,pass,fail,228
|
|
119
|
+
R30,judge-a,2,pass,fail,228
|
|
120
|
+
R30,judge-b,1,pass,fail,228
|
|
121
|
+
R30,judge-b,2,fail,fail,228
|
|
122
|
+
R31,judge-a,1,pass,fail,241
|
|
123
|
+
R31,judge-a,2,pass,fail,241
|
|
124
|
+
R31,judge-b,1,pass,fail,241
|
|
125
|
+
R31,judge-b,2,fail,fail,241
|
|
126
|
+
R32,judge-a,1,pass,fail,254
|
|
127
|
+
R32,judge-a,2,pass,fail,254
|
|
128
|
+
R32,judge-b,1,pass,fail,254
|
|
129
|
+
R32,judge-b,2,pass,fail,254
|
|
130
|
+
R33,judge-a,1,pass,fail,267
|
|
131
|
+
R33,judge-a,2,fail,fail,267
|
|
132
|
+
R33,judge-b,1,pass,fail,267
|
|
133
|
+
R33,judge-b,2,pass,fail,267
|
|
134
|
+
R34,judge-a,1,pass,fail,280
|
|
135
|
+
R34,judge-a,2,pass,fail,280
|
|
136
|
+
R34,judge-b,1,pass,fail,280
|
|
137
|
+
R34,judge-b,2,pass,fail,280
|
|
138
|
+
R35,judge-a,1,pass,fail,293
|
|
139
|
+
R35,judge-a,2,pass,fail,293
|
|
140
|
+
R35,judge-b,1,pass,fail,293
|
|
141
|
+
R35,judge-b,2,fail,fail,293
|
|
142
|
+
R36,judge-a,1,pass,fail,306
|
|
143
|
+
R36,judge-a,2,pass,fail,306
|
|
144
|
+
R36,judge-b,1,pass,fail,306
|
|
145
|
+
R36,judge-b,2,pass,fail,306
|
|
146
|
+
R37,judge-a,1,pass,fail,319
|
|
147
|
+
R37,judge-a,2,pass,fail,319
|
|
148
|
+
R37,judge-b,1,pass,fail,319
|
|
149
|
+
R37,judge-b,2,pass,fail,319
|
|
150
|
+
R38,judge-a,1,fail,fail,44
|
|
151
|
+
R38,judge-a,2,fail,fail,44
|
|
152
|
+
R38,judge-b,1,pass,fail,44
|
|
153
|
+
R38,judge-b,2,pass,fail,44
|
|
154
|
+
R39,judge-a,1,fail,fail,56
|
|
155
|
+
R39,judge-a,2,fail,fail,56
|
|
156
|
+
R39,judge-b,1,fail,fail,56
|
|
157
|
+
R39,judge-b,2,fail,fail,56
|
|
158
|
+
R40,judge-a,1,fail,fail,68
|
|
159
|
+
R40,judge-a,2,fail,fail,68
|
|
160
|
+
R40,judge-b,1,fail,fail,68
|
|
161
|
+
R40,judge-b,2,fail,fail,68
|
|
162
|
+
R41,judge-a,1,pass,,163
|
|
163
|
+
R41,judge-a,2,pass,,163
|
|
164
|
+
R41,judge-b,1,pass,,163
|
|
165
|
+
R41,judge-b,2,pass,,163
|
|
166
|
+
R42,judge-a,1,pass,,216
|
|
167
|
+
R42,judge-a,2,pass,,216
|
|
168
|
+
R42,judge-b,1,pass,,216
|
|
169
|
+
R42,judge-b,2,pass,,216
|
|
170
|
+
R43,judge-a,1,pass,,119
|
|
171
|
+
R43,judge-a,2,pass,,119
|
|
172
|
+
R43,judge-b,1,pass,,119
|
|
173
|
+
R43,judge-b,2,pass,,119
|
|
174
|
+
R44,judge-a,1,fail,,52
|
|
175
|
+
R44,judge-a,2,fail,,52
|
|
176
|
+
R44,judge-b,1,fail,,52
|
|
177
|
+
R44,judge-b,2,fail,,52
|
|
178
|
+
R45,judge-a,1,pass,,225
|
|
179
|
+
R45,judge-a,2,pass,,225
|
|
180
|
+
R45,judge-b,1,pass,,225
|
|
181
|
+
R45,judge-b,2,pass,,225
|
|
182
|
+
R46,judge-a,1,pass,,128
|
|
183
|
+
R46,judge-a,2,pass,,128
|
|
184
|
+
R46,judge-b,1,pass,,128
|
|
185
|
+
R46,judge-b,2,pass,,128
|
|
186
|
+
R47,judge-a,1,pass,,181
|
|
187
|
+
R47,judge-a,2,pass,,181
|
|
188
|
+
R47,judge-b,1,pass,,181
|
|
189
|
+
R47,judge-b,2,pass,,181
|
|
190
|
+
R48,judge-a,1,pass,,234
|
|
191
|
+
R48,judge-a,2,pass,,234
|
|
192
|
+
R48,judge-b,1,pass,,234
|
|
193
|
+
R48,judge-b,2,fail,,234
|
|
194
|
+
R49,judge-a,1,pass,,137
|
|
195
|
+
R49,judge-a,2,pass,,137
|
|
196
|
+
R49,judge-b,1,pass,,137
|
|
197
|
+
R49,judge-b,2,pass,,137
|
|
198
|
+
R50,judge-a,1,pass,,190
|
|
199
|
+
R50,judge-a,2,pass,,190
|
|
200
|
+
R50,judge-b,1,pass,,190
|
|
201
|
+
R50,judge-b,2,pass,,190
|
|
202
|
+
R51,judge-a,1,pass,,93
|
|
203
|
+
R51,judge-a,2,pass,,93
|
|
204
|
+
R51,judge-b,1,pass,,93
|
|
205
|
+
R51,judge-b,2,pass,,93
|
|
206
|
+
R52,judge-a,1,pass,,146
|
|
207
|
+
R52,judge-a,2,pass,,146
|
|
208
|
+
R52,judge-b,1,pass,,146
|
|
209
|
+
R52,judge-b,2,fail,,146
|
|
210
|
+
R53,judge-a,1,pass,,199
|
|
211
|
+
R53,judge-a,2,pass,,199
|
|
212
|
+
R53,judge-b,1,pass,,199
|
|
213
|
+
R53,judge-b,2,pass,,199
|
|
214
|
+
R54,judge-a,1,pass,,102
|
|
215
|
+
R54,judge-a,2,pass,,102
|
|
216
|
+
R54,judge-b,1,pass,,102
|
|
217
|
+
R54,judge-b,2,pass,,102
|
|
218
|
+
R55,judge-a,1,pass,,155
|
|
219
|
+
R55,judge-a,2,pass,,155
|
|
220
|
+
R55,judge-b,1,pass,,155
|
|
221
|
+
R55,judge-b,2,pass,,155
|
|
222
|
+
R56,judge-a,1,pass,,208
|
|
223
|
+
R56,judge-a,2,pass,,208
|
|
224
|
+
R56,judge-b,1,pass,,208
|
|
225
|
+
R56,judge-b,2,pass,,208
|
|
226
|
+
R57,judge-a,1,pass,,111
|
|
227
|
+
R57,judge-a,2,pass,,111
|
|
228
|
+
R57,judge-b,1,pass,,111
|
|
229
|
+
R57,judge-b,2,pass,,111
|
|
230
|
+
R58,judge-a,1,pass,,164
|
|
231
|
+
R58,judge-a,2,pass,,164
|
|
232
|
+
R58,judge-b,1,pass,,164
|
|
233
|
+
R58,judge-b,2,pass,,164
|
|
234
|
+
R59,judge-a,1,fail,,61
|
|
235
|
+
R59,judge-a,2,fail,,61
|
|
236
|
+
R59,judge-b,1,fail,,61
|
|
237
|
+
R59,judge-b,2,fail,,61
|
|
238
|
+
R60,judge-a,1,pass,,120
|
|
239
|
+
R60,judge-a,2,pass,,120
|
|
240
|
+
R60,judge-b,1,pass,,120
|
|
241
|
+
R60,judge-b,2,pass,,120
|
|
242
|
+
R61,judge-a,1,pass,,173
|
|
243
|
+
R61,judge-a,2,pass,,173
|
|
244
|
+
R61,judge-b,1,pass,,173
|
|
245
|
+
R61,judge-b,2,fail,,173
|
|
246
|
+
R62,judge-a,1,pass,,226
|
|
247
|
+
R62,judge-a,2,pass,,226
|
|
248
|
+
R62,judge-b,1,pass,,226
|
|
249
|
+
R62,judge-b,2,pass,,226
|
|
250
|
+
R63,judge-a,1,pass,,129
|
|
251
|
+
R63,judge-a,2,pass,,129
|
|
252
|
+
R63,judge-b,1,pass,,129
|
|
253
|
+
R63,judge-b,2,pass,,129
|
|
254
|
+
R64,judge-a,1,pass,,182
|
|
255
|
+
R64,judge-a,2,pass,,182
|
|
256
|
+
R64,judge-b,1,pass,,182
|
|
257
|
+
R64,judge-b,2,pass,,182
|
|
258
|
+
R65,judge-a,1,pass,,235
|
|
259
|
+
R65,judge-a,2,pass,,235
|
|
260
|
+
R65,judge-b,1,pass,,235
|
|
261
|
+
R65,judge-b,2,pass,,235
|
|
262
|
+
R66,judge-a,1,fail,,70
|
|
263
|
+
R66,judge-a,2,fail,,70
|
|
264
|
+
R66,judge-b,1,pass,,70
|
|
265
|
+
R66,judge-b,2,fail,,70
|
|
266
|
+
R67,judge-a,1,pass,,191
|
|
267
|
+
R67,judge-a,2,pass,,191
|
|
268
|
+
R67,judge-b,1,pass,,191
|
|
269
|
+
R67,judge-b,2,pass,,191
|
|
270
|
+
R68,judge-a,1,pass,,94
|
|
271
|
+
R68,judge-a,2,pass,,94
|
|
272
|
+
R68,judge-b,1,pass,,94
|
|
273
|
+
R68,judge-b,2,pass,,94
|
|
274
|
+
R69,judge-a,1,pass,,147
|
|
275
|
+
R69,judge-a,2,pass,,147
|
|
276
|
+
R69,judge-b,1,pass,,147
|
|
277
|
+
R69,judge-b,2,pass,,147
|
|
278
|
+
R70,judge-a,1,pass,,200
|
|
279
|
+
R70,judge-a,2,fail,,200
|
|
280
|
+
R70,judge-b,1,pass,,200
|
|
281
|
+
R70,judge-b,2,pass,,200
|
|
282
|
+
R71,judge-a,1,pass,,58
|
|
283
|
+
R71,judge-a,2,pass,,58
|
|
284
|
+
R71,judge-b,1,fail,,58
|
|
285
|
+
R71,judge-b,2,fail,,58
|
|
286
|
+
R72,judge-a,1,pass,,156
|
|
287
|
+
R72,judge-a,2,pass,,156
|
|
288
|
+
R72,judge-b,1,pass,,156
|
|
289
|
+
R72,judge-b,2,pass,,156
|
|
290
|
+
R73,judge-a,1,pass,,209
|
|
291
|
+
R73,judge-a,2,pass,,209
|
|
292
|
+
R73,judge-b,1,pass,,209
|
|
293
|
+
R73,judge-b,2,fail,,209
|
|
294
|
+
R74,judge-a,1,pass,,112
|
|
295
|
+
R74,judge-a,2,pass,,112
|
|
296
|
+
R74,judge-b,1,pass,,112
|
|
297
|
+
R74,judge-b,2,pass,,112
|
|
298
|
+
R75,judge-a,1,pass,,165
|
|
299
|
+
R75,judge-a,2,pass,,165
|
|
300
|
+
R75,judge-b,1,pass,,165
|
|
301
|
+
R75,judge-b,2,pass,,165
|
|
302
|
+
R76,judge-a,1,pass,,218
|
|
303
|
+
R76,judge-a,2,pass,,218
|
|
304
|
+
R76,judge-b,1,pass,,218
|
|
305
|
+
R76,judge-b,2,pass,,218
|
|
306
|
+
R77,judge-a,1,pass,,66
|
|
307
|
+
R77,judge-a,2,pass,,66
|
|
308
|
+
R77,judge-b,1,fail,,66
|
|
309
|
+
R77,judge-b,2,fail,,66
|
|
310
|
+
R78,judge-a,1,pass,,174
|
|
311
|
+
R78,judge-a,2,pass,,174
|
|
312
|
+
R78,judge-b,1,pass,,174
|
|
313
|
+
R78,judge-b,2,pass,,174
|
|
314
|
+
R79,judge-a,1,pass,,227
|
|
315
|
+
R79,judge-a,2,pass,,227
|
|
316
|
+
R79,judge-b,1,pass,,227
|
|
317
|
+
R79,judge-b,2,fail,,227
|
|
318
|
+
R80,judge-a,1,pass,,130
|
|
319
|
+
R80,judge-a,2,pass,,130
|
|
320
|
+
R80,judge-b,1,pass,,130
|
|
321
|
+
R80,judge-b,2,pass,,130
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Reply-drafter judge — notes for the release gate
|
|
2
|
+
|
|
3
|
+
**Team:** Customer Support Tooling, Larkspur Telecom (fictional)
|
|
4
|
+
**Feature:** the reply drafter suggests an answer to each incoming support ticket; agents edit and send.
|
|
5
|
+
|
|
6
|
+
## What we did
|
|
7
|
+
|
|
8
|
+
- Sampled 80 drafted replies (R01–R80) from last week's tickets.
|
|
9
|
+
- Two LLM judges graded every reply pass/fail against one rubric: "Is this a good reply the agent
|
|
10
|
+
could send as-is?" Each judge graded each reply twice (trial 1 and 2), same prompt, same settings.
|
|
11
|
+
- `judge-a` is the larger model.
|
|
12
|
+
- `judge-b` is the cheaper model, about a fifth of the cost per call.
|
|
13
|
+
- Our senior support lead, Maren Holt, graded R01–R40 by hand against the refund and contract
|
|
14
|
+
policy. She did not see the judges' grades.
|
|
15
|
+
- `length` is the reply's word count.
|
|
16
|
+
|
|
17
|
+
All of it is in `judgements.csv`.
|
|
18
|
+
|
|
19
|
+
## What we want to decide
|
|
20
|
+
|
|
21
|
+
From next sprint, every prompt change to the reply drafter ships only if **`judge-b` passes at
|
|
22
|
+
least 85% of a fresh 80-reply sample**. `judge-b`, because it is cheaper; `judge-a` stays as a
|
|
23
|
+
spot check.
|
|
24
|
+
|
|
25
|
+
## Why we think it's fine
|
|
26
|
+
|
|
27
|
+
Tomás (ML engineer): "The two judges agree on 91% of replies, so the grading is reliable. Both
|
|
28
|
+
pass about 89% today, so we're comfortably over the 85% bar."
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
item,judge,trial,winner,position,length,other_length
|
|
2
|
+
E01,grader-1,1,v2,A,191,149
|
|
3
|
+
E01,grader-1,2,v1,A,149,191
|
|
4
|
+
E02,grader-1,1,v2,B,152,178
|
|
5
|
+
E02,grader-1,2,v1,B,178,152
|
|
6
|
+
E03,grader-1,1,v2,A,193,137
|
|
7
|
+
E03,grader-1,2,v2,A,193,137
|
|
8
|
+
E04,grader-1,1,v1,A,166,154
|
|
9
|
+
E04,grader-1,2,v1,A,166,154
|
|
10
|
+
E05,grader-1,1,v1,A,125,195
|
|
11
|
+
E05,grader-1,2,v1,A,125,195
|
|
12
|
+
E06,grader-1,1,v2,B,156,154
|
|
13
|
+
E06,grader-1,2,v2,A,156,154
|
|
14
|
+
E07,grader-1,1,v1,B,183,197
|
|
15
|
+
E07,grader-1,2,tie,,,
|
|
16
|
+
E08,grader-1,1,v1,A,142,158
|
|
17
|
+
E08,grader-1,2,v1,B,142,158
|
|
18
|
+
E09,grader-1,1,v1,A,171,199
|
|
19
|
+
E09,grader-1,2,v2,A,199,171
|
|
20
|
+
E10,grader-1,1,v1,A,130,160
|
|
21
|
+
E10,grader-1,2,v2,A,160,130
|
|
22
|
+
E11,grader-1,1,v1,A,159,201
|
|
23
|
+
E11,grader-1,2,v2,A,201,159
|
|
24
|
+
E12,grader-1,1,v2,B,162,188
|
|
25
|
+
E12,grader-1,2,v2,B,162,188
|
|
26
|
+
E13,grader-1,1,v2,A,203,147
|
|
27
|
+
E13,grader-1,2,v1,B,147,203
|
|
28
|
+
E14,grader-1,1,v1,A,176,164
|
|
29
|
+
E14,grader-1,2,v1,A,176,164
|
|
30
|
+
E15,grader-1,1,v2,A,205,135
|
|
31
|
+
E15,grader-1,2,v1,A,135,205
|
|
32
|
+
E16,grader-1,1,v1,A,164,166
|
|
33
|
+
E16,grader-1,2,v2,A,166,164
|
|
34
|
+
E17,grader-1,1,v2,A,207,123
|
|
35
|
+
E17,grader-1,2,v2,A,207,123
|
|
36
|
+
E18,grader-1,1,v2,A,168,152
|
|
37
|
+
E18,grader-1,2,v1,B,152,168
|
|
38
|
+
E19,grader-1,1,tie,,,
|
|
39
|
+
E19,grader-1,2,v2,A,209,181
|
|
40
|
+
E20,grader-1,1,v1,A,140,170
|
|
41
|
+
E20,grader-1,2,v2,A,170,140
|
|
42
|
+
E21,grader-1,1,v2,B,211,169
|
|
43
|
+
E21,grader-1,2,v1,A,169,211
|
|
44
|
+
E22,grader-1,1,v1,A,128,172
|
|
45
|
+
E22,grader-1,2,v2,A,172,128
|
|
46
|
+
E23,grader-1,1,v2,B,213,157
|
|
47
|
+
E23,grader-1,2,v1,B,157,213
|
|
48
|
+
E24,grader-1,1,v1,B,186,174
|
|
49
|
+
E24,grader-1,2,v2,A,174,186
|
|
50
|
+
E25,grader-1,1,v1,A,145,215
|
|
51
|
+
E25,grader-1,2,v2,A,215,145
|
|
52
|
+
E26,grader-1,1,v1,A,174,176
|
|
53
|
+
E26,grader-1,2,v2,A,176,174
|
|
54
|
+
E27,grader-1,1,v2,B,217,133
|
|
55
|
+
E27,grader-1,2,v2,A,217,133
|
|
56
|
+
E28,grader-1,1,v2,A,178,162
|
|
57
|
+
E28,grader-1,2,v2,A,178,162
|
|
58
|
+
E29,grader-1,1,v2,A,219,121
|
|
59
|
+
E29,grader-1,2,v1,B,121,219
|
|
60
|
+
E30,grader-1,1,v1,B,150,180
|
|
61
|
+
E30,grader-1,2,v1,B,150,180
|
|
62
|
+
E31,grader-1,1,v1,A,179,221
|
|
63
|
+
E31,grader-1,2,v2,A,221,179
|
|
64
|
+
E32,grader-1,1,v1,B,138,182
|
|
65
|
+
E32,grader-1,2,v1,A,138,182
|
|
66
|
+
E33,grader-1,1,v1,A,167,223
|
|
67
|
+
E33,grader-1,2,tie,,,
|
|
68
|
+
E34,grader-1,1,v2,A,184,126
|
|
69
|
+
E34,grader-1,2,v1,B,126,184
|
|
70
|
+
E35,grader-1,1,v2,A,225,155
|
|
71
|
+
E35,grader-1,2,v2,A,225,155
|
|
72
|
+
E36,grader-1,1,v2,A,186,184
|
|
73
|
+
E36,grader-1,2,v2,A,186,184
|
|
74
|
+
E37,grader-1,1,v2,A,227,143
|
|
75
|
+
E37,grader-1,2,v2,B,227,143
|
|
76
|
+
E38,grader-1,1,v2,A,188,172
|
|
77
|
+
E38,grader-1,2,v1,A,172,188
|
|
78
|
+
E39,grader-1,1,v2,A,229,131
|
|
79
|
+
E39,grader-1,2,v2,A,229,131
|
|
80
|
+
E40,grader-1,1,v1,A,160,190
|
|
81
|
+
E40,grader-1,2,v1,B,160,190
|
|
82
|
+
E41,grader-1,1,v1,A,189,151
|
|
83
|
+
E41,grader-1,2,v2,B,151,189
|
|
84
|
+
E42,grader-1,1,v1,A,148,192
|
|
85
|
+
E42,grader-1,2,v1,B,148,192
|
|
86
|
+
E43,grader-1,1,v2,A,153,177
|
|
87
|
+
E43,grader-1,2,v2,B,153,177
|
|
88
|
+
E44,grader-1,1,v2,A,194,136
|
|
89
|
+
E44,grader-1,2,v2,B,194,136
|
|
90
|
+
E45,grader-1,1,tie,,,
|
|
91
|
+
E45,grader-1,2,v1,B,165,155
|
|
92
|
+
E46,grader-1,1,v1,B,124,196
|
|
93
|
+
E46,grader-1,2,v2,A,196,124
|
|
94
|
+
E47,grader-1,1,v1,B,153,157
|
|
95
|
+
E47,grader-1,2,v1,B,153,157
|
|
96
|
+
E48,grader-1,1,v2,A,198,182
|
|
97
|
+
E48,grader-1,2,v2,A,198,182
|
|
98
|
+
E49,grader-1,1,v2,A,159,141
|
|
99
|
+
E49,grader-1,2,v2,A,159,141
|
|
100
|
+
E50,grader-1,1,v2,A,200,170
|
|
101
|
+
E50,grader-1,2,v2,A,200,170
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Onboarding email prompt — v1 vs v2
|
|
2
|
+
|
|
3
|
+
**Team:** Growth, Quillmark (fictional)
|
|
4
|
+
|
|
5
|
+
- 50 sign-up profiles (E01–E50). For each, prompt v1 and prompt v2 each wrote a welcome email.
|
|
6
|
+
- One LLM grader (`grader-1`) saw both emails for a profile and picked the better one, or `tie`.
|
|
7
|
+
It ran twice per profile (trial 1 and 2). Which email was shown as A was randomised on every run.
|
|
8
|
+
- `winner` is the prompt whose email the grader picked. `position` is where that email was shown
|
|
9
|
+
(A first, B second). `length` is the picked email's word count; `other_length` is the other's.
|
|
10
|
+
|
|
11
|
+
Data: `pairwise.csv`.
|
|
12
|
+
|
|
13
|
+
## Result so far
|
|
14
|
+
|
|
15
|
+
v2 won 53 comparisons, v1 won 43, 4 ties. Plan: switch production to v2 this sprint and use the
|
|
16
|
+
same grader setup for every future prompt change.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
ticket_id,prompt_version,reply_words,judge_score,human_score
|
|
2
|
+
TCK-4101,v1,97,3,
|
|
3
|
+
TCK-4102,v1,104,2,
|
|
4
|
+
TCK-4103,v1,91,5,
|
|
5
|
+
TCK-4104,v1,104,4,3
|
|
6
|
+
TCK-4105,v1,56,5,5
|
|
7
|
+
TCK-4106,v1,57,4,4
|
|
8
|
+
TCK-4107,v1,74,3,
|
|
9
|
+
TCK-4108,v1,76,3,
|
|
10
|
+
TCK-4109,v1,62,3,
|
|
11
|
+
TCK-4110,v1,59,5,5
|
|
12
|
+
TCK-4111,v1,82,2,
|
|
13
|
+
TCK-4112,v1,101,5,
|
|
14
|
+
TCK-4113,v1,71,4,4
|
|
15
|
+
TCK-4114,v1,57,3,
|
|
16
|
+
TCK-4115,v1,73,3,
|
|
17
|
+
TCK-4116,v1,61,2,
|
|
18
|
+
TCK-4117,v1,83,3,3
|
|
19
|
+
TCK-4118,v1,78,1,1
|
|
20
|
+
TCK-4119,v1,62,5,5
|
|
21
|
+
TCK-4120,v1,73,2,3
|
|
22
|
+
TCK-4121,v1,66,4,4
|
|
23
|
+
TCK-4122,v1,118,1,
|
|
24
|
+
TCK-4123,v1,57,2,
|
|
25
|
+
TCK-4124,v1,64,2,
|
|
26
|
+
TCK-4125,v1,106,5,
|
|
27
|
+
TCK-4126,v1,90,4,
|
|
28
|
+
TCK-4127,v1,56,2,
|
|
29
|
+
TCK-4128,v1,96,4,
|
|
30
|
+
TCK-4129,v1,63,3,
|
|
31
|
+
TCK-4130,v1,68,2,
|
|
32
|
+
TCK-4131,v1,71,5,
|
|
33
|
+
TCK-4132,v1,65,4,
|
|
34
|
+
TCK-4133,v1,108,4,
|
|
35
|
+
TCK-4134,v1,85,2,2
|
|
36
|
+
TCK-4135,v1,101,4,
|
|
37
|
+
TCK-4136,v1,107,4,
|
|
38
|
+
TCK-4137,v1,90,2,
|
|
39
|
+
TCK-4138,v1,86,3,
|
|
40
|
+
TCK-4139,v1,96,3,
|
|
41
|
+
TCK-4140,v1,92,3,3
|
|
42
|
+
TCK-4141,v1,65,5,5
|
|
43
|
+
TCK-4142,v1,76,4,4
|
|
44
|
+
TCK-4143,v1,83,1,1
|
|
45
|
+
TCK-4144,v1,85,3,
|
|
46
|
+
TCK-4145,v1,125,3,3
|
|
47
|
+
TCK-4146,v1,74,4,
|
|
48
|
+
TCK-4147,v1,70,3,
|
|
49
|
+
TCK-4148,v1,70,3,
|
|
50
|
+
TCK-4149,v2,229,5,4
|
|
51
|
+
TCK-4150,v2,198,5,
|
|
52
|
+
TCK-4151,v2,240,4,3
|
|
53
|
+
TCK-4152,v2,225,5,4
|
|
54
|
+
TCK-4153,v2,201,5,
|
|
55
|
+
TCK-4154,v2,228,3,
|
|
56
|
+
TCK-4155,v2,211,5,
|
|
57
|
+
TCK-4156,v2,186,5,
|
|
58
|
+
TCK-4157,v2,244,4,
|
|
59
|
+
TCK-4158,v2,246,4,
|
|
60
|
+
TCK-4159,v2,188,5,
|
|
61
|
+
TCK-4160,v2,223,5,
|
|
62
|
+
TCK-4161,v2,211,5,4
|
|
63
|
+
TCK-4162,v2,250,3,
|
|
64
|
+
TCK-4163,v2,198,3,3
|
|
65
|
+
TCK-4164,v2,141,3,
|
|
66
|
+
TCK-4165,v2,239,5,
|
|
67
|
+
TCK-4166,v2,232,5,
|
|
68
|
+
TCK-4167,v2,198,5,
|
|
69
|
+
TCK-4168,v2,157,4,
|
|
70
|
+
TCK-4169,v2,156,4,
|
|
71
|
+
TCK-4170,v2,138,3,
|
|
72
|
+
TCK-4171,v2,202,5,
|
|
73
|
+
TCK-4172,v2,214,3,3
|
|
74
|
+
TCK-4173,v2,228,5,
|
|
75
|
+
TCK-4174,v2,140,2,2
|
|
76
|
+
TCK-4175,v2,210,4,3
|
|
77
|
+
TCK-4176,v2,143,4,
|
|
78
|
+
TCK-4177,v2,227,4,4
|
|
79
|
+
TCK-4178,v2,170,3,
|
|
80
|
+
TCK-4179,v2,237,5,
|
|
81
|
+
TCK-4180,v2,148,3,
|
|
82
|
+
TCK-4181,v2,238,3,
|
|
83
|
+
TCK-4182,v2,220,3,
|
|
84
|
+
TCK-4183,v2,200,4,3
|
|
85
|
+
TCK-4184,v2,184,4,3
|
|
86
|
+
TCK-4185,v2,157,5,
|
|
87
|
+
TCK-4186,v2,143,1,
|
|
88
|
+
TCK-4187,v2,174,4,3
|
|
89
|
+
TCK-4188,v2,159,1,1
|
|
90
|
+
TCK-4189,v2,150,4,
|
|
91
|
+
TCK-4190,v2,258,4,2
|
|
92
|
+
TCK-4191,v2,170,5,5
|
|
93
|
+
TCK-4192,v2,250,5,5
|
|
94
|
+
TCK-4193,v2,248,5,
|
|
95
|
+
TCK-4194,v2,220,5,
|
|
96
|
+
TCK-4195,v2,169,5,
|
|
97
|
+
TCK-4196,v2,237,2,
|