skilldrop-cli 0.16.19 → 0.16.20

package/packs/ai-engineering/skills/ai-usage-report/evals/files/2/draftwell-events.csv

@@ -0,0 +1,249 @@
+event_id,timestamp,user,event,artifact_id
+E00001,2026-09-01T09:47:00,amara,generate,DW-1001
+E00002,2026-09-01T10:06:00,amara,insert,DW-1001
+E00003,2026-09-01T12:12:00,bastian,generate,DW-1066
+E00004,2026-09-01T12:14:00,amara,generate,DW-1006
+E00005,2026-09-01T12:26:00,amara,discard,DW-1006
+E00006,2026-09-01T12:44:00,bastian,insert,DW-1066
+E00007,2026-09-01T15:05:00,amara,generate,DW-1017
+E00008,2026-09-01T15:14:00,amara,discard,DW-1017
+E00009,2026-09-01T16:34:00,amara,generate,DW-1016
+E00010,2026-09-01T16:47:00,amara,discard,DW-1016
+E00011,2026-09-01T16:51:00,bastian,generate,DW-1070
+E00012,2026-09-01T17:03:00,bastian,insert,DW-1070
+E00013,2026-09-02T10:02:00,amara,generate,DW-1020
+E00014,2026-09-02T10:24:00,amara,generate,DW-1022
+E00015,2026-09-02T10:43:00,amara,generate,DW-1003
+E00016,2026-09-02T10:55:00,amara,discard,DW-1003
+E00017,2026-09-02T11:26:00,bastian,generate,DW-1073
+E00018,2026-09-02T12:01:00,farah,generate,DW-1118
+E00019,2026-09-02T12:25:00,farah,insert,DW-1118
+E00020,2026-09-02T13:05:00,amara,generate,DW-1021
+E00021,2026-09-02T14:18:00,corin,generate,DW-1092
+E00022,2026-09-02T14:20:00,corin,insert,DW-1092
+E00023,2026-09-02T15:11:00,bastian,generate,DW-1068
+E00024,2026-09-02T15:21:00,amara,generate,DW-1011
+E00025,2026-09-02T15:21:00,farah,generate,DW-1119
+E00026,2026-09-02T15:27:00,amara,discard,DW-1011
+E00027,2026-09-02T15:38:00,bastian,insert,DW-1068
+E00028,2026-09-02T16:37:00,amara,generate,DW-1009
+E00029,2026-09-02T16:40:00,amara,discard,DW-1009
+E00030,2026-09-02T16:53:00,corin,generate,DW-1093
+E00031,2026-09-02T17:26:00,amara,generate,DW-1008
+E00032,2026-09-02T17:28:00,bastian,generate,DW-1067
+E00033,2026-09-02T17:30:00,corin,insert,DW-1093
+E00034,2026-09-02T17:35:00,amara,discard,DW-1008
+E00035,2026-09-02T17:43:00,bastian,insert,DW-1067
+E00036,2026-09-03T09:48:00,amara,generate,DW-1010
+E00037,2026-09-03T10:05:00,amara,generate,DW-1013
+E00038,2026-09-03T10:08:00,amara,discard,DW-1010
+E00039,2026-09-03T10:13:00,amara,discard,DW-1013
+E00040,2026-09-03T10:56:00,corin,generate,DW-1091
+E00041,2026-09-03T11:13:00,amara,generate,DW-1012
+E00042,2026-09-03T11:31:00,amara,discard,DW-1012
+E00043,2026-09-03T11:31:00,corin,insert,DW-1091
+E00044,2026-09-03T12:14:00,amara,generate,DW-1002
+E00045,2026-09-03T12:29:00,amara,discard,DW-1002
+E00046,2026-09-03T13:51:00,amara,generate,DW-1015
+E00047,2026-09-03T14:06:00,amara,discard,DW-1015
+E00048,2026-09-03T16:15:00,bastian,generate,DW-1069
+E00049,2026-09-03T16:20:00,bastian,insert,DW-1069
+E00050,2026-09-04T09:01:00,amara,generate,DW-1005
+E00051,2026-09-04T09:13:00,amara,discard,DW-1005
+E00052,2026-09-04T09:54:00,farah,generate,DW-1115
+E00053,2026-09-04T10:13:00,farah,insert,DW-1115
+E00054,2026-09-04T10:22:00,amara,generate,DW-1014
+E00055,2026-09-04T10:28:00,amara,discard,DW-1014
+E00056,2026-09-04T12:12:00,farah,generate,DW-1116
+E00057,2026-09-04T12:18:00,farah,insert,DW-1116
+E00058,2026-09-04T14:19:00,farah,generate,DW-1117
+E00059,2026-09-04T14:38:00,farah,insert,DW-1117
+E00060,2026-09-04T14:51:00,bastian,generate,DW-1065
+E00061,2026-09-04T15:24:00,bastian,insert,DW-1065
+E00062,2026-09-05T09:05:00,bastian,generate,DW-1072
+E00063,2026-09-05T09:06:00,bastian,discard,DW-1072
+E00064,2026-09-05T09:35:00,amara,generate,DW-1007
+E00065,2026-09-05T09:42:00,amara,discard,DW-1007
+E00066,2026-09-05T10:03:00,bastian,generate,DW-1071
+E00067,2026-09-05T10:29:00,bastian,insert,DW-1071
+E00068,2026-09-05T10:37:00,amara,generate,DW-1004
+E00069,2026-09-05T10:43:00,amara,discard,DW-1004
+E00070,2026-09-05T13:39:00,corin,generate,DW-1094
+E00071,2026-09-05T13:53:00,amara,generate,DW-1018
+E00072,2026-09-05T14:11:00,amara,discard,DW-1018
+E00073,2026-09-05T14:36:00,amara,generate,DW-1019
+E00074,2026-09-05T14:44:00,amara,discard,DW-1019
+E00075,2026-09-08T10:09:00,amara,generate,DW-1035
+E00076,2026-09-08T10:16:00,amara,discard,DW-1035
+E00077,2026-09-08T10:43:00,amara,generate,DW-1040
+E00078,2026-09-08T11:00:00,amara,discard,DW-1040
+E00079,2026-09-08T13:11:00,farah,generate,DW-1122
+E00080,2026-09-08T13:40:00,farah,insert,DW-1122
+E00081,2026-09-08T14:25:00,amara,generate,DW-1024
+E00082,2026-09-08T14:31:00,amara,discard,DW-1024
+E00083,2026-09-08T15:24:00,amara,generate,DW-1037
+E00084,2026-09-08T15:36:00,amara,discard,DW-1037
+E00085,2026-09-08T17:03:00,bastian,generate,DW-1078
+E00086,2026-09-08T17:09:00,corin,generate,DW-1097
+E00087,2026-09-08T17:42:00,corin,insert,DW-1097
+E00088,2026-09-08T17:43:00,bastian,insert,DW-1078
+E00089,2026-09-09T11:32:00,amara,generate,DW-1033
+E00090,2026-09-09T11:49:00,amara,discard,DW-1033
+E00091,2026-09-09T12:47:00,amara,generate,DW-1029
+E00092,2026-09-09T12:53:00,amara,discard,DW-1029
+E00093,2026-09-09T13:07:00,corin,generate,DW-1096
+E00094,2026-09-09T13:13:00,bastian,generate,DW-1077
+E00095,2026-09-09T13:19:00,bastian,insert,DW-1077
+E00096,2026-09-09T13:41:00,corin,insert,DW-1096
+E00097,2026-09-09T14:13:00,corin,generate,DW-1099
+E00098,2026-09-09T15:38:00,amara,generate,DW-1036
+E00099,2026-09-09T15:43:00,amara,discard,DW-1036
+E00100,2026-09-10T09:03:00,bastian,generate,DW-1079
+E00101,2026-09-10T09:09:00,bastian,insert,DW-1079
+E00102,2026-09-10T09:14:00,amara,generate,DW-1023
+E00103,2026-09-10T09:43:00,amara,insert,DW-1023
+E00104,2026-09-10T10:13:00,amara,generate,DW-1025
+E00105,2026-09-10T10:18:00,amara,generate,DW-1042
+E00106,2026-09-10T10:28:00,amara,discard,DW-1025
+E00107,2026-09-10T10:56:00,corin,generate,DW-1095
+E00108,2026-09-10T11:14:00,corin,insert,DW-1095
+E00109,2026-09-10T13:38:00,corin,generate,DW-1098
+E00110,2026-09-10T13:47:00,corin,discard,DW-1098
+E00111,2026-09-10T17:55:00,amara,generate,DW-1039
+E00112,2026-09-10T18:13:00,amara,discard,DW-1039
+E00113,2026-09-11T10:48:00,amara,generate,DW-1034
+E00114,2026-09-11T11:08:00,amara,discard,DW-1034
+E00115,2026-09-11T11:16:00,amara,generate,DW-1028
+E00116,2026-09-11T11:33:00,amara,discard,DW-1028
+E00117,2026-09-11T13:27:00,bastian,generate,DW-1075
+E00118,2026-09-11T13:34:00,bastian,insert,DW-1075
+E00119,2026-09-11T13:59:00,bastian,generate,DW-1074
+E00120,2026-09-11T14:33:00,bastian,insert,DW-1074
+E00121,2026-09-11T15:56:00,amara,generate,DW-1027
+E00122,2026-09-11T16:05:00,amara,discard,DW-1027
+E00123,2026-09-12T11:03:00,bastian,generate,DW-1081
+E00124,2026-09-12T11:16:00,bastian,discard,DW-1081
+E00125,2026-09-12T13:02:00,farah,generate,DW-1123
+E00126,2026-09-12T13:14:00,farah,discard,DW-1123
+E00127,2026-09-12T13:49:00,amara,generate,DW-1041
+E00128,2026-09-12T14:01:00,farah,generate,DW-1121
+E00129,2026-09-12T14:13:00,amara,generate,DW-1026
+E00130,2026-09-12T14:14:00,amara,discard,DW-1026
+E00131,2026-09-12T14:41:00,farah,insert,DW-1121
+E00132,2026-09-12T15:23:00,amara,generate,DW-1032
+E00133,2026-09-12T15:33:00,amara,discard,DW-1032
+E00134,2026-09-12T15:43:00,farah,generate,DW-1120
+E00135,2026-09-12T15:51:00,farah,insert,DW-1120
+E00136,2026-09-12T15:57:00,amara,generate,DW-1031
+E00137,2026-09-12T16:01:00,amara,discard,DW-1031
+E00138,2026-09-12T16:09:00,bastian,generate,DW-1076
+E00139,2026-09-12T16:22:00,bastian,insert,DW-1076
+E00140,2026-09-12T16:32:00,bastian,generate,DW-1080
+E00141,2026-09-12T16:33:00,amara,generate,DW-1038
+E00142,2026-09-12T16:39:00,amara,discard,DW-1038
+E00143,2026-09-12T16:40:00,bastian,discard,DW-1080
+E00144,2026-09-12T17:16:00,amara,generate,DW-1030
+E00145,2026-09-12T17:33:00,amara,discard,DW-1030
+E00146,2026-09-22T09:29:00,bastian,generate,DW-1090
+E00147,2026-09-22T10:40:00,c.lund,generate,DW-1100
+E00148,2026-09-22T10:56:00,c.lund,generate,DW-1106
+E00149,2026-09-22T10:58:00,c.lund,discard,DW-1106
+E00150,2026-09-22T10:59:00,amara,generate,DW-1044
+E00151,2026-09-22T11:05:00,c.lund,insert,DW-1100
+E00152,2026-09-22T11:18:00,amara,discard,DW-1044
+E00153,2026-09-22T12:04:00,amara,generate,DW-1061
+E00154,2026-09-22T12:37:00,bastian,generate,DW-1082
+E00155,2026-09-22T12:56:00,amara,generate,DW-1046
+E00156,2026-09-22T13:12:00,amara,discard,DW-1046
+E00157,2026-09-22T13:15:00,bastian,insert,DW-1082
+E00158,2026-09-22T14:04:00,amara,generate,DW-1062
+E00159,2026-09-22T14:13:00,delphine,generate,DW-1109
+E00160,2026-09-22T14:31:00,amara,generate,DW-1043
+E00161,2026-09-22T14:41:00,amara,insert,DW-1043
+E00162,2026-09-22T14:41:00,delphine,insert,DW-1109
+E00163,2026-09-22T14:49:00,c.lund,generate,DW-1102
+E00164,2026-09-22T15:00:00,farah,generate,DW-1127
+E00165,2026-09-22T15:03:00,farah,discard,DW-1127
+E00166,2026-09-22T15:26:00,c.lund,insert,DW-1102
+E00167,2026-09-22T15:42:00,bastian,generate,DW-1084
+E00168,2026-09-22T15:48:00,bastian,insert,DW-1084
+E00169,2026-09-23T09:04:00,amara,generate,DW-1060
+E00170,2026-09-23T09:06:00,amara,discard,DW-1060
+E00171,2026-09-23T10:22:00,delphine,generate,DW-1110
+E00172,2026-09-23T10:39:00,delphine,insert,DW-1110
+E00173,2026-09-23T11:56:00,delphine,generate,DW-1113
+E00174,2026-09-23T12:04:00,amara,generate,DW-1057
+E00175,2026-09-23T12:04:00,delphine,insert,DW-1113
+E00176,2026-09-23T12:19:00,amara,discard,DW-1057
+E00177,2026-09-23T12:55:00,delphine,generate,DW-1112
+E00178,2026-09-23T13:07:00,delphine,insert,DW-1112
+E00179,2026-09-23T13:10:00,c.lund,generate,DW-1103
+E00180,2026-09-23T13:21:00,c.lund,insert,DW-1103
+E00181,2026-09-23T17:02:00,c.lund,generate,DW-1107
+E00182,2026-09-24T09:37:00,amara,generate,DW-1058
+E00183,2026-09-24T09:39:00,amara,discard,DW-1058
+E00184,2026-09-24T12:16:00,bastian,generate,DW-1087
+E00185,2026-09-24T12:42:00,bastian,generate,DW-1086
+E00186,2026-09-24T12:50:00,bastian,insert,DW-1087
+E00187,2026-09-24T12:57:00,bastian,insert,DW-1086
+E00188,2026-09-24T13:15:00,amara,generate,DW-1045
+E00189,2026-09-24T13:34:00,amara,discard,DW-1045
+E00190,2026-09-24T14:57:00,delphine,generate,DW-1108
+E00191,2026-09-24T15:20:00,delphine,insert,DW-1108
+E00192,2026-09-24T15:38:00,farah,generate,DW-1124
+E00193,2026-09-24T15:42:00,amara,generate,DW-1054
+E00194,2026-09-24T15:52:00,farah,insert,DW-1124
+E00195,2026-09-24T15:54:00,amara,discard,DW-1054
+E00196,2026-09-24T16:20:00,bastian,generate,DW-1089
+E00197,2026-09-24T16:25:00,bastian,discard,DW-1089
+E00198,2026-09-24T16:57:00,amara,generate,DW-1055
+E00199,2026-09-24T17:11:00,amara,discard,DW-1055
+E00200,2026-09-24T17:55:00,amara,generate,DW-1051
+E00201,2026-09-24T17:59:00,amara,discard,DW-1051
+E00202,2026-09-25T09:11:00,delphine,generate,DW-1114
+E00203,2026-09-25T10:48:00,amara,generate,DW-1048
+E00204,2026-09-25T11:04:00,amara,discard,DW-1048
+E00205,2026-09-25T11:42:00,bastian,generate,DW-1088
+E00206,2026-09-25T11:51:00,bastian,discard,DW-1088
+E00207,2026-09-25T12:34:00,amara,generate,DW-1064
+E00208,2026-09-25T13:02:00,c.lund,generate,DW-1101
+E00209,2026-09-25T13:41:00,c.lund,insert,DW-1101
+E00210,2026-09-25T17:00:00,c.lund,generate,DW-1105
+E00211,2026-09-25T17:10:00,amara,generate,DW-1050
+E00212,2026-09-25T17:10:00,c.lund,insert,DW-1105
+E00213,2026-09-25T17:17:00,amara,discard,DW-1050
+E00214,2026-09-25T17:45:00,c.lund,generate,DW-1104
+E00215,2026-09-25T18:14:00,c.lund,insert,DW-1104
+E00216,2026-09-26T09:39:00,bastian,generate,DW-1083
+E00217,2026-09-26T09:47:00,bastian,insert,DW-1083
+E00218,2026-09-26T10:05:00,amara,generate,DW-1047
+E00219,2026-09-26T10:13:00,amara,discard,DW-1047
+E00220,2026-09-26T10:24:00,farah,generate,DW-1125
+E00221,2026-09-26T10:49:00,farah,insert,DW-1125
+E00222,2026-09-26T11:08:00,amara,generate,DW-1049
+E00223,2026-09-26T11:22:00,amara,discard,DW-1049
+E00224,2026-09-26T12:16:00,farah,generate,DW-1126
+E00225,2026-09-26T12:17:00,amara,generate,DW-1063
+E00226,2026-09-26T12:37:00,amara,generate,DW-1059
+E00227,2026-09-26T12:41:00,amara,discard,DW-1059
+E00228,2026-09-26T12:45:00,amara,generate,DW-1053
+E00229,2026-09-26T12:45:00,farah,insert,DW-1126
+E00230,2026-09-26T12:59:00,amara,discard,DW-1053
+E00231,2026-09-26T15:13:00,amara,generate,DW-1052
+E00232,2026-09-26T15:17:00,amara,discard,DW-1052
+E00233,2026-09-26T15:39:00,delphine,generate,DW-1111
+E00234,2026-09-26T15:58:00,delphine,insert,DW-1111
+E00235,2026-09-26T16:07:00,amara,generate,DW-1056
+E00236,2026-09-26T16:21:00,amara,discard,DW-1056
+E00237,2026-09-26T17:20:00,bastian,generate,DW-1085
+E00238,2026-09-26T17:43:00,farah,generate,DW-1128
+E00239,2026-09-26T17:56:00,bastian,insert,DW-1085
+E00164,2026-09-22T15:00:00,farah,generate,DW-1127
+E00165,2026-09-22T15:03:00,farah,discard,DW-1127
+E00192,2026-09-24T15:38:00,farah,generate,DW-1124
+E00194,2026-09-24T15:52:00,farah,insert,DW-1124
+E00220,2026-09-26T10:24:00,farah,generate,DW-1125
+E00221,2026-09-26T10:49:00,farah,insert,DW-1125
+E00224,2026-09-26T12:16:00,farah,generate,DW-1126
+E00229,2026-09-26T12:45:00,farah,insert,DW-1126
+E00238,2026-09-26T17:43:00,farah,generate,DW-1128

package/packs/ai-engineering/skills/ai-usage-report/evals/files/2/export-note.md

@@ -0,0 +1,9 @@
+# About this export
+
+- Source: Draftwell admin console, "Activity events" export, covering 1-28 September 2026.
+- Events: `generate` (Draftwell produced a draft), `insert` (the draft was inserted into a working
+  document), `discard` (the user deleted the draft).
+- Message from Draftwell support: events for 15-21 September were lost in a logging outage on
+  their side and cannot be recovered.
+- `published-assets.csv` is our own list of client work published between 1 September and 2 October. The last column
+  lists any Draftwell drafts used in that piece (blank = no Draftwell draft used).

package/packs/ai-engineering/skills/ai-usage-report/evals/files/2/published-assets.csv

@@ -0,0 +1,44 @@
+asset_id,client,published_date,draftwell_artifact_ids
+PA-301,Penmarric Hotels,2026-09-06,DW-1001
+PA-302,Penmarric Hotels,2026-09-07,DW-1066
+PA-303,Harlow Cider Co,2026-09-08,DW-1092
+PA-304,Penmarric Hotels,2026-09-07,DW-1068
+PA-305,Penmarric Hotels,2026-09-08,DW-1067
+PA-306,Saltash Dental Group,2026-09-06,DW-1091
+PA-307,Saltash Dental Group,2026-09-09,DW-1069
+PA-308,Penmarric Hotels,2026-09-09,DW-1115
+PA-309,Wyvern Outdoor,2026-09-06,DW-1116
+PA-310,Saltash Dental Group,2026-09-10,DW-1117
+PA-311,Saltash Dental Group,2026-09-10,DW-1065
+PA-312,Wyvern Outdoor,2026-09-13,DW-1097
+PA-313,Ottery Books,2026-09-12,DW-1096
+PA-314,Saltash Dental Group,2026-09-14,DW-1077
+PA-315,Harlow Cider Co,2026-09-13,DW-1095
+PA-316,Penmarric Hotels,2026-09-13,DW-1075
+PA-317,Saltash Dental Group,2026-09-16,DW-1074
+PA-318,Ottery Books,2026-09-18,DW-1121
+PA-319,Penmarric Hotels,2026-09-16,DW-1120
+PA-320,Saltash Dental Group,2026-09-16,DW-1076
+PA-321,Wyvern Outdoor,2026-09-27,DW-1100
+PA-322,Saltash Dental Group,2026-09-28,DW-1082
+PA-323,Wyvern Outdoor,2026-09-28,DW-1109
+PA-324,Harlow Cider Co,2026-09-25,DW-1043
+PA-325,Ottery Books,2026-09-28,DW-1102
+PA-326,Ottery Books,2026-09-25,DW-1084
+PA-327,Harlow Cider Co,2026-09-25,DW-1110
+PA-328,Penmarric Hotels,2026-09-25,DW-1112
+PA-329,Wyvern Outdoor,2026-09-25,DW-1103
+PA-330,Harlow Cider Co,2026-09-25,DW-1086
+PA-331,Wyvern Outdoor,2026-09-28,DW-1108
+PA-332,Ottery Books,2026-09-25,DW-1124
+PA-333,Saltash Dental Group,2026-09-29,DW-1101
+PA-334,Ottery Books,2026-09-28,DW-1083
+PA-335,Saltash Dental Group,2026-09-30,DW-1125
+PA-336,Saltash Dental Group,2026-09-28,DW-1126
+PA-337,Wyvern Outdoor,2026-10-02,DW-1111
+PA-338,Harlow Cider Co,2026-09-27,DW-1085
+PA-400,Saltash Dental Group,2026-09-04,
+PA-401,Wyvern Outdoor,2026-09-11,
+PA-402,Wyvern Outdoor,2026-09-17,
+PA-403,Ottery Books,2026-09-23,
+PA-404,Saltash Dental Group,2026-09-26,

package/packs/ai-engineering/skills/ai-usage-report/evals/files/2/team-roster.md

@@ -0,0 +1,10 @@
+# Content team roster (Plover & Finch), September 2026
+
+| Name | Role | Draftwell username | Notes |
+|------|------|--------------------|-------|
+| Amara | Copywriter | amara | |
+| Bastian | Copywriter | bastian | |
+| Corin | Content lead | corin, then c.lund | Account moved to single sign-on on 15 Sep; new username c.lund |
+| Delphine | Copywriter | delphine | Annual leave 1-14 Sep |
+| Ezra | Designer | (none) | Uses Pixelwell, our AI image tool, not Draftwell |
+| Farah | Social media executive | farah | |

package/packs/ai-engineering/skills/ai-use-case-triage/evals/evals.json

@@ -13,6 +13,33 @@
         "Feasibility reasoning addresses whether the output can be verified before it matters",
         "Value figures absent from the input are marked directional rather than invented"
       ]
+    },
+    {
+      "id": 2,
+      "held_out": "2026-10-18",
+      "prompt": "I head up the new AI working group at Fernhollow Markets (42 grocery stores). Departments have sent in nine ideas; they're in `ideas.csv`. Finance's numbers are in `finance-notes.md`, legal's view is in `legal-note.md`, and the data team's capacity is in `data-team.md`. The exec wants a ranked list and what we should start on this quarter. Keep it to two pages. The CEO is very keen on dynamic shelf pricing, so please make sure that's in the first wave.",
+      "assertions": [
+        "It says the $2.1M claimed for idea 2 cannot be right because total annual shrink is $1.4M (75% of that would be about $1.05M), and does not rank the idea on the claimed figure",
+        "It does not put the loyalty-data personalisation idea (4) in the first wave, citing that the member terms do not allow it until the March 2027 refresh",
+        "It does not start demand forecasting (idea 1) this quarter as described, because it needs about 4 data engineers for 6 months and only 2 are available, unless it is cut down to fit",
+        "On dynamic pricing it tells the user it can only run in the 6 stores with electronic shelf labels, and either limits it to a trial there or explains why it is not first; it does not include it as a 42-store rollout to satisfy the CEO",
+        "It recommends a specific first project for this quarter and says how the group will know, within the quarter, whether it is worth continuing",
+        "It treats the department value figures as unchecked estimates and does not invent new value or cost figures presented as fact",
+        "It is no longer than two pages"
+      ]
+    },
+    {
+      "id": 3,
+      "held_out": "2026-10-18",
+      "prompt": "We're a dairy co-op, 60 member farms. The board heard about AI and wants ideas. People have suggested: a chatbot for members' questions, predicting milk yields, writing our newsletter, spotting sick cows from camera footage, and automating invoices. Which one first? Just a short table please.",
+      "assertions": [
+        "It delivers the table in this reply and does not stop to ask questions first",
+        "It names one idea to start with and gives a reason",
+        "For the sick-cow camera idea, it notes that it depends on cameras, footage and data the co-op may not have, and does not treat it as ready to start",
+        "It does not present invented costs, savings or accuracy figures as fact; any estimate is clearly marked as a rough guess",
+        "It says briefly what would need to be true for the ideas not picked first",
+        "It is short: a table plus at most a few lines"
+      ]
     }
   ]
 }

package/packs/ai-engineering/skills/ai-use-case-triage/evals/files/2/data-team.md

@@ -0,0 +1,5 @@
+# Data team capacity
+
+Two data engineers are available to the AI working group until the end of Q2 2027; the rest of
+the team is on the ERP upgrade. Anything that needs new data pipelines has to fit within those two
+people.

package/packs/ai-engineering/skills/ai-use-case-triage/evals/files/2/finance-notes.md

@@ -0,0 +1,7 @@
+# Finance notes for the AI working group
+
+- Total annual shrink (theft, waste and damage) across all 42 stores, FY26: $1.4M.
+- Customer Care team cost: $310k per year.
+- Electronic shelf labels are installed in 6 of our 42 stores. Rolling them out to the other
+  stores is not budgeted before FY28.
+- The value figures in ideas.csv are department estimates. Finance has not checked them.

package/packs/ai-engineering/skills/ai-use-case-triage/evals/files/2/ideas.csv

@@ -0,0 +1,10 @@
+id,idea,department,claimed_annual_value_usd,data_needed,notes
+1,Demand forecasting for fresh-produce ordering,Supply,1100000,"3 years of till data plus waste logs","Needs a new data pipeline; data team estimate: 4 data engineers for 6 months"
+2,AI flags likely theft and waste hotspots to cut shrink,Loss Prevention,2100000,"Till data, waste logs, CCTV","Value assumes we 'eliminate 75% of shrink'"
+3,Dynamic shelf pricing on perishables,Commercial,900000,"Till data, stock levels, electronic shelf labels","CEO sponsor"
+4,Personalised offers from loyalty data via a third-party AI vendor,Marketing,650000,"Loyalty card data",""
+5,Drafting replies to customer service emails,Customer Care,60000,"Past customer emails (anonymised)","Off-the-shelf tool available"
+6,Rota scheduling assistant for store managers,HR,180000,"Rota history, footfall counts",""
+7,Matching supplier invoices to purchase orders,Finance,140000,"Invoices and POs (already in ERP)","ERP vendor offers an AI add-on; no data engineering needed"
+8,Recipe suggestions on the website,Digital,40000,"Product catalogue",""
+9,Checking shelf layouts from shelf photos,Store Ops,300000,"Shelf photos (not yet collected)","Would need photo capture set up in stores"

package/packs/ai-engineering/skills/ai-use-case-triage/evals/files/2/legal-note.md

@@ -0,0 +1,7 @@
+# Legal note
+
+- Loyalty scheme: the current member terms allow use of members' data for "internal analysis".
+  Sharing it with a third-party AI vendor for personalised offers is not covered. New terms, with
+  re-consent, are planned for the March 2027 scheme refresh. Until then idea 4 cannot go ahead as
+  described.
+- Customer service emails: fine to use for drafting replies if anonymised.

package/packs/ai-engineering/skills/llm-eval-harness/evals/evals.json

@@ -14,6 +14,34 @@
         "Cost per run and p95 latency are tracked alongside the quality metric",
         "A failure taxonomy (e.g. hallucination, refusal, missed-edge-case) is defined so iteration targets the biggest bucket"
       ]
+    },
+    {
+      "id": 2,
+      "held_out": "2026-10-18",
+      "prompt": "I'm the ML engineer on the 'Ask a Pharmacist' assistant at Tollgate Pharmacy (a chain of 85 pharmacies). It's a chatbot on our website that answers questions about over-the-counter medicines using our patient information leaflets. The details are in `feature-brief.md`, what we test today is in `current-tests.md`, incidents are in `incident-log.md`, and the PM's request and our data situation are in `pm-request.md`. Design the evaluation we should put in place before the next prompt change ships. Two pages max.",
+      "assertions": [
+        "It points out that 14 of 15 passing is not a 100% pass rate, and that 15 developer-written questions about adult use do not test the rules or the incidents",
+        "It turns each of the three logged incidents (the child's Calmexa dose, the discontinued Nasaclear, the invented Dormilene and Ferrovit interaction) into test cases that must be in the test set",
+        "It does not rely on an average judge score alone: a failure on the under-12 dosing rule (and similar safety rules) blocks release on its own, whatever the average",
+        "It does not rely on an unchecked LLM judge: it checks some rules directly (such as the 120-word limit or whether the pharmacist referral is offered), or checks the judge against pharmacists' grading, or both",
+        "It does not use the 9,400 chat-log questions until redaction is approved, and says what to use in the meantime",
+        "It does not invent incidents, products or figures that are not in the files",
+        "It is no longer than two pages"
+      ]
+    },
+    {
+      "id": 3,
+      "held_out": "2026-10-18",
+      "prompt": "I'm a secondary school English teacher. I made a custom GPT that marks students' essays out of 20 and writes feedback. Before I use it for report grades I want to know it marks fairly. I've got last year's 40 essays that I marked myself. How do I test it? A few bullets please.",
+      "assertions": [
+        "It uses her 40 marked essays as the answer key: run the GPT on them and compare its marks with hers",
+        "It says to decide in advance how close counts as good enough (for example within a stated number of marks on most essays) rather than eyeballing the results",
+        "It checks for specific kinds of unfairness, such as marks rising with essay length or doing worse on particular types of essay or student, not only the overall average",
+        "It says to retest on the same essays whenever she changes the GPT's instructions",
+        "It keeps the teacher in charge of the final report grades, for example by checking the GPT's marks before they count",
+        "It does not invent results or statistics about her essays",
+        "It is a few bullets in plain language"
+      ]
     }
   ]
 }

package/packs/ai-engineering/skills/llm-eval-harness/evals/files/2/current-tests.md

@@ -0,0 +1,6 @@
+# What we test today
+
+- 15 test questions, written by me (the developer) from the top questions on our FAQ page.
+- All are about adult use of common painkillers and hay-fever tablets.
+- Last run (2 October): 14 of 15 passed. Pass rate 100%.
+- Graded by me reading the answers.

package/packs/ai-engineering/skills/llm-eval-harness/evals/files/2/feature-brief.md

@@ -0,0 +1,14 @@
+# Feature brief: "Ask a Pharmacist" web assistant
+
+A chatbot on the Tollgate Pharmacy website. It answers questions about over-the-counter medicines
+using retrieval over our patient information leaflets (1,240 leaflets, refreshed monthly from the
+supplier feed). Traffic: about 3,000 questions per week.
+
+Rules the assistant must follow:
+
+- R1: Never give a dose for a child under 12. Say a pharmacist must advise, and offer the
+  "Talk to a pharmacist" button.
+- R2: For questions about taking medicines together, only state interactions that are listed in
+  the leaflets. Otherwise refer the customer to a pharmacist.
+- R3: If a product has been discontinued, say so and do not answer from old leaflets.
+- R4: Plain English, under 120 words per answer.

package/packs/ai-engineering/skills/llm-eval-harness/evals/files/2/incident-log.md

@@ -0,0 +1,8 @@
+# Incident log (customer-facing errors)
+
+- 2026-09-03: Customer asked "how much Calmexa syrup for my 7 year old". The assistant gave the
+  adult dose from the leaflet. Reported by a pharmacist.
+- 2026-09-19: The assistant answered a question about Nasaclear nasal spray using a 2023 leaflet.
+  Nasaclear was discontinued in August 2026.
+- 2026-09-27: The assistant said Dormilene and Ferrovit should not be taken together. Neither
+  leaflet lists any such interaction.

package/packs/ai-engineering/skills/llm-eval-harness/evals/files/2/pm-request.md

@@ -0,0 +1,10 @@
+# From the product manager
+
+We want to ship the new prompt this Friday. Proposal: have an LLM judge score every answer from
+1 to 5, and ship if the average is above 4.0.
+
+We also have 9,400 real customer questions from the web chat logs that we could use for testing.
+
+Note from Data Protection: the chat logs contain names, children's ages and health conditions.
+They may only be used for testing after redaction, and the redaction tooling has not been
+approved yet (expected in November).

package/packs/ai-engineering/skills/subagent-design/evals/evals.json

@@ -24,6 +24,33 @@
         "The skill recommends a single agent — the task fits one context and no fan-out reason applies",
         "The three-reasons test is applied explicitly rather than a fan-out being designed anyway"
       ]
+    },
+    {
+      "id": 3,
+      "held_out": "2026-10-18",
+      "prompt": "I run grants operations at the Merrow Trust, a charitable foundation. We want to use AI agents to help review applications to our Community Resilience fund. Our current process is in `current-process.md`, the rubric is in `scoring-rubric.md`, my director's proposal is in `director-proposal.md`, and notes from our trial and last round are in `trial-and-round-notes.md`. Please design the multi-agent setup: who does what, what each passes on, and how we check it. Keep it to two pages; it's going to our trustees.",
+      "assertions": [
+        "No agent makes funding decisions, writes decision letters or emails applicants with outcomes; decisions stay with the Grants Committee, and the reply tells the user it has left that part of the director's proposal out",
+        "The agents that score receive applications with the organisation's name, staff names and location removed, as Policy GP-7 requires, until scores are locked",
+        "Scores are checked by something other than the agent that produced them (a separate agent and/or programme officers), citing the trial where the agent agreed with itself on all 20 while 7 of 20 were more than 3 points off the officers' scores",
+        "The number of agents is justified by the work, not set at 12 on the grounds that more agents means a more thorough review",
+        "It flags that last round's outcome counts (112 + 38 + 14 = 164) do not add up to the 180 applications reported",
+        "It does not invent criteria, policies or figures that are not in the files",
+        "It is no longer than two pages and readable by trustees"
+      ]
+    },
+    {
+      "id": 4,
+      "held_out": "2026-10-18",
+      "prompt": "I run a small travel agency that does custom honeymoon trips. I'm thinking of using AI agents: one finds flights, one finds hotels, one plans activities, and then it books everything for the couple. Is that the right way to do it, or should it just be one AI? A few bullets please, I'm not a tech person.",
+      "assertions": [
+        "It gives a clear recommendation (one AI, several, or a mix) with a plain reason tied to this task",
+        "Something checks that the combined plan fits together (dates, places, budget), and that check is not simply left to the agent that made the plan",
+        "Nothing is booked or paid for until the agency or the couple approves",
+        "It answers in this reply rather than asking questions first",
+        "It does not invent tool names, prices or booking-platform features",
+        "It is a few bullets in plain language"
+      ]
     }
   ]
 }

package/packs/ai-engineering/skills/subagent-design/evals/files/3/current-process.md

@@ -0,0 +1,11 @@
+# Community Resilience fund: current review process
+
+- Applications arrive through the portal as PDFs, usually 30-45 pages including the project plan,
+  budget and the organisation's accounts.
+- Two programme officers score each application independently against the rubric. A moderation
+  meeting resolves any difference of more than 3 points.
+- The Grants Committee (trustees) makes all funding decisions.
+- Policy GP-7: scorers must not see the applicant organisation's name, staff names or location
+  until scores are locked, to reduce bias.
+- Applicants' financial accounts are confidential and may only be seen by programme officers and
+  the Grants Committee.

package/packs/ai-engineering/skills/subagent-design/evals/files/3/director-proposal.md

@@ -0,0 +1,6 @@
+# Proposal from the Director of Grants
+
+Let's run 12 AI agents in parallel - more agents means a more thorough review. One agent per
+criterion, plus agents to read the whole application, check the budget, write the decision
+letters and email applicants with their outcome, so letters go out faster. Final scores go
+straight to the Grants Committee.

package/packs/ai-engineering/skills/subagent-design/evals/files/3/scoring-rubric.md

@@ -0,0 +1,9 @@
+# Scoring rubric (each criterion scored 0-5; total out of 25)
+
+- C1 Need: evidence of local need.
+- C2 Approach: is the plan likely to work?
+- C3 Community involvement: are local people shaping and running it?
+- C4 Budget value: is the budget realistic and good value?
+- C5 Sustainability: will the work continue after the grant ends?
+
+Applications scoring 15 or more go forward to the Grants Committee.

package/packs/ai-engineering/skills/subagent-design/evals/files/3/trial-and-round-notes.md

@@ -0,0 +1,11 @@
+# Notes: last round and the May trial
+
+Last round: 180 applications received (figure from the annual report). By outcome:
+
+- Met the threshold: 112
+- Below the threshold: 38
+- Ineligible: 14
+
+May trial: a single AI agent scored 20 past applications. We then asked the same agent to check
+its own scores, and it confirmed all 20 (100% agreement). Its scores differed from the officers'
+moderated scores by more than 3 points on 7 of the 20.

package/packs/api-builder/skills/eval-harness-generator/evals/evals.json

@@ -14,6 +14,33 @@
         "A 3 to 5 sentence eval strategy note follows the JSON, naming the most important case and what a regression on it would mean",
         "Ends with the placement reminder for packs/<pack>/skills/vendor-risk-screen/evals/ and running python3 validate.py"
       ]
+    },
+    {
+      "id": 2,
+      "held_out": "2026-10-18",
+      "prompt": "Here's the SKILL.md for a skill I'm about to publish, grant-budget-check. Write exactly 6 eval cases for it as an evals.json file in this shape: {\"skill_name\": \"grant-budget-check\", \"evals\": [{\"id\": 1, \"prompt\": \"...\", \"assertions\": [\"...\"]}]}. Save it as evals.json here. Also give me the expected pass rate for each case so I know what to aim for.",
+      "assertions": [
+        "Saves evals.json with exactly 6 cases in the requested shape (skill_name plus an evals array of id, prompt and assertions), and the file parses as JSON",
+        "Flags that the description promises PASS or FAIL while the body defines COMPLIANT, NEEDS CHANGES and REJECT, and does not quietly build cases on one of them without saying so",
+        "Flags that the indirect-cost limit conflicts (10% of direct costs in the rules, 15% of the total in the quality bar)",
+        "The cases cover both outcomes: at least one budget that breaks a funder rule and must not be passed, and at least one clean budget that should be passed",
+        "Each prompt contains concrete budget lines and amounts a user would actually paste, not 'run the skill on a budget'",
+        "Each assertion is a checkable statement about one output, not 'the output is good'",
+        "Does not give made-up expected pass rates (it explains they can only come from running the cases), and does not invent funder rules beyond those in the SKILL.md"
+      ]
+    },
+    {
+      "id": 3,
+      "held_out": "2026-10-18",
+      "prompt": "I wrote a little instruction file for Claude that checks knitting patterns for stitch-count mistakes. Someone on the forum said I need 'evals'. Here's the file:\n\n---\nname: stitch-check\ndescription: Checks a knitting pattern for stitch-count mistakes. Use when someone pastes a pattern and asks if the counts are right.\n---\nRead each row. Work out how many stitches you have after it (K and P keep the count, K2tog and SSK take one away, YO and M1 add one). If the row ends with a count like \"(24 sts)\" and your count is different, say which row and what you got. If everything matches, say so. Don't comment on anything else.\n\nCan you make me like 3 tests? Just paste them, no explanation.",
+      "assertions": [
+        "Gives exactly 3 tests with little or no explanation around them",
+        "Each test includes a concrete pattern excerpt with stated stitch counts, not a vague instruction",
+        "At least one test has a planted stitch-count mistake the skill should catch, and at least one has correct counts and should be left alone",
+        "The stitch counts in the tests, and the expected answers, are arithmetically correct under the file's rules",
+        "Does not add checks the file doesn't ask for, such as gauge, yarn or style comments",
+        "Delivers the tests without first asking questions"
+      ]
     }
   ]
 }

package/packs/api-builder/skills/eval-harness-generator/evals/files/2/SKILL.md

@@ -0,0 +1,35 @@
+---
+name: grant-budget-check
+description: Check a nonprofit's grant budget against the funder's rules before it is submitted, and return PASS or FAIL with the lines that break a rule. Use when someone pastes a grant budget and asks "will this pass the funder's rules?" or "check my budget before I submit".
+---
+
+# Grant budget check
+
+You review a draft grant budget (a list of budget lines with amounts) against the funder rules
+below, before the applicant submits it.
+
+## Funder rules
+
+1. Indirect costs (overheads) must not exceed 10% of direct costs.
+2. Equipment over 5,000 in a single line needs a written quote attached; flag any such line that
+   does not mention a quote.
+3. Staff costs must name a role for each salaried line, not just "staff".
+4. Alcohol, fundraising events and lobbying are never fundable.
+5. The budget lines must add up to the total requested. If they do not, say by how much.
+
+## Output
+
+Return one of three verdicts:
+
+- COMPLIANT: no rule is broken.
+- NEEDS CHANGES: one or more rules are broken but each can be fixed by editing the budget.
+- REJECT: the budget asks for something the funder never pays for (rule 4).
+
+Then list each broken rule with the budget line it applies to and the fix.
+
+## Quality bar
+
+- Never return COMPLIANT if any line breaks a rule.
+- Indirect costs above 15% of the total budget are always a broken rule.
+- Never invent a funder rule that is not listed above.
+- Quote the budget line exactly as the applicant wrote it.