reeval 0.8.0 → 0.9.0

package/dist/studio-app/index.html

@@ -6,8 +6,8 @@
     <meta name="referrer" content="no-referrer" />
     <link rel="icon" type="image/svg+xml" href="./favicon.svg" />
     <title>reeval Studio</title>
-    <script type="module" crossorigin src="./assets/index-Bgr-efZm.js"></script>
-    <link rel="stylesheet" crossorigin href="./assets/index-CUx6GoL5.css">
+    <script type="module" crossorigin src="./assets/index-CROH2NB3.js"></script>
+    <link rel="stylesheet" crossorigin href="./assets/index-B9MCTfuu.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/{summary-CJdxaRdM.js → summary-2OAWF2N7.js}

@@ -1,8 +1,8 @@
-import { a as providerModel } from "./model-Cl507trk.js";
-import { l as writeCached } from "./files-B6aVVGrk.js";
-import { r as fromRecorded, t as attemptSummary } from "./tools-D0gF2QoV.js";
-import { n as allRuns } from "./commands-B618jg3C.js";
-import { a as failedAnalysis, i as defaultBase, o as prepareAnalysis, s as promptOf, t as analyseDigest } from "./command-BqJtFiPy.js";
+import { a as providerModel } from "./model-B1BZoZ9b.js";
+import { l as writeCached } from "./files-DJUFg-4e.js";
+import { r as fromRecorded, t as attemptSummary } from "./tools-B10Uj7OI.js";
+import { n as allRuns } from "./commands-CLmPBrTS.js";
+import { a as failedAnalysis, i as defaultBase, o as prepareAnalysis, s as promptOf, t as analyseDigest } from "./command-DUT5l13I.js";
 import { join } from "node:path";
 import { generateText, stepCountIs, tool } from "ai";
 import { z } from "zod";
@@ -189,4 +189,4 @@ function fakeSummaryModel(p) {
 //#endregion
 export { writeRunSummary };
 
-//# sourceMappingURL=summary-CJdxaRdM.js.map
+//# sourceMappingURL=summary-2OAWF2N7.js.map

package/dist/{summary-CJdxaRdM.js.map → summary-2OAWF2N7.js.map}

@@ -1 +1 @@
-{"version":3,"file":"summary-CJdxaRdM.js","names":[],"sources":["../src/analysis/agent/summary.ts"],"mappings":";;;;;;;;;AA4BA,MAAM,aAAa;AAEnB,MAAa,uBAAuB;;;;;;AAOpC,SAAgB,aAAa,GAAqB,UAAkB;CAClE,IAAI,OAAO;CACX,MAAM,SAAY,UAAoC;EACpD,MAAM,OAAO,KAAK,UAAU,KAAK,CAAC,CAAC;EACnC,IAAI,OAAO,MACT,OAAO,EACL,OACE,8FACJ;EAEF,QAAQ;EACR,OAAO;CACT;CACA,MAAM,OAAO,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,MAAO,IAAI,CAAC,CAAC,IAAI,CAAC,CAAE;CACjE,OAAO;EACL,eAAe,KAAK;GAClB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,0CAA+C,EACzE,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,OAAO,EAAE,QAAQ,MAAM,MAAM,EAAE,OAAO,EAAE;IAC9C,IAAI,MAAM;KACR,MAAM,EAAE,UAAU,WAAW,GAAG,SAAS;KACzC,OAAO,MAAM,IAAI;IACnB;IACA,OAAO,EACL,OAAO,EAAE,OAAO,SAAS,MAAM,MAAM,EAAE,OAAO,EAAE,IAC5C,GAAG,GAAG,8BACN,uBAAuB,GAAG,aAAa,EAAE,OAAO,QAAQ,KAAK,IAAI,KAAK,OAAO,GACnF;GACF;EACF,CAAC;EACD,cAAc,KAAK;GACjB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,gDAAqD,EAC/E,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,IAAI,EAAE,OAAO,SAAS;IAC5B,MAAM,MAAM,KAAK,KAAK,MAAM,MAAM,EAAE,OAAO,UAAU,EAAE,KAAK;IAC5D,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS,MAAM,EAAE,QAAQ,CAAC,CAAC,MAAM,MAAM,EAAE,QAAQ,GAAG,GAAG;IACxF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QACjB,OAAO,EACL,OAAO,cAAc,GAAG,qBAAqB,OAAO,KAAK,EAAE,OAAO,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,GACzF;IAEF,MAAM,OAAO,MAAM,aAAa,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,GAAG,MAAM;IACvE,OAAO,MAAM;KAAE,GAAG,eAAe,IAAI;KAAG;IAAG,CAAC;GAC9C;EACF,CAAC;CACH;AACF;AAIA,MAAM,WAAW,MAAe,OAAO,MAAM,YAAY,MAAM,QAAQ,WAAW;;AAGlF,SAAgB,UAAU,OAA8B;CACtD,MAAM,KAAK,MAAM,QAAQ,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC;CACjD,MAAM,WAAW,GACd,QAAQ,MAAM,EAAE,SAAS,eAAe,CAAC,CACzC,KAAK,OAAO;EAAE,GAAI,EAAE;EAA6C,UAAU;CAAE,EAAE;CAClF,IAAI,CAAC,GAAG,QAAQ,OAAO;EAAE;EAAU,MAAM;CAAG;CAC5C,OAAO;EACL;EACA,MAAM,uGAAuG,GAC1G,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,KAAK,UAAU,EAAE,KAAK,EAAE,IAAI,KAAK,UAAU,EAAE,MAAM,GAAG,CAAC,CAC/E,KAAK,IAAI;CACd;AACF;;AAGA,eAAsB,YACpB,GACA,OACA,QACsF;CACtF,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,OAAO,iBAAiB,CAAC,IAAI,MAAM,cAAc,KAAK;EACnE,QAAQ;EACR,QAAQ,SAAS,EAAE,MAAM;EACzB,OAAO,aAAa,GAAG,OAAO,SAAS,iBAAiB,CAAC;EACzD,UAAU,YAAA,CAAyB;EACnC,cAAc,EAAE,iBACd,eAAe,IAAoB,EAAE,YAAY,OAAgB,IAAI,KAAA;EACvE,aAAa;EACb,iBAAiB,OAAO,SAAS;EACjC,YAAY;CACd,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,SAAS,MAC3B,EAAE,YAAY,KAAK,OAAO;GAAE,MAAM,EAAE;GAAU,OAAO,EAAE;GAAO,QAAQ,EAAE;EAAO,EAAE,CACnF;EACA,OAAO;GACL,aAAa,OAAO,YAAY,eAAe;GAC/C,cAAc,OAAO,YAAY,gBAAgB;EACnD;CACF;AACF;;;;;AAQA,eAAsB,gBAAgB,SAId;CACtB,MAAM,EAAE,QAAQ,OAAO,UAAU;CACjC,MAAM,UAAU,MAAM,QAAQ,MAAM;CACpC,MAAM,OAAO,QAAQ,MAAM,MAAM,EAAE,UAAU,KAAK;CAClD,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM,uBAAuB;CAC3D,MAAM,WAAW,MAAM,gBACrB,QACA,MACA,MAAM,YAAY,QAAQ,SAAS,IAAI,GACvC,IACF;CACA,MAAM,UAAU,KAAK,IAAI;CACzB,IAAI;CACJ,IAAI;EACF,MAAM,EAAE,OAAO,UAAU,MAAM,YAAY,UAAU,OAAO,MAAM;EAClE,MAAM,MAAM,cAAc,QAAQ,UAAU,OAAO,UAAU,KAAK,CAAC;EACnE,MAAM,QAAQ,GAAkB,MAAe,MAAM,OAAO,OAAO,IAAI;EACvE,IAAI,QAAQ;GACV,aAAa,KAAK,IAAI,MAAM,aAAa,MAAM,WAAW;GAC1D,cAAc,KAAK,IAAI,MAAM,cAAc,MAAM,YAAY;EAC/D;EACA,IAAI,YAAY,KAAK,IAAI,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrE,MAAM,MAAM,eAAe,QAAQ,UAAU,OAAO,SAAS,KAAK,IAAI,IAAI,OAAO;CACnF;CACA,MAAM,OAAO,MAAM,YAAY,OAAO,MAAM,SAAS,KAAK,GAAG;CAC7D,OAAO;EAAE;EAAK,KAAK,SAAS;EAAK;CAAK;AACxC;;;;;;AASA,SAAS,iBAAiB,GAAoC;CAC5D,MAAM,QAAQ;EACZ,aAAa;GAAE,OAAO;GAAG,SAAS;GAAG,WAAW;GAAG,YAAY;EAAE;EACjE,cAAc;GAAE,OAAO;GAAG,MAAM;GAAG,WAAW;EAAE;CAClD;CACA,MAAM,UAAU,SAAoB,YAAmC;EACrE;EACA,cAAc;GAAE,SAAS;GAAQ,KAAK,WAAW,SAAS,SAAS;EAAa;EAChF;EACA,UAAU,CAAC;EACX,UAAU,EAAE,SAAS,gBAAgB;CACvC;CACA,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAmE;GACpF,MAAM,WAAW,QAAQ,OAAO,MAC7B,MACC,EAAE,SAAS,UACX,MAAM,QAAQ,EAAE,OAAO,KACtB,EAAE,QAAyB,MAAM,MAAM,EAAE,SAAS,aAAa,CACpE;GACA,MAAM,QAAQ,CACZ,GAAG,EAAE,OAAO,QACT,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAQ;IAAE,MAAM;IAAiB,OAAO,EAAE,GAAG;GAAE,EAAE,GACzD,GAAG,OAAO,QAAQ,EAAE,OAAO,QAAQ,CAAC,CACjC,QAAQ,GAAG,OAAO,EAAE,YAAY,MAAM,CAAC,CACvC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,CAAC,SAAS;IAAE,MAAM;IAAgB,OAAO,EAAE,GAAG;GAAE,EAAE,CAC5D;GACA,IAAI,YAAY,CAAC,MAAM,QACrB,OAAO,OACL,CAAC;IAAE,MAAM;IAAQ,MAAM,QAAQ,WAAW,kBAAkB,UAAU;GAAG,CAAC,GAC1E,MACF;GAEF,OAAO,OACL,MAAM,KAAK,GAAG,OAAO;IACnB,MAAM;IACN,YAAY,QAAQ,EAAE,KAAK,GAAG;IAC9B,UAAU,EAAE;IACZ,OAAO,KAAK,UAAU,EAAE,KAAK;GAC/B,EAAE,GACF,YACF;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF"}
+{"version":3,"file":"summary-2OAWF2N7.js","names":[],"sources":["../src/analysis/agent/summary.ts"],"mappings":";;;;;;;;;AA4BA,MAAM,aAAa;AAEnB,MAAa,uBAAuB;;;;;;AAOpC,SAAgB,aAAa,GAAqB,UAAkB;CAClE,IAAI,OAAO;CACX,MAAM,SAAY,UAAoC;EACpD,MAAM,OAAO,KAAK,UAAU,KAAK,CAAC,CAAC;EACnC,IAAI,OAAO,MACT,OAAO,EACL,OACE,8FACJ;EAEF,QAAQ;EACR,OAAO;CACT;CACA,MAAM,OAAO,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,MAAO,IAAI,CAAC,CAAC,IAAI,CAAC,CAAE;CACjE,OAAO;EACL,eAAe,KAAK;GAClB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,0CAA+C,EACzE,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,OAAO,EAAE,QAAQ,MAAM,MAAM,EAAE,OAAO,EAAE;IAC9C,IAAI,MAAM;KACR,MAAM,EAAE,UAAU,WAAW,GAAG,SAAS;KACzC,OAAO,MAAM,IAAI;IACnB;IACA,OAAO,EACL,OAAO,EAAE,OAAO,SAAS,MAAM,MAAM,EAAE,OAAO,EAAE,IAC5C,GAAG,GAAG,8BACN,uBAAuB,GAAG,aAAa,EAAE,OAAO,QAAQ,KAAK,IAAI,KAAK,OAAO,GACnF;GACF;EACF,CAAC;EACD,cAAc,KAAK;GACjB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,gDAAqD,EAC/E,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,IAAI,EAAE,OAAO,SAAS;IAC5B,MAAM,MAAM,KAAK,KAAK,MAAM,MAAM,EAAE,OAAO,UAAU,EAAE,KAAK;IAC5D,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS,MAAM,EAAE,QAAQ,CAAC,CAAC,MAAM,MAAM,EAAE,QAAQ,GAAG,GAAG;IACxF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QACjB,OAAO,EACL,OAAO,cAAc,GAAG,qBAAqB,OAAO,KAAK,EAAE,OAAO,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,GACzF;IAEF,MAAM,OAAO,MAAM,aAAa,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,GAAG,MAAM;IACvE,OAAO,MAAM;KAAE,GAAG,eAAe,IAAI;KAAG;IAAG,CAAC;GAC9C;EACF,CAAC;CACH;AACF;AAIA,MAAM,WAAW,MAAe,OAAO,MAAM,YAAY,MAAM,QAAQ,WAAW;;AAGlF,SAAgB,UAAU,OAA8B;CACtD,MAAM,KAAK,MAAM,QAAQ,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC;CACjD,MAAM,WAAW,GACd,QAAQ,MAAM,EAAE,SAAS,eAAe,CAAC,CACzC,KAAK,OAAO;EAAE,GAAI,EAAE;EAA6C,UAAU;CAAE,EAAE;CAClF,IAAI,CAAC,GAAG,QAAQ,OAAO;EAAE;EAAU,MAAM;CAAG;CAC5C,OAAO;EACL;EACA,MAAM,uGAAuG,GAC1G,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,KAAK,UAAU,EAAE,KAAK,EAAE,IAAI,KAAK,UAAU,EAAE,MAAM,GAAG,CAAC,CAC/E,KAAK,IAAI;CACd;AACF;;AAGA,eAAsB,YACpB,GACA,OACA,QACsF;CACtF,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,OAAO,iBAAiB,CAAC,IAAI,MAAM,cAAc,KAAK;EACnE,QAAQ;EACR,QAAQ,SAAS,EAAE,MAAM;EACzB,OAAO,aAAa,GAAG,OAAO,SAAS,iBAAiB,CAAC;EACzD,UAAU,YAAA,CAAyB;EACnC,cAAc,EAAE,iBACd,eAAe,IAAoB,EAAE,YAAY,OAAgB,IAAI,KAAA;EACvE,aAAa;EACb,iBAAiB,OAAO,SAAS;EACjC,YAAY;CACd,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,SAAS,MAC3B,EAAE,YAAY,KAAK,OAAO;GAAE,MAAM,EAAE;GAAU,OAAO,EAAE;GAAO,QAAQ,EAAE;EAAO,EAAE,CACnF;EACA,OAAO;GACL,aAAa,OAAO,YAAY,eAAe;GAC/C,cAAc,OAAO,YAAY,gBAAgB;EACnD;CACF;AACF;;;;;AAQA,eAAsB,gBAAgB,SAId;CACtB,MAAM,EAAE,QAAQ,OAAO,UAAU;CACjC,MAAM,UAAU,MAAM,QAAQ,MAAM;CACpC,MAAM,OAAO,QAAQ,MAAM,MAAM,EAAE,UAAU,KAAK;CAClD,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM,uBAAuB;CAC3D,MAAM,WAAW,MAAM,gBACrB,QACA,MACA,MAAM,YAAY,QAAQ,SAAS,IAAI,GACvC,IACF;CACA,MAAM,UAAU,KAAK,IAAI;CACzB,IAAI;CACJ,IAAI;EACF,MAAM,EAAE,OAAO,UAAU,MAAM,YAAY,UAAU,OAAO,MAAM;EAClE,MAAM,MAAM,cAAc,QAAQ,UAAU,OAAO,UAAU,KAAK,CAAC;EACnE,MAAM,QAAQ,GAAkB,MAAe,MAAM,OAAO,OAAO,IAAI;EACvE,IAAI,QAAQ;GACV,aAAa,KAAK,IAAI,MAAM,aAAa,MAAM,WAAW;GAC1D,cAAc,KAAK,IAAI,MAAM,cAAc,MAAM,YAAY;EAC/D;EACA,IAAI,YAAY,KAAK,IAAI,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrE,MAAM,MAAM,eAAe,QAAQ,UAAU,OAAO,SAAS,KAAK,IAAI,IAAI,OAAO;CACnF;CACA,MAAM,OAAO,MAAM,YAAY,OAAO,MAAM,SAAS,KAAK,GAAG;CAC7D,OAAO;EAAE;EAAK,KAAK,SAAS;EAAK;CAAK;AACxC;;;;;;AASA,SAAS,iBAAiB,GAAoC;CAC5D,MAAM,QAAQ;EACZ,aAAa;GAAE,OAAO;GAAG,SAAS;GAAG,WAAW;GAAG,YAAY;EAAE;EACjE,cAAc;GAAE,OAAO;GAAG,MAAM;GAAG,WAAW;EAAE;CAClD;CACA,MAAM,UAAU,SAAoB,YAAmC;EACrE;EACA,cAAc;GAAE,SAAS;GAAQ,KAAK,WAAW,SAAS,SAAS;EAAa;EAChF;EACA,UAAU,CAAC;EACX,UAAU,EAAE,SAAS,gBAAgB;CACvC;CACA,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAmE;GACpF,MAAM,WAAW,QAAQ,OAAO,MAC7B,MACC,EAAE,SAAS,UACX,MAAM,QAAQ,EAAE,OAAO,KACtB,EAAE,QAAyB,MAAM,MAAM,EAAE,SAAS,aAAa,CACpE;GACA,MAAM,QAAQ,CACZ,GAAG,EAAE,OAAO,QACT,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAQ;IAAE,MAAM;IAAiB,OAAO,EAAE,GAAG;GAAE,EAAE,GACzD,GAAG,OAAO,QAAQ,EAAE,OAAO,QAAQ,CAAC,CACjC,QAAQ,GAAG,OAAO,EAAE,YAAY,MAAM,CAAC,CACvC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,CAAC,SAAS;IAAE,MAAM;IAAgB,OAAO,EAAE,GAAG;GAAE,EAAE,CAC5D;GACA,IAAI,YAAY,CAAC,MAAM,QACrB,OAAO,OACL,CAAC;IAAE,MAAM;IAAQ,MAAM,QAAQ,WAAW,kBAAkB,UAAU;GAAG,CAAC,GAC1E,MACF;GAEF,OAAO,OACL,MAAM,KAAK,GAAG,OAAO;IACnB,MAAM;IACN,YAAY,QAAQ,EAAE,KAAK,GAAG;IAC9B,UAAU,EAAE;IACZ,OAAO,KAAK,UAAU,EAAE,KAAK;GAC/B,EAAE,GACF,YACF;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF"}

package/dist/{summary-k3mhWWXv.js → summary-BTksrIxn.js}

@@ -1,7 +1,8 @@
 import { n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
-import { E as runtimeLabel, H as cellContrast, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, b as isUngatedMiss, g as harnessLabel, h as formatUsage, q as effortLabel, x as languageLabel } from "./model-BndnmcLm.js";
+import { E as runtimeLabel, H as cellContrast, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, b as isUngatedMiss, g as harnessLabel, h as formatUsage, q as effortLabel, x as languageLabel } from "./model-BOnlAPzs.js";
+import { a as familyLiftLabel, i as familyHeadLabel, o as familyRateLabel, r as familyEffortLabel } from "./family-D2b7WD10.js";
 import { a as sampleReasons } from "./pairs-DDsjgm5F.js";
-import { o as warnings } from "./state-B72h_jEw.js";
+import { o as warnings } from "./state-CiXyLqg0.js";
 //#region src/history/summary.ts
 const OUTCOME_MARK = {
 	pass: "✓",
@@ -37,6 +38,37 @@ function liftLabel(lift) {
 	const counts = `${variant.pass}/${variant.counted} vs ${control.pass}/${control.counted}`;
 	return `lift ${signedPoints(lift.points)} pts (${range}, ${counts})${lift.withinNoise ? ", within noise" : ""}`;
 }
+const answered = (t) => t.pass + t.fail + t.notRun;
+const VERDICT_ORDER = {
+	discriminates: 0,
+	mixed: 1
+};
+/**
+* A variant cell's checks against its control's, those that tell them apart first:
+* `loads the skill 2/2 vs 0/2 · exports greet 2/2 vs 2/2 (doesn't discriminate)`. Empty
+* without a control's tally.
+*/
+function checksLabel(cell) {
+	return (cell.checks ?? []).filter((k) => k.control && k.verdict).map((k, i) => ({
+		k,
+		i
+	})).sort((a, b) => (VERDICT_ORDER[a.k.verdict ?? ""] ?? 2) - (VERDICT_ORDER[b.k.verdict ?? ""] ?? 2) || a.i - b.i).map(({ k }) => {
+		const control = k.control;
+		const notRun = k.notRun + control.notRun;
+		const notes = [notRun ? `${notRun} not run` : "", k.verdict === "both-pass" || k.verdict === "both-fail" ? "doesn't discriminate" : ""].filter(Boolean);
+		return `${k.label} ${k.pass}/${answered(k)} vs ${control.pass}/${answered(control)}${notes.length ? ` (${notes.join(", ")})` : ""}`;
+	}).join(" · ");
+}
+/** Every variant cell's checks that can't tell it from its control: they pass, or fail, on both. */
+function undiscriminating(evals) {
+	return evals.flatMap((entry) => entry.cells.flatMap((cell) => (cell.checks ?? []).filter((k) => k.verdict === "both-pass" || k.verdict === "both-fail").map((check) => ({
+		entry,
+		cell,
+		check
+	}))));
+}
+/** `evals/s.eval.ts › greets  fake/skilled/fake  exports greet: passes on both`. */
+const undiscriminatingLabel = ({ entry, cell, check }) => `${entry.id}  ${cell.key}  ${check.label}: ${check.verdict === "both-pass" ? "passes" : "fails"} on both`;
 const rateText = (r) => `${r.name} ${r.wins}/${r.n} (${Math.round(r.wins / r.n * 100)}%)`;
 /**
 * One compare question's answer: `"Which is cleaner?" within: with-skill 2/2 (100%), plain 0/2
@@ -68,6 +100,7 @@ function cellCounts(e) {
 			c.gate === "none" ? "not gated" : "",
 			c.gate !== "all" && c.gate !== "none" ? `gate ${gateLabel(c.gate)}` : "",
 			c.lift ? liftLabel(c.lift) : "",
+			...[checksLabel(c)].flatMap((checks) => checks ? [`checks ${checks}`] : []),
 			...[cellContrast(e, c.key)].flatMap((effort) => effort ? [`effort ${effortLabel(effort)}`] : []),
 			...[judgeRatesLabel(cellJudgeRates(e, c.key))].filter(Boolean),
 			...[formatUsage(cellUsage(e, c.key))].filter((u) => Boolean(u)),
@@ -104,6 +137,26 @@ function stateRows(run) {
 	rows.push(["Runner", run.runner.kind === "ci" ? `CI (${run.runner.provider})${run.runner.runUrl ? ` ${run.runner.runUrl}` : ""}` : `local · ${run.author}`]);
 	return rows;
 }
+/** Each pooled `evaluate.each`: its cases, and its pooled rates or lifts. */
+function familySection(run) {
+	if (!run.families?.length) return [];
+	const lines = [
+		"## Families",
+		"",
+		"| | Family | Pooled |",
+		"|---|---|---|"
+	];
+	for (const f of run.families) {
+		const pooled = f.lifts.length ? f.lifts.map((l) => {
+			const gate = l.minLift === void 0 ? "" : ` (minLift ${Math.round(l.minLift * 100)})`;
+			const effort = familyEffortLabel(l);
+			return `${l.variant} ${familyLiftLabel(l)}${gate}${effort ? `; effort ${effort}` : ""}`;
+		}) : f.rates.map((r) => `${r.cell} ${familyRateLabel(r)}`);
+		lines.push(`| ${OUTCOME_MARK[f.outcome]} | ${cell(`${f.id} (${familyHeadLabel(f)})`)} | ${cell(pooled.join(" · "))} |`);
+	}
+	lines.push("");
+	return lines;
+}
 function renderSummary(run) {
 	const lines = [
 		`# reeval run ${run.runId}`,
@@ -122,6 +175,7 @@ function renderSummary(run) {
 		...warnings(run).map((w) => `> **Warning:** ${w}\n`),
 		`Record: [${run.runId}.json](../${run.runId}.json) · artifacts: ${run.artifacts}`,
 		"",
+		...familySection(run),
 		"## Evals",
 		"",
 		"| | Eval | Time or cells | Judge |",
@@ -195,6 +249,6 @@ function attemptDetail(lines, a) {
 	if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
 }
 //#endregion
-export { liftLabel as a, stateRows as c, gateLabel as i, summaryCounts as l, answersLabel as n, renderSummary as o, cellCounts as r, repeatsVerdict as s, OUTCOME_MARK as t };
+export { gateLabel as a, repeatsVerdict as c, undiscriminating as d, undiscriminatingLabel as f, checksLabel as i, stateRows as l, answersLabel as n, liftLabel as o, cellCounts as r, renderSummary as s, OUTCOME_MARK as t, summaryCounts as u };
 
-//# sourceMappingURL=summary-k3mhWWXv.js.map
+//# sourceMappingURL=summary-BTksrIxn.js.map

package/dist/summary-BTksrIxn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"summary-BTksrIxn.js","names":[],"sources":["../src/history/summary.ts"],"mappings":";;;;;;AAqBA,MAAa,eAAwC;CACnD,MAAM;CACN,MAAM;CACN,aAAa;CACb,SAAS;AACX;;AAGA,MAAM,qBAAqB;AAE3B,MAAM,QAAQ,MAAc,EAAE,QAAQ,OAAO,KAAK,CAAC,CAAC,QAAQ,OAAO,GAAG;AAEtE,SAAgB,cAAc,GAAiC;CAC7D,MAAM,QAAQ;EACZ,EAAE,QAAQ,GAAG,EAAE,KAAK;EACpB,EAAE,OAAO,EAAE,WAAW,KAAK,GAAG,EAAE,OAAO,EAAE,SAAS;EAClD,EAAE,WAAW,KAAK,GAAG,EAAE,SAAS;EAChC,EAAE,cAAc,GAAG,EAAE,WAAW;EAChC,EAAE,WAAW,GAAG,EAAE,QAAQ;CAC5B,CAAC,CAAC,OAAO,OAAO;CAChB,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,MAAM,QAAQ,EAAE,aAAa,EAAE,QAAQ,KAAK,KAAK,EAAE,SAAS,aAAa,EAAE,MAAM;CACjF,OAAO,GAAG,MAAM,KAAK,KAAK,IAAI;AAChC;AAEA,MAAM,gBAAgB,MAAc,GAAG,KAAK,IAAI,MAAM,MAAM,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG;;AAGxF,MAAa,aAAa,SACxB,SAAS,SAAS,SAAS,SAAS,OAAO,YAAY,KAAK;;;;;AAM9D,SAAgB,UAAU,MAA+C;CACvE,MAAM,EAAE,UAAU,SAAS,YAAY;CACvC,MAAM,QAAQ,OAAO,aAAa,SAAS,EAAE,EAAE,GAAG,aAAa,SAAS,EAAE;CAC1E,MAAM,SAAS,GAAG,QAAQ,KAAK,GAAG,QAAQ,QAAQ,MAAM,QAAQ,KAAK,GAAG,QAAQ;CAChF,OAAO,QAAQ,aAAa,KAAK,MAAM,EAAE,QAAQ,MAAM,IAAI,OAAO,GAAG,KAAK,cAAc,mBAAmB;AAC7G;AAEA,MAAM,YAAY,MAAkB,EAAE,OAAO,EAAE,OAAO,EAAE;AAExD,MAAM,gBAAwC;CAAE,eAAe;CAAG,OAAO;AAAE;;;;;;AAO3E,SAAgB,YAAY,MAA0B;CAEpD,QADc,KAAK,UAAU,CAAC,EAAA,CAAG,QAAQ,MAAM,EAAE,WAAW,EAAE,OACpD,CAAC,CACR,KAAK,GAAG,OAAO;EAAE;EAAG;CAAE,EAAE,CAAC,CACzB,MACE,GAAG,OACD,cAAc,EAAE,EAAE,WAAW,OAAO,MAAM,cAAc,EAAE,EAAE,WAAW,OAAO,MAC/E,EAAE,IAAI,EAAE,CACZ,CAAC,CACA,KAAK,EAAE,QAAQ;EACd,MAAM,UAAU,EAAE;EAClB,MAAM,SAAS,EAAE,SAAS,QAAQ;EAClC,MAAM,QAAQ,CACZ,SAAS,GAAG,OAAO,YAAY,IAC/B,EAAE,YAAY,eAAe,EAAE,YAAY,cAAc,yBAAyB,EACpF,CAAC,CAAC,OAAO,OAAO;EAChB,OAAO,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,GAAG,SAAS,CAAC,EAAE,MAAM,QAAQ,KAAK,GAAG,SAAS,OAAO,IAAI,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK;CACjI,CAAC,CAAC,CACD,KAAK,KAAK;AACf;;AAGA,SAAgB,iBAAiB,OAAoB;CACnD,OAAO,MAAM,SAAS,UACpB,MAAM,MAAM,SAAS,UAClB,KAAK,UAAU,CAAC,EAAA,CACd,QAAQ,MAAM,EAAE,YAAY,eAAe,EAAE,YAAY,WAAW,CAAC,CACrE,KAAK,WAAW;EAAE;EAAO;EAAM;CAAM,EAAE,CAC5C,CACF;AACF;;AAGA,MAAa,yBAAyB,EACpC,OACA,MACA,YAEA,GAAG,MAAM,GAAG,IAAI,KAAK,IAAI,IAAI,MAAM,MAAM,IAAI,MAAM,YAAY,cAAc,WAAW,QAAQ;AAElG,MAAM,YAAY,MAChB,GAAG,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,IAAI,KAAK,MAAO,EAAE,OAAO,EAAE,IAAK,GAAG,EAAE;;;;;AAMlE,SAAgB,aAAa,QAA+B;CAC1D,MAAM,IAAI,IAAI,KAAK,OAAO,UAAU,EAAE,EAAE;CACxC,MAAM,OAAO,OAAO,MAAM,QAAQ,MAAM,EAAE,WAAW,KAAK,CAAC,CAAC;CAC5D,MAAM,OAAQ,CAAC,UAAU,QAAQ,CAAC,CAAW,SAAS,QAAQ;EAC5D,MAAM,QAAQ,OAAO,MAAM,QAAQ,MAAM,EAAE,QAAQ,GAAG;EACtD,OAAO,MAAM,SAAS,CAAC,GAAG,IAAI,IAAI,MAAM,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;CACzE,CAAC;CACD,MAAM,QAAQ,OAAO,MAAM;CAC3B,MAAM,QAAQ,GAAG,MAAM,OAAO,UAAU,IAAI,KAAK,IAAI,IAAI,KAAK,MAAM,SAAS,IAAI,KAAK;CACtF,MAAM,KAAK,OAAO,iBAAiB,OAAO;CAC1C,MAAM,SAAS,OAAO,UAAU,KAAA,IAAY,cAAc,KAAK,OAAO,OAAO,GAAG,MAAM;CACtF,IAAI,UAAU,KAAK,OAAO,UAAU,KAAA,GAClC,OAAO,GAAG,EAAE;CAEd,OAAO,GAAG,EAAE,GAAG;EAAC,GAAG;EAAM;EAAO;CAAE,CAAC,CAAC,KAAK,KAAK,IAAI;AACpD;;AAGA,SAAgB,WAAW,GAAsB;CAC/C,OAAO,EAAE,MACN,KAAK,MAAM;EACV,MAAM,QAAQ;GACZ,EAAE,SAAS,aAAa,EAAE,MAAM,MAAM,MAAM,EAAE,SAAS,SAAS,IAAI,YAAY;GAChF,EAAE,SAAS,SAAS,cAAc;GAClC,EAAE,SAAS,SAAS,EAAE,SAAS,SAAS,QAAQ,UAAU,EAAE,IAAI,MAAM;GACtE,EAAE,OAAO,UAAU,EAAE,IAAI,IAAI;GAC7B,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,WAAY,SAAS,CAAC,UAAU,QAAQ,IAAI,CAAC,CAAE;GAC5E,GAAG,CAAC,aAAa,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,WACnC,SAAS,CAAC,UAAU,YAAY,MAAM,GAAG,IAAI,CAAC,CAChD;GACA,GAAG,CAAC,gBAAgB,eAAe,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,OAAO;GAC7D,GAAG,CAAC,YAAY,UAAU,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAmB,QAAQ,CAAC,CAAC;GAC3E,EAAE,UAAU,SAAS,aAAa,EAAE,UAAU,KAAK,IAAI,MAAM;EAC/D,CAAC,CAAC,OAAO,OAAO;EAChB,OAAO,GAAG,EAAE,IAAI,GAAG,EAAE,KAAK,GAAG,EAAE,OAAO,EAAE,OAAO,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK;CAC3F,CAAC,CAAC,CACD,KAAK,KAAK;AACf;AAEA,SAAS,UAAU,GAAsB;CACvC,MAAM,aAAa,EAAE,SAAS,SAAS,MAAM,EAAE,UAAU;CACzD,IAAI,WAAW,WAAW,GAAG,OAAO;CACpC,OAAO,WAAW,KAAK,MAAM,GAAG,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,KAAK,IAAI;AAC7F;;AAGA,MAAM,SAAS,SAAiB,KAAK,QAAQ,YAAY,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;;;;;AAMxE,SAAgB,UAAU,KAAyC;CACjE,MAAM,EAAE,KAAK,OAAO,QAAQ;CAC5B,MAAM,OAAgC,CAAC;CACvC,KAAK,KAAK,CACR,eACA,MACI,GAAG,IAAI,IAAI,MAAM,GAAG,EAAE,IAAI,IAAI,SAAS,OAAO,IAAI,WAAW,KAAK,IAAI,QAAQ,YAAY,UAAU,gDACpG,sBACN,CAAC;CACD,IAAI,KAAK,OACP,KAAK,KAAK,CACR,eACA,IAAI,gBACA,mBAAmB,IAAI,cAAc,MAAM,iBAAiB,MAAM,IAAI,cAAc,MAAM,MAC1F,cAAc,IAAI,qBAAqB,YAAY,IAAI,WAAW,YAAY,MAAM,IAAI,QAAQ,MAAM,IAC5G,CAAC;CAEH,KAAK,KAAK,CACR,aACA,IAAI,aAAa,UAAU,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,KAC7E,YACJ,CAAC;CACD,KAAK,KAAK,CACR,UACA,IAAI,OAAO,SACP,IAAI,OACD,KACE,MACC,GAAG,EAAE,IAAI,QAAQ,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,MAAM,qCAAqC,SAAS,EAAE,YAAY,QAAQ,EAAE,aAAa,iBAAiB,EAAE,WAAW,KAAK,GAAG,MAAM,EAAE,WAAW,MAAM,MAAM,KAAK,EAAE,SAAS,SAAS,cAAc,EAAE,QAAQ,OAAO,MAAM,EAAE,SAAS,IAC3R,CAAC,CACA,KAAK,IAAI,IACZ,qBACN,CAAC;CACD,KAAK,KAAK,CAAC,aAAa,IAAI,UAAU,IAAI,YAAY,CAAC,CAAC,KAAK,IAAI,KAAK,MAAM,CAAC;CAC7E,KAAK,KAAK,CACR,gBACA,IAAI,aACD,KACE,MACC,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,IAAI,EAAE,OAAO,SAAS,aAAa,EAAE,OAAO,KAAK,MAAM,GAAG,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE,OAAO,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,IAAI,MAAM,KAAK,EAAE,IAAI,SAAS,UAAU,EAAE,IAAI,KAAK,IAAI,MAAM,KAAK,EAAE,QAAQ,SAAS,cAAc,EAAE,QAAQ,KAAK,IAAI,MAAM,IACjR,CAAC,CACA,KAAK,IAAI,KAAK,MACnB,CAAC;CACD,KAAK,KAAK,CACR,SACA,GAAG,IAAI,MAAM,SAAS,oBAAoB,IAAI,MAAM,eAAe,SAAS,YAAY,IAAI,MAAM,eAAe,KAAK,IAAI,EAAE,KAAK,GAAG,eAAe,IAAI,MAAM,UAAU,aAAa,IAAI,MAAM,SAChM,CAAC;CACD,KAAK,KAAK,CACR,WACA,IAAI,QAAQ,SACR,GAAG,IAAI,QAAQ,OAAO,OAAO,IAAI,QAAQ,WAAW,IAAI,KAAK,IAAI,IAAI,IAAI,QAAQ,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,MACrI,qDACN,CAAC;CACD,KAAK,KAAK,CACR,YACA,OAAO,QAAQ,IAAI,QAAQ,CAAC,CACzB,KAAK,CAAC,MAAM,OAAO,GAAG,KAAK,GAAG,GAAG,CAAC,CAClC,KAAK,KAAK,CACf,CAAC;CACD,MAAM,WAAW,cAAc,GAAG;CAClC,KAAK,KAAK,CACR,UACA,GAAG,IAAI,GAAG,KAAK,aAAa,GAAG,IAAI,WAAW,MAAM,aAAa,GAAG,KAAK,IAAI,GAAG,GAAG,IAAI,MACzF,CAAC;CACD,KAAK,KAAK,CACR,UACA,IAAI,OAAO,SAAS,OAChB,OAAO,IAAI,OAAO,SAAS,GAAG,IAAI,OAAO,SAAS,IAAI,IAAI,OAAO,WAAW,OAC5E,WAAW,IAAI,QACrB,CAAC;CACD,OAAO;AACT;;AAGA,SAAS,cAAc,KAA0B;CAC/C,IAAI,CAAC,IAAI,UAAU,QAAQ,OAAO,CAAC;CACnC,MAAM,QAAQ;EAAC;EAAe;EAAI;EAAyB;CAAe;CAC1E,KAAK,MAAM,KAAK,IAAI,UAAU;EAC5B,MAAM,SAAS,EAAE,MAAM,SACnB,EAAE,MAAM,KAAK,MAAM;GACjB,MAAM,OAAO,EAAE,YAAY,KAAA,IAAY,KAAK,aAAa,KAAK,MAAM,EAAE,UAAU,GAAG,EAAE;GACrF,MAAM,SAAS,kBAAkB,CAAC;GAClC,OAAO,GAAG,EAAE,QAAQ,GAAG,gBAAgB,CAAC,IAAI,OAAO,SAAS,YAAY,WAAW;EACrF,CAAC,IACD,EAAE,MAAM,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,gBAAgB,CAAC,GAAG;EACxD,MAAM,KACJ,KAAK,aAAa,EAAE,SAAS,KAAK,KAAK,GAAG,EAAE,GAAG,IAAI,gBAAgB,CAAC,EAAE,EAAE,EAAE,KAAK,KAAK,OAAO,KAAK,KAAK,CAAC,EAAE,GAC1G;CACF;CACA,MAAM,KAAK,EAAE;CACb,OAAO;AACT;AAEA,SAAgB,cAAc,KAAwB;CACpD,MAAM,QAAQ;EACZ,gBAAgB,IAAI;EACpB;EACA,GAAG,cAAc,IAAI,OAAO,IAAI,IAAI,WAAW,aAAa,KAAK,QAAQ,IAAI,WAAW,WAAW,mBAAmB,IAAI,OAAO,IAAI,aAAa,IAAI,UAAU,IAAI,eAAe,IAAI,UAAU,EAAE;EACnM;EACA,SAAS,IAAI,UAAU,WAAW,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,OAAO,QAAQ,KAAK,IAAI,KAAK,MAAM,WAAW,IAAI,KAAK,MAAM,WAAW,IAAI,KAAK,SAAS,WAAW,IAAI,SAAS,aAAa,CAAC,IAAI,OAAO,aAAa,KAAA,IAAY,GAAG,IAAI,OAAO,SAAS,aAAa,IAAI,IAAI,OAAO,YAAY,EAAE,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,IAAI,MAAM;EACzU;EACA,GAAG,IAAI,KAAK,gBAAgB,KAAK,MAAM,kBAAkB,EAAE,QAAQ,GAAG;EACtE;EACA;EACA;EACA;EACA,GAAG,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK,EAAE,KAAK,KAAK,CAAC,EAAE,GAAG;EACzD;EACA,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,MAAM,kBAAkB,EAAE,GAAG;EACnD,YAAY,IAAI,MAAM,YAAY,IAAI,MAAM,sBAAsB,IAAI;EACtE;EACA,GAAG,cAAc,GAAG;EACpB;EACA;EACA;EACA;CACF;CACA,KAAK,MAAM,KAAK,IAAI,OAAO;EACzB,MAAM,IAAI,EAAE,SAAS;EACrB,MAAM,OAAO,EAAE,OACX,SACA,EAAE,SAAS,SAAS,IAClB,WAAW,CAAC,IACZ,IACE,eAAe,EAAE,UAAU,IAC3B;EACR,MAAM,KAAK,KAAK,aAAa,EAAE,SAAS,KAAK,KAAK,EAAE,EAAE,EAAE,KAAK,KAAK,IAAI,EAAE,KAAK,UAAU,CAAC,EAAE,GAAG;CAC/F;CAGA,MAAM,QAAQ,MAAgB,MAAM,UAAU,MAAM,gBAAgB,IAAI,MAAM,SAAS,IAAI;CAC3F,KAAK,MAAM,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,OAAO,CAAC,GAAG;EAChF,IAAI,EAAE,YAAY,WAAW;EAC7B,MAAM,KAAK,IAAI,OAAO,aAAa,EAAE,SAAS,GAAG,EAAE,MAAM,EAAE;EAC3D,MAAM,QAAQ,EAAE,SAAS,SAAS;EAClC,IAAI,OAAO,MAAM,KAAK,UAAU,WAAW,CAAC,KAAK,EAAE;EACnD,IAAI,EAAE,SAAS,QAAQ;GACrB,KAAK,MAAM,KAAK,EAAE,SAAS;IACzB,MAAM,KAAK,eAAe,KAAK,aAAa,CAAC,CAAC,GAAG;IACjD,KAAK,MAAM,UAAU,cAAc,EAAE,KAAK,GAAG,MAAM,KAAK,OAAO,KAAK,MAAM,GAAG;GAC/E;GACA,MAAM,KAAK,EAAE;EACf;EACA,MAAM,eAAe,MAAsB,cAAc,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,OAAO;EACnF,KAAK,MAAM,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,YAAY,CAAC,CAAC,GAAG;GAC/E,IAAI,EAAE,YAAY,WAAW;GAC7B,IAAI,cAAc,GAAG,CAAC,GAAG;IACvB,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE,GAAG,uBAAuB,EAAE;IAC9D,MAAM,KAAK,6CAA6C,EAAE;IAC1D,cAAc,OAAO,CAAC;IACtB,MAAM,KAAK,IAAI,YAAY;IAC3B;GACF;GACA,IAAI,OAAO,MAAM,KAAK,QAAQ,aAAa,EAAE,SAAS,GAAG,EAAE,MAAM,EAAE;GACnE,cAAc,OAAO,CAAC;EACxB;CACF;CACA,MAAM,KAAK,EAAE;CACb,OAAO,MAAM,KAAK,IAAI;AACxB;;AAGA,MAAa,kBAAkB,MAC7B,EAAE,OAAO,SAAS,WAAW,EAAE,WAAW,MAAM,MAAM,EAAE,YAAY,MAAM;AAE5E,SAAS,cAAc,OAAiB,GAAwB;CAC9D,IAAI,EAAE,SAAS,CAAC,eAAe,CAAC,GAC9B,MAAM,KACJ,KAAK,EAAE,MAAM,KAAK,KAClB,IACA,OACA,EAAE,MAAM,QAAQ,MAAM,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,GAClD,OACA,EACF;CAEF,KAAK,MAAM,QAAQ,EAAE,OAAO;EAC1B,MAAM,KACJ,aAAa,KAAK,OAAO,IAAI,eAAe,KAAK,UAAU,EAAE,KAAK,KAAK,KAAK,KAAK,MAAM,GAAG,GAAG,GAC/F;EACA,IAAI,KAAK,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,KAAK,KAAK,GAAG,GAAG,GAAG;CACxE;CACA,MAAM,IAAI,EAAE;CACZ,MAAM,KACJ,iBAAiB,eAAe,EAAE,OAAO,EAAE,WAAW,eAAe,EAAE,OAAO,EAAE,YAAY,eAAe,EAAE,QAAQ,EAAE,WAAW,eAAe,EAAE,OAAO,GAC5J;CACA,MAAM,MAAM,EAAE;CACd,MAAM,OAAO,GAAkB,SAC7B,MAAM,OAAO,GAAG,KAAK,iBAAiB,GAAG,EAAE,GAAG;CAChD,MAAM,KACJ,YAAY,IAAI,UAAU,gBAAgB,IAAI,MAAM,sBAAsB,IAAI,IAAI,aAAa,cAAc,EAAE,KAAK,IAAI,IAAI,YAAY,SAAS,EAAE,KAAK,eAAe,IAAI,WAAW,EAAE,WAC1L;CACA,KAAK,MAAM,KAAK,EAAE,YAAY;EAC5B,MAAM,UAAU,EAAE,SAAS,SACvB,WAAW,EAAE,QAAQ,OAAO,GAAG,EAAE,QAAQ,WAAW,IAAI,UAAU,cAClE;EACJ,MAAM,KACJ,aAAa,EAAE,QAAQ,KAAK,EAAE,MAAM,QAAQ,CAAC,EAAE,cAAc,EAAE,UAAU,IAAI,EAAE,QAAQ,QAAQ,KAAK,EAAE,UACxG;CACF;CACA,KAAK,MAAM,KAAK,EAAE,QAChB,MAAM,KACJ,gBAAgB,EAAE,SAAS,MAAM,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,EAAE,IAAI,KAAK,MAAM,IAC3F;CAEF,IAAI,EAAE,aAAa,QACjB,MAAM,KACJ,cAAc,EAAE,aAAa,KAAK,MAAO,EAAE,QAAQ,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,WAAW,GAAG,EAAE,OAAO,KAAK,EAAE,KAAK,GAAI,CAAC,CAAC,KAAK,IAAI,GACjI;CAEF,MAAM,OAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE;CAC7B,IAAI,MAAM;EACR,MAAM,QAAQ,KAAK,SAAS,qBAAqB,GAAG,KAAK,MAAM,GAAG,kBAAkB,EAAE,KAAK;EAC3F,MAAM,KACJ,IACA,iBACA,IACA,GAAG,MACA,QAAQ,CAAC,CACT,MAAM,IAAI,CAAC,CACX,KAAK,MAAM,KAAK,GAAG,GACtB,EACF;CACF;CACA,MAAM,QAAQ;EACZ,EAAE,MAAM,YAAY,eAAe,EAAE,MAAM,SAAS,KAAK;EACzD,EAAE,MAAM,cAAc,gBAAgB,EAAE,MAAM,WAAW,KAAK;EAC9D,EAAE,MAAM,QAAQ,UAAU,EAAE,MAAM,KAAK,KAAK;EAC5C,EAAE,MAAM,cAAc,iBAAiB,EAAE,MAAM,WAAW,KAAK;CACjE,CAAC,CAAC,OAAO,OAAO;CAChB,IAAI,MAAM,QAAQ,MAAM,KAAK,YAAY,MAAM,KAAK,KAAK,GAAG;AAC9D"}

package/dist/testing.d.ts

@@ -1,4 +1,4 @@
-import { Pt as Wait, p as CaptureUseAction, u as CaptureInstruction } from "./types-_LOFdPUf.js";
+import { Ft as Wait, p as CaptureUseAction, u as CaptureInstruction } from "./types-D8RLfkFP.js";
 //#region src/capture/fake-instructions.d.ts
 declare const FAKE_INSTRUCTIONS: unique symbol;
 type FakeInstructionStep = CaptureUseAction[] | {

package/dist/{tools-D0gF2QoV.js → tools-B10Uj7OI.js}

@@ -1,7 +1,7 @@
 import { s as recordFileName } from "./ids-Bp6ut7ZD.js";
 import { n as clip } from "./text-gEcKjAyp.js";
-import { S as parseTranscript } from "./model-BndnmcLm.js";
-import { s as mergeOutcome } from "./assemble-BbFVjHeS.js";
+import { S as parseTranscript } from "./model-BOnlAPzs.js";
+import { s as mergeOutcome } from "./assemble-BCZzTyR7.js";
 import { join } from "node:path";
 import { readFile } from "node:fs/promises";
 //#region src/analysis/agent/tools.ts
@@ -36,7 +36,7 @@ function stepOf(event) {
 			return `${event.tool}${input ? `: ${input}` : ""}${event.ok ? "" : " (failed)"}`;
 		}
 		case "file-edit": return `edit (${event.kind}): ${event.path}`;
-		case "web-search": return `web-search: ${clip(event.query, 120)}`;
+		case "web-search": return `web-search: ${clip(event.query, 120)}${event.ok === false ? " (failed)" : ""}`;
 		case "web-fetch": return `web-fetch: ${event.url}${event.ok ? "" : " (failed)"}`;
 		case "error": return `error: ${clip(event.message, 200)}`;
 		case "retry": return `retry: ${clip(event.message, 200)}`;
@@ -246,4 +246,4 @@ function divergence(failed, passed, window = 6) {
 //#endregion
 export { stagedAttempts as a, listOutcomes as i, divergence as n, fromRecorded as r, attemptSummary as t };
 
-//# sourceMappingURL=tools-D0gF2QoV.js.map
+//# sourceMappingURL=tools-B10Uj7OI.js.map

package/dist/tools-B10Uj7OI.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools-B10Uj7OI.js","names":[],"sources":["../src/analysis/agent/tools.ts"],"mappings":";;;;;;;AA8BA,MAAM,aAAa;CACjB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;AAGA,SAAS,aAAa,OAAwB;CAC5C,IAAI,OAAO,UAAU,UAAU,OAAO,KAAK,MAAM,QAAQ,QAAQ,GAAG,GAAG,GAAG;CAC1E,IAAI,OAAO,UAAU,YAAY,UAAU,MAAM,OAAO;CACxD,MAAM,SAAS;CACf,KAAK,MAAM,OAAO,YAAY;EAC5B,MAAM,QAAQ,OAAO;EACrB,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,KAAK,MAAM,QAAQ,QAAQ,GAAG,GAAG,GAAG;EACnF,IAAI,MAAM,QAAQ,KAAK,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,QAAQ,GAClE,OAAO,KAAK,MAAM,KAAK,GAAG,GAAG,GAAG;CACpC;CACA,OAAO,KAAK,KAAK,UAAU,KAAK,GAAG,GAAG;AACxC;;AAGA,SAAgB,OAAO,OAAuC;CAC5D,QAAQ,MAAM,MAAd;EACE,KAAK,QAAQ;GACX,MAAM,QAAQ,aAAa,MAAM,KAAK;GACtC,OAAO,GAAG,MAAM,OAAO,QAAQ,KAAK,UAAU,KAAK,MAAM,KAAK,KAAK;EACrE;EACA,KAAK,aACH,OAAO,SAAS,MAAM,KAAK,KAAK,MAAM;EACxC,KAAK,cACH,OAAO,eAAe,KAAK,MAAM,OAAO,GAAG,IAAI,MAAM,OAAO,QAAQ,cAAc;EACpF,KAAK,aACH,OAAO,cAAc,MAAM,MAAM,MAAM,KAAK,KAAK;EACnD,KAAK,SACH,OAAO,UAAU,KAAK,MAAM,SAAS,GAAG;EAC1C,KAAK,SACH,OAAO,UAAU,KAAK,MAAM,SAAS,GAAG;EAC1C,SACE,OAAO;CACX;AACF;AAEA,MAAM,SAAS,WAA8B,OAAO,SAAS,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;AAElF,MAAM,QAAQ,MAAc,QAAiB,KAAK,UAAU,MAAM,OAAO,IAAI,KAAK,MAAM,CAAC,GAAG;AAC5F,MAAM,QAAQ,MAAc,QAC1B,KAAK,UAAU,MAAM,OAAO,GAAG,KAAK,MAAM,GAAG,GAAG,EAAE,MAAM,KAAK,SAAS,IAAI;;AAG5E,SAAgB,WAAW,QAAoB,QAA+C;CAC5F,MAAM,UAAU,OAAO;CACvB,IAAI,CAAC,SAAS,OAAO;CACrB,MAAM,EAAE,SAAS,UAAU,aAAa,QAAQ,MAAM;CACtD,OAAO;EACL,IAAI,GAAG,QAAQ,KAAK,IAAI,GAAG,QAAQ;EACnC,MAAM,QAAQ,KAAK;EACnB,GAAG,QAAQ;EACX;EACA,OAAO,QAAQ;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ,IAAI;EAC9D,SAAS,QAAQ,UAAU,CAAC,EAAA,CAAG,KAAK,OAAO;GACzC,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,QAAQ,KAAK,GAAG,EAAE,OAAO,IAAI,EAAE,SAAS,KAAK,GAAG,GAAG;EACrD,EAAE;EACF,aAAa,QAAQ,cAAc,CAAC,EAAA,CAAG,KAAK,OAAO;GACjD,UAAU,EAAE;GACZ,SAAS,EAAE;GACX,OAAO,EAAE;EACX,EAAE;EACF,QAAQ,QAAQ,SAAS,CAAC,EAAA,CAAG,KAAK,OAAO;GACvC,QAAQ,EAAE;GACV,MAAM,EAAE;GACR,QAAQ,EAAE;GACV,OAAO,MAAM,EAAE,UAAU;EAC3B,EAAE;EACF,QAAQ,QAAQ,MAAM,SAAS,CAAC,EAAA,CAAG,KAAK,MAAM,EAAE,IAAI;EACpD,OAAO,QAAQ,MAAM,SAAS;CAChC;AACF;;AAGA,eAAsB,eACpB,YACA,SACwB;CACxB,MAAM,QAAuB,CAAC;CAC9B,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,OAAO,KAAK,YAAY,SAAS,eAAe,OAAO,GAAG,CAAC;EACjE,IAAI,SAA4B;EAChC,IAAI;GACF,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;EAClD,QAAQ,CAER;EACA,MAAM,OAAO,WAAW,QAAQ,MAAM;EACtC,IAAI,MAAM,MAAM,KAAK,IAAI;CAC3B;CACA,OAAO;AACT;;AAGA,eAAsB,aAAa,QAAgB,SAA8C;CAC/F,MAAM,QAAQ,SACZ,OAAO,SAAS,KAAK,QAAQ,IAAI,GAAG,MAAM,CAAC,CAAC,YAAY,EAAE,IAAI,QAAQ,QAAQ,EAAE;CAClF,MAAM,QAAQ,gBAAgB,MAAM,KAAK,QAAQ,MAAM,YAAY,IAAI,CAAC;CACxE,OAAO;EACL,IAAI,QAAQ;EACZ,MAAM,QAAQ;EACd,GAAG,QAAQ;EACX,SAAS,QAAQ;EACjB,OAAO,QAAQ,QAAQ;GAAE,MAAM,QAAQ,MAAM;GAAM,SAAS,QAAQ,MAAM;EAAQ,IAAI;EACtF,QAAQ,MAAM,QAAQ,IACpB,QAAQ,OAAO,IAAI,OAAO,OAAO;GAC/B,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,QAAQ,MAAM,MAAM,KAAK,EAAE,KAAK,IAAI,EAAA,CAAG,KAAK,GAAG,GAAG;EACpD,EAAE,CACJ;EACA,YAAY,QAAQ,WAAW,KAAK,OAAO;GACzC,UAAU,EAAE;GACZ,SAAS,EAAE;GACX,OAAO,EAAE;EACX,EAAE;EACF,OAAO,QAAQ,MAAM,KAAK,GAAG,OAAO;GAClC,QAAQ,EAAE;GACV,MAAM,EAAE;GACR,QAAQ,EAAE;GACV,OAAO,MACL,MAAM,QACH,MACC,EAAE,SAAS,KAAK,EAAE,SAAS,YAAY,EAAE,SAAS,UACtD,CACF;EACF,EAAE;EACF,OAAO,QAAQ,aAAa,KAAK,MAAM,EAAE,IAAI;EAC7C,OAAO,MAAM,KAAK,QAAQ,MAAM,MAAM,IAAI;CAC5C;AACF;;AAMA,SAAgB,eAAe,UAAuC;CACpE,MAAM,SAAS,SAAS,QAAQ,MAAM,EAAE,YAAY,MAAM;CAC1D,MAAM,QAAQ,OAAoB,GAAgB,MAChD,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC;CAC1C,OAAO,SACJ,QAAQ,MAAM,EAAE,YAAY,MAAM,CAAC,CACnC,SAAS,MAAM;EACd,MAAM,WAAW,OAAO,QAAQ,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC;EACvE,MAAM,QAAQ,OAAO,QAAQ,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC;EACpE,MAAM,QAAQ,YAAY;EAC1B,OAAO,QACH,CACE;GACE,QAAQ,EAAE;GACV,QAAQ,MAAM;GACd,OAAO,WAAY,SAAoB;EACzC,CACF,IACA,CAAC;CACP,CAAC;AACL;AAEA,MAAM,aAAa,MAAmB,EAAE,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,MAAM,QAAQ,CAAC;;AAGpF,SAAgB,aAAa,UAAyB;CACpD,OAAO;EACL,UAAU,SAAS,KAAK,OAAO;GAC7B,IAAI,EAAE;GACN,SAAS,EAAE;GACX,OAAO,EAAE,QAAQ,GAAG,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,MAAM,SAAS,GAAG,MAAM;GACpE,cAAc,EAAE,OAAO,QAAQ,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,OAAO;GAC3E,YAAY,EAAE,WAAW,KACtB,MAAM,GAAG,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,IAAI,KAAK,EAAE,UAAU,GAAG,GACpE;GACA,OAAO,EAAE,MAAM;GACf,OAAO,UAAU,CAAC;GAClB,OAAO,EAAE,MAAM;EACjB,EAAE;EACF,OAAO,eAAe,QAAQ;CAChC;AACF;;AAGA,SAAgB,eAAe,GAAgB;CAC7C,OAAO;EACL,IAAI,EAAE;EACN,SAAS,EAAE;EACX,OAAO,EAAE,QAAQ;GAAE,MAAM,EAAE,MAAM;GAAM,SAAS,KAAK,EAAE,MAAM,SAAS,GAAI;EAAE,IAAI;EAChF,OAAO,EAAE,MAAM,KAAK,GAAG,OAAO;GAC5B,MAAM,IAAI;GACV,QAAQ,KAAK,EAAE,QAAQ,GAAG;GAC1B,QAAQ,EAAE;GACV,OACE,EAAE,MAAM,SAAS,KACb;IAAC,GAAG,EAAE,MAAM,MAAM,GAAG,EAAE;IAAG,KAAK,EAAE,MAAM,SAAS,GAAG;IAAW,GAAG,EAAE,MAAM,MAAM,GAAG;GAAC,IACnF,EAAE;GACR,MAAM,KAAK,EAAE,MAAM,GAAG;EACxB,EAAE;EACF,QAAQ,EAAE,OAAO,KAAK,OAAO;GAC3B,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,GAAI,EAAE,aAAa,IAAI,EAAE,QAAQ,EAAE,OAAO,IAAI,CAAC;EACjD,EAAE;EACF,YAAY,EAAE;EACd,OAAO,EAAE,MAAM,MAAM,GAAG,EAAE;EAC1B,MAAM,KAAK,EAAE,OAAO,GAAI;CAC1B;AACF;;;;;AAMA,SAAgB,WAAW,QAAqB,QAAqB,SAAS,GAAG;CAC/E,MAAM,IAAI,OAAO,MAAM,SAAS,MAAM,EAAE,KAAK;CAC7C,MAAM,IAAI,OAAO,MAAM,SAAS,MAAM,EAAE,KAAK;CAC7C,IAAI,OAAO;CACX,OAAO,OAAO,EAAE,UAAU,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO;CAClE,MAAM,WAAW,OAAO,EAAE,UAAU,OAAO,EAAE;CAC7C,MAAM,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC;CACjC,MAAM,WAAW,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,MAAM,CAAC,CAAC,KAAK,MAAM,EAAE,OAAO,CAAC,CAAC;CACxF,MAAM,QAAQ,GAAgB,YAC5B,EAAE,OAAO,MAAM,MAAM,EAAE,YAAY,OAAO,CAAC,EAAE,YAAY;CAC3D,MAAM,WAAW,GAAgB,aAC/B,EAAE,WAAW,MAAM,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,WAAW;CAChE,MAAM,YAAY,CAChB,GAAG,IAAI,IAAI,CAAC,GAAG,OAAO,YAAY,GAAG,OAAO,UAAU,CAAC,CAAC,KAAK,MAAM,EAAE,QAAQ,CAAC,CAChF;CACA,MAAM,QAAQ,MAAmB,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,IAAI,GAAG;CACrE,OAAO;EACL,QAAQ,OAAO;EACf,QAAQ,OAAO;EACf,OAAO,OAAO,QAAQ,GAAG,OAAO,MAAM,KAAK,IAAI,KAAK,OAAO,MAAM,SAAS,GAAG,MAAM;EACnF,eAAe;;EAEf,YAAY,WAAW,OAAO,IAAI;EAClC,OAAO;GAAE,QAAQ,EAAE;GAAQ,QAAQ,EAAE;EAAO;EAC5C,aAAa,EAAE,MAAM,MAAM,OAAO,MAAM;EACxC,aAAa,EAAE,MAAM,MAAM,OAAO,MAAM;EACxC,QAAQ,SACL,QAAQ,MAAM,KAAK,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAClD,KAAK,OAAO;GACX,SAAS;GACT,QAAQ,KAAK,QAAQ,CAAC;GACtB,QAAQ,KAAK,QAAQ,CAAC;GACtB,cAAc,OAAO,OAAO,MAAM,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,UAAU;EACtE,EAAE;EACJ,YAAY,UACT,QAAQ,MAAM,QAAQ,QAAQ,CAAC,MAAM,QAAQ,QAAQ,CAAC,CAAC,CAAC,CACxD,KAAK,OAAO;GACX,UAAU,KAAK,GAAG,GAAG;GACrB,QAAQ,QAAQ,QAAQ,CAAC;GACzB,QAAQ,QAAQ,QAAQ,CAAC;EAC3B,EAAE;EACJ,OAAO;GACL,YAAY,OAAO,MAAM,QAAQ,MAAM,CAAC,OAAO,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE;GAC7E,YAAY,OAAO,MAAM,QAAQ,MAAM,CAAC,OAAO,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE;EAC/E;EACA,WAAW;GAAE,QAAQ,KAAK,MAAM;GAAG,QAAQ,KAAK,MAAM;EAAE;EACxD,MAAM;GAAE,QAAQ,KAAK,OAAO,OAAO,IAAI;GAAG,QAAQ,KAAK,OAAO,OAAO,IAAI;EAAE;CAC7E;AACF"}

package/dist/types-C5vI3g1C.js.map

@@ -1 +1 @@
-{"version":3,"file":"types-C5vI3g1C.js","names":[],"sources":["../src/sandbox/types.ts"],"mappings":";;AAoCA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAgB;AAClB"}
+{"version":3,"file":"types-C5vI3g1C.js","names":[],"sources":["../src/sandbox/types.ts"],"mappings":";;AA0CA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAgB;AAClB"}

package/dist/{types-_LOFdPUf.d.ts → types-D8RLfkFP.d.ts}

@@ -273,11 +273,11 @@ type AttemptRecord = {
     usage: TokenUsage | null;
   };
   timings: PhaseTimings;
-  /** Counts from the transcript, so a run's shape is visible without reading it. */
+  /** Counts from the transcript, so a run's shape is visible without reading it. `webSearches` includes refused searches. */
   activity: {
     tools: number;
     fileEdits: number;
-    /** Null when a turn with none couldn't report them (`unreported`), not a count of 0 that isn't one. */
+    /** Every `web-search` event, including refused ones. Null when a turn with none couldn't report them (`unreported`). */
     webSearches: number | null;
     webFetches: number | null;
     reasoningMs: number;
@@ -291,6 +291,8 @@ type AttemptRecord = {
     durationMs: number;
     log?: FileRef;
   }>;
+  /** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
+  assertions?: AssertionRecord[];
   /** Files under a dependency directory (node_modules/, .venv/, …) collapse to one entry with `count`. */
   changedFiles: Array<FileChange & {
     count?: number;
@@ -325,7 +327,7 @@ type EvalEntry = {
   outcome: Outcome;
   /** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
   todo: boolean;
-  /** pass / (pass + fail) over the gated cells' attempts; infra errors and skips excluded. */
+  /** pass@1, pass / (pass + fail), over the gated cells' attempts; infra errors and skips excluded. */
   passRate: number | null;
   cells: CellRecord[];
   attempts: AttemptRecord[];
@@ -336,8 +338,112 @@ type EvalEntry = {
    * (rates by cut, votes, ties, a failed call, a sample of reasons).
    */
   answers?: CompareAnswer[];
+  /** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
+  family?: {
+    id: string;
+    name: string;
+    case: number;
+    cases: number;
+  };
+};
+/** A 95% interval on a pooled rate or difference. */
+type FamilyInterval = {
+  lo: number;
+  hi: number;
+  level: 0.95;
+  /**
+   * `wilson` / `newcombe-paired`: one counted attempt a case and side, so each case is one
+   * Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
+   * attempts within a case never counted as independent.
+   */
+  method: 'wilson' | 'newcombe-paired' | 't-cases';
+};
+/** One cell's pass rate over a family's cases. */
+type FamilyRate = {
+  /** The cell's key, the same in every case. */
+  cell: string;
+  /** Cases with at least one counted attempt in this cell. */
+  cases: number;
+  /** Passes and counted attempts over those cases. */
+  pass: number;
+  counted: number;
+  /** The mean of the cases' pass@1; at one attempt a case, `pass / counted`. */
+  rate: number;
+  interval: FamilyInterval;
+};
+/**
+ * `lift` / `drop`: the interval excludes 0. `noise`: it doesn't. `insufficient`: fewer than 6
+ * discordant pairs (one attempt a case) or 6 paired cases (several), where no test can decide.
+ */
+type FamilyVerdict = 'lift' | 'drop' | 'noise' | 'insufficient';
+/** One effort metric over a family's paired cases. */
+type FamilyEffortMetric = {
+  /** Cases with a value on both sides. */
+  pairs: number;
+  /** The median over cases of the variant's per-case median, over the control's; null at 0. */
+  ratio: number | null;
+  /** Wilcoxon signed-rank over the per-case differences; null when every difference is 0. */
+  p: number | null;
+  /** `insufficient` below 6 non-zero differences, where the exact test can't reach 0.05. */
+  verdict: 'fewer' | 'more' | 'noise' | 'insufficient';
+};
+/** The variant's effort against the control's, paired by case. */
+type FamilyEffort = {
+  tools: FamilyEffortMetric;
+  agentMs: FamilyEffortMetric;
+  /** Over the cases where every counted attempt reported usage; omitted when none did. */
+  tokens?: FamilyEffortMetric;
+};
+/** A variant cell against its control over a family's cases, paired by case. */
+type FamilyLift = {
+  /** The variant cell's key. */
+  variant: string;
+  /** The control cell's key. */
+  vs: string;
+  /** Cases with counted attempts on both sides. */
+  pairs: number;
+  /** Mean pass@1 over the paired cases on each side. */
+  variantRate: number;
+  controlRate: number;
+  /** Set at one attempt a case and side: pairs where only one side passed. */
+  discordant?: {
+    variantOnly: number;
+    controlOnly: number;
+  };
+  /** Variant minus control, from -1 to 1. */
+  points: number;
+  interval: FamilyInterval;
+  /** McNemar's exact p (one attempt a case), else Wilcoxon signed-rank over the cases. */
+  p: number | null;
+  /** The interval spans 0. */
+  withinNoise: boolean;
+  verdict: FamilyVerdict;
+  /** The compare's `minLift`: on a family it gates this lower bound, not each case's. */
+  minLift?: number;
+  /** `fail` when `interval.lo` is below `minLift`. */
+  outcome: 'pass' | 'fail';
+  effort?: FamilyEffort;
 };
-/** A variant cell's pass rate against the control's, on the same harness and sandbox. */
+/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an eval. */
+type FamilyRecord = {
+  /** `file › group › … › template`. */
+  id: string;
+  file: string;
+  group: string[];
+  /** The name template, e.g. `fixes $id`. */
+  name: string;
+  /** Cases in the list. */
+  cases: number;
+  /** Cases with an eval in this run; fewer when a filter or tag left some out. */
+  ran: number;
+  /** One per cell. */
+  rates: FamilyRate[];
+  /** One per variant cell, with `compare`. */
+  lifts: FamilyLift[];
+  /** `fail` when a lift is below its `minLift`. The cases still pass or fail on their own. */
+  outcome: 'pass' | 'fail';
+};
+/** A variant cell's pass@1 against the control's, on the same harness and sandbox. */
 type Lift = {
   /** The control cell's key. */
   vs: string;
@@ -350,7 +456,7 @@ type Lift = {
     pass: number;
     counted: number;
   };
-  /** Variant pass rate minus control pass rate, from -1 to 1. */
+  /** Variant pass@1 minus control pass@1, from -1 to 1. */
   points: number;
   /** 95% interval on the difference, by Newcombe's hybrid score method from two Wilson intervals. */
   interval: {
@@ -402,14 +508,39 @@ type CellRecord = Cell & {
   fail: number;
   infraError: number;
   skipped: number;
+  /** pass@1: pass / (pass + fail); infra errors and skips excluded. */
   passRate: number | null;
   /**
-   * By `gate`. `'all'` matches `{ passRate: 1 }` (infra only when nothing counted); `'none'`
+   * By `gate`. `'all'` (pass^k: every counted attempt passes) matches `{ passRate: 1 }`;
+   * `{ passRate }` is a pass@1 threshold. Infra only when nothing counted; `'none'`
    * reports the worst outcome. A variant below its `minLift` fails too.
    */
   outcome: Outcome;
   lift?: Lift;
+  /**
+   * Each check (assertion label) seen in any attempt of the eval, over this cell's counted
+   * attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
+   */
+  checks?: CellCheck[];
 };
+/** How often one check passed: a counted attempt that never reached it counts it `notRun`. */
+type CheckTally = {
+  pass: number;
+  fail: number;
+  error: number;
+  notRun: number;
+};
+/**
+ * One check over a cell's counted attempts. `verdict` (variant cells): `discriminates` when it
+ * passes on every counted attempt of one side and none of the other; `both-pass` or `both-fail`
+ * when it does the same on both sides, so it can't tell them apart; `mixed` otherwise.
+ */
+type CellCheck = CheckTally & {
+  label: string;
+  control?: CheckTally;
+  verdict?: CheckVerdict;
+};
+type CheckVerdict = 'discriminates' | 'both-pass' | 'both-fail' | 'mixed';
 /** Counts attempts, except `evals`. */
 type RunSummary = {
   evals: number;
@@ -508,6 +639,8 @@ type RunRecord = {
   };
   summary: RunSummary;
   evals: EvalEntry[];
+  /** Each pooled `evaluate.each` that ran; unset when none did. */
+  families?: FamilyRecord[];
 };
 /** What ran and graded the agents: the machine, images, harnesses and judge. */
 type Stack = {
@@ -646,6 +779,8 @@ type HistoryEntry = {
     passRate: number | null;
     cells: HistoryCell[];
   }>;
+  /** The run's pooled families; unset when none ran. */
+  families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'>>;
 };
 //#endregion
 //#region src/types.d.ts
@@ -680,6 +815,8 @@ type TranscriptEvent = EventTime & ({
   query: string;
   /** How many sources the harness says it used; Cursor sends no URLs or titles. */
   references: number | null;
+  /** Whether the search succeeded. Absent means it did. */
+  ok?: boolean;
   durationMs: number | null;
 } | {
   type: 'web-fetch';
@@ -1000,6 +1137,12 @@ type AttemptContext = {
   capture(target: number | ServeHandle, options?: CaptureCall): Promise<Capture>;
   /** Workspace-relative, or `~/` for the home directory; `null` when there's no such file. */
   readFile(path: string): Promise<string | null>;
+  /**
+   * Writes a file into the sandbox (workspace-relative, or `~/`), creating its directories.
+   * Later turns' `diff` leaves it out, as it does the environment's files, but shows any change
+   * made to it afterwards. Needs a sandbox, same as `c.exec`.
+   */
+  writeFile(path: string, contents: string | Uint8Array): Promise<void>;
   /**
    * Starts the sandbox and the agent session without a turn, for checks on what the environment
    * set up. `c.send()` starts them too; calling both starts them once.
@@ -1125,16 +1268,18 @@ type CursorHarnessDescriptor = {
    */
   startTimeout?: Duration;
 };
+/** Reasoning effort the Claude Code Agent SDK `/effort` control takes. */
+type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
 /** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
 type ClaudeCodeHarnessDescriptor = {
   kind: 'claude-code';
   /** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
   model: string;
   /**
-   * Reasoning effort the Agent SDK `/effort` control takes, e.g. `low`, `medium`, `high`,
+   * Reasoning effort the Agent SDK `/effort` control takes: `low`, `medium`, `high`,
    * `xhigh` (Extra in the UI) or `max`.
    */
-  effort?: string;
+  effort?: ClaudeCodeEffort;
   /** Cell id; defaults to `claude-code:<model>`, or `claude-code:<model>[effort=<value>]` when `effort` is set. */
   id?: string;
   /** The most Claude Code attempts at once, across the run, e.g. for the account's rate limits. */
@@ -1296,8 +1441,13 @@ type DockerSandboxDescriptor = {
   root?: boolean;
   /** Linux capabilities to add back, e.g. `['NET_RAW']`; every other one is dropped. */
   capabilities?: string[];
-  /** The container's memory, with no swap on top: `'4g'` (default), `'512m'`, or bytes. */
+  /** The container's memory: `'4g'` (default), `'512m'`, or bytes. */
   memory?: string | number;
+  /**
+   * Swap on top of `memory`, in the same units. Default: as much as `memory`; `0` makes `memory`
+   * the hard ceiling. A host without swap has none to give.
+   */
+  swap?: string | number;
   /** CPUs the container may use, fractions allowed. Default 2, capped at the Docker host's. */
   cpus?: number;
   /** Processes and threads at once. Default 1024. */
@@ -1671,7 +1821,8 @@ type CompareAnswer = {
 type PinnableKey = 'harness' | 'runs' | 'compare';
 /**
  * How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
- * passes; `{ passRate }` passes when at least that share of its passes and fails are passes.
+ * passes (pass^k); `{ passRate }` passes when its pass@1 (passes over passes and fails) is at
+ * least that share.
  */
 type GatePolicy = 'all' | {
   passRate: number;
@@ -1921,11 +2072,23 @@ type PrepareFailureRecord = {
   log: string | null;
   setBy: string[];
 };
-/** Why an attempt failed; `prepare` when its sandbox couldn't be prepared. */
+/**
+ * The sandbox ran out of a resource during the attempt: `oom` is the kernel killing `kills`
+ * processes at its memory limit. The attempt is then an infra error, whatever its checks said.
+ */
+type ResourceFailureRecord = {
+  cause: 'oom';
+  kills: number;
+};
+/**
+ * Why an attempt failed; `prepare` when its sandbox couldn't be prepared, `resource` when it ran
+ * out of one.
+ */
 type AttemptError = {
   kind: ErrorKind;
   message: string;
   prepare?: PrepareFailureRecord;
+  resource?: ResourceFailureRecord;
 };
 /** Where an eval's time went, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
 type PhaseTimings = {
@@ -2060,6 +2223,25 @@ type StartRetry = {
   /** How long it waited before giving up on that start. */
   waitedMs: number;
 };
+/** One `expect` or `expect.soft` matcher call an attempt made. */
+type AssertionRecord = {
+  /**
+   * `expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or
+   * the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt.
+   */
+  label: string;
+  /** The eval gave the label. */
+  labelled: boolean;
+  /** `toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`. */
+  matcher: string;
+  /** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
+  status: 'pass' | 'fail' | 'error';
+  soft: boolean;
+  /** Why it failed or couldn't answer, cut at 1 KB. */
+  message?: string;
+  /** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
+  durationMs?: number;
+};
 /** Which environment ran as the control, so the studio can show it. */
 type EvalCompareMeta = {
   baseline: string;
@@ -2087,5 +2269,5 @@ type SandboxWarning = {
   message: string;
 };
 //#endregion
-export { HarnessDescriptor as $, TOKENS_UNREPORTED as $t, DockerSandboxDescriptor as A, TurnData as At, EvalFn as B, FileRef as Bt, CompareOptions as C, SecretTemplate as Ct, CursorHarnessDescriptor as D, TokenUsage as Dt, ConfigJudgeOptions as E, SkillSource as Et, Environment as F, WebEventType as Ft, FakeJudgeModel as G, RunRecord as Gt, ExecResult as H, HistoryEntry as Ht, EnvironmentHelpers as I, ArtifactRef as It, FakeScriptStep as J, EFFORT_ALPHA as Jt, FakeNoteInput as K, StackSummary as Kt, EnvironmentInput as L, AttemptRecord as Lt, EgressPolicy as M, Verdict as Mt, EgressRecord as N, Viewport as Nt, Diff as O, TranscriptEvent as Ot, EnvVars as P, Wait as Pt, HarnessData as Q, Spread as Qt, EnvironmentSpec as R, CellRecord as Rt, CompareCut as S, Secret as St, ConfigHelpers as T, SetupStep as Tt, FakeAnalysisModel as U, JudgementRecord as Ut, EvalOptions as V, HistoryCell as Vt, FakeHarnessDescriptor as W, Lift as Wt, GooseHarnessDescriptor as X, EffortSample as Xt, FakeVote as Y, EffortContrast as Yt, GroupOptions as Z, EffortVerdict as Zt, Cell as _, Profile as _t, BackgroundHandle as a, effortChangeLabel as an, JudgeCallOptions as at, CodexHarnessDescriptor as b, SandboxDescriptor as bt, Capture as c, effortSample as cn, JudgeOptions as ct, CaptureOptions as d, formatCount as dn, OpenCodeHarnessDescriptor as dt, VERDICT_TEXT as en, HarnessEdit as et, CaptureUse as f, formatRatio as fn, Outcome as ft, CaptureWindow as g, PinnableKey as gt, CaptureWait as h, PiHarnessDescriptor as ht, AttemptContext as i, contrastEffort as in, JudgeArtifactRef as it, Duration as j, TurnStatus as jt, DirectoryOptions as k, Turn as kt, CaptureCall as l, effortSummary as ln, JudgeState as lt, CaptureUseScript as m, tokenValues as mn, PairVote as mt, AnalysisConfig as n, cellEffort as nn, HistoryConfig as nt, BlindPair as o, effortLabel as on, JudgeFailure as ot, CaptureUseAction as p, formatSeconds as pn, PairFactor as pt, FakeSandboxDescriptor as q, CellEffort as qt, AntigravityHarnessDescriptor as r, cellSamples as rn, HostFile as rt, Budget as s, effortOf as sn, JudgeModel as st, AgentHarnessDescriptor as t, cellContrast as tn, HarnessOption as tt, CaptureInstruction as u, effortVerdict as un, McpServer as ut, CellRole as v, ProfileDefaults as vt, CompareQuestion as w, ServeHandle as wt, CompareAnswer as x, SandboxWarning as xt, ClaudeCodeHarnessDescriptor as y, ProfileLimit as yt, EsConfig as z, EvalEntry as zt };
-//# sourceMappingURL=types-_LOFdPUf.d.ts.map
+export { HarnessData as $, EffortContrast as $t, DirectoryOptions as A, Turn as At, EsConfig as B, EvalEntry as Bt, CompareCut as C, Secret as Ct, ConfigJudgeOptions as D, SkillSource as Dt, ConfigHelpers as E, SetupStep as Et, EnvVars as F, Wait as Ft, FakeHarnessDescriptor as G, HistoryCell as Gt, EvalOptions as H, FamilyRate as Ht, Environment as I, WebEventType as It, FakeSandboxDescriptor as J, Lift as Jt, FakeJudgeModel as K, HistoryEntry as Kt, EnvironmentHelpers as L, ArtifactRef as Lt, Duration as M, TurnStatus as Mt, EgressPolicy as N, Verdict as Nt, CursorHarnessDescriptor as O, TokenUsage as Ot, EgressRecord as P, Viewport as Pt, GroupOptions as Q, EFFORT_ALPHA as Qt, EnvironmentInput as R, AttemptRecord as Rt, CompareAnswer as S, SandboxWarning as St, CompareQuestion as T, ServeHandle as Tt, ExecResult as U, FamilyRecord as Ut, EvalFn as V, FamilyLift as Vt, FakeAnalysisModel as W, FileRef as Wt, FakeVote as X, StackSummary as Xt, FakeScriptStep as Y, RunRecord as Yt, GooseHarnessDescriptor as Z, CellEffort as Zt, Cell as _, formatSeconds as _n, PinnableKey as _t, BackgroundHandle as a, cellContrast as an, JudgeArtifactRef as at, ClaudeCodeHarnessDescriptor as b, ProfileLimit as bt, Capture as c, contrastEffort as cn, JudgeModel as ct, CaptureOptions as d, effortOf as dn, McpServer as dt, EffortSample as en, HarnessDescriptor as et, CaptureUse as f, effortSample as fn, OpenCodeHarnessDescriptor as ft, CaptureWindow as g, formatRatio as gn, PiHarnessDescriptor as gt, CaptureWait as h, formatCount as hn, PairVote as ht, AttemptContext as i, VERDICT_TEXT as in, HostFile as it, DockerSandboxDescriptor as j, TurnData as jt, Diff as k, TranscriptEvent as kt, CaptureCall as l, effortChangeLabel as ln, JudgeOptions as lt, CaptureUseScript as m, effortVerdict as mn, PairFactor as mt, AnalysisConfig as n, Spread as nn, HarnessOption as nt, BlindPair as o, cellEffort as on, JudgeCallOptions as ot, CaptureUseAction as p, effortSummary as pn, Outcome as pt, FakeNoteInput as q, JudgementRecord as qt, AntigravityHarnessDescriptor as r, TOKENS_UNREPORTED as rn, HistoryConfig as rt, Budget as s, cellSamples as sn, JudgeFailure as st, AgentHarnessDescriptor as t, EffortVerdict as tn, HarnessEdit as tt, CaptureInstruction as u, effortLabel as un, JudgeState as ut, CellRole as v, tokenValues as vn, Profile as vt, CompareOptions as w, SecretTemplate as wt, CodexHarnessDescriptor as x, SandboxDescriptor as xt, ClaudeCodeEffort as y, ProfileDefaults as yt, EnvironmentSpec as z, CellRecord as zt };
+//# sourceMappingURL=types-D8RLfkFP.d.ts.map