reeval 0.0.1 → 0.3.0

package/THIRD_PARTY_NOTICES.md

@@ -44,8 +44,8 @@ with its own licence.
 | @radix-ui/react-use-size | 1.1.4 | MIT | dist/studio-app |
 | @radix-ui/react-visually-hidden | 1.2.11 | MIT | dist/studio-app |
 | @tanstack/history | 1.162.4 | MIT | dist/studio-app |
-| @tanstack/query-core | 5.103.2 | MIT | dist/studio-app |
-| @tanstack/react-query | 5.103.2 | MIT | dist/studio-app |
+| @tanstack/query-core | 5.104.0 | MIT | dist/studio-app |
+| @tanstack/react-query | 5.104.0 | MIT | dist/studio-app |
 | @tanstack/react-router | 1.170.39 | MIT | dist/studio-app |
 | @tanstack/react-store | 0.11.1 | MIT | dist/studio-app |
 | @tanstack/react-virtual | 3.14.13 | MIT | dist/studio-app |
@@ -1163,7 +1163,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ```
 
-## @tanstack/query-core@5.103.2
+## @tanstack/query-core@5.104.0
 
 - Licence: MIT
 - Source: git+https://github.com/TanStack/query.git
@@ -1193,7 +1193,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ```
 
-## @tanstack/react-query@5.103.2
+## @tanstack/react-query@5.104.0
 
 - Licence: MIT
 - Source: git+https://github.com/TanStack/query.git

package/assets/package.json

@@ -4,6 +4,6 @@
   "private": true,
   "type": "module",
   "dependencies": {
-    "@earendil-works/pi-coding-agent": "0.87.1"
+    "@earendil-works/pi-coding-agent": "0.99.1"
   }
 }

package/assets/pnpm-lock.yaml

@@ -9,8 +9,8 @@ importers:
   .:
     dependencies:
       '@earendil-works/pi-coding-agent':
-        specifier: 0.87.1
-        version: 0.87.1(ws@8.22.0)
+        specifier: 0.99.1
+        version: 0.99.1(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(ws@8.22.0)
 
 packages:
 
@@ -107,30 +107,38 @@ packages:
     resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==}
     engines: {node: '>=6.9.0'}
 
-  '@earendil-works/chord@0.87.1':
-    resolution: {integrity: sha512-bg7IkJGFcEaMqqYgOGUiq5Ky9RghpRfrlZ8I/v/1b4bBZ02A7t3E+6uhPRbadwWb/kWsnVFbZsqOKRN4a3LLCg==}
+  '@earendil-works/chord@0.99.1':
+    resolution: {integrity: sha512-4xyn0IBzJ+Xu/iOGi2hjXJGAR61QEhEWZsIqTDqr+GmItdquYwBO5jYFnqGiBaTqlY12/EpM7QHoEKSHbyvOug==}
     engines: {node: '>=22.19.0'}
 
-  '@earendil-works/pi-agent-core@0.87.1':
-    resolution: {integrity: sha512-Zev3B0HK7YS5A4EZQ2XnEqiJuirx6QBiltJ+LpmjV5a/+2IU0cfKtIfnkNkORK707XOvKBY2WRtk7cAwHpbh2Q==}
+  '@earendil-works/pi-agent-core@0.99.1':
+    resolution: {integrity: sha512-zywvWnj5FujeuFI/x/CJHwwxhcLIQgjqseTA+bQgX4O8gJTcgjRd/I8SZnQDqJvxC9QcV12ujiGLviv6EgwcCg==}
     engines: {node: '>=22.19.0'}
 
-  '@earendil-works/pi-ai@0.87.1':
-    resolution: {integrity: sha512-X/3PfQBnnoeVdO9Cv8zHghUMglzlgNZYGNzoPnbRoGnHl3Rw3TlA2UKSUB7BRHUOxMryHXYa8dnjWZlbRheDZA==}
+  '@earendil-works/pi-ai@0.99.1':
+    resolution: {integrity: sha512-4nV9JKc94iPX8bwdGPc2nTuVPKIPsffhnp3WoN9NYCNqbtoOF8LhYcIs/+Sn/alroqJK/5QRu6/Z6Ck+n0hyBA==}
     engines: {node: '>=22.19.0'}
     hasBin: true
 
-  '@earendil-works/pi-coding-agent@0.87.1':
-    resolution: {integrity: sha512-m8ArJUtVcQMSe1lLE/Ei7vX/JV7O39sWmWBsXV2NOU70F0qCp8GubA24pT3LnwTmM6LL2xV80/h6sQg85n69ew==}
+  '@earendil-works/pi-codemode@0.99.1':
+    resolution: {integrity: sha512-oh8TMsBI3SWTN3xTQtX8u5n+BKhnVXcFagroWumfn6/WWfBnDYL/LmeQtjLb83WRTb9rcu+ZdK8rFa4vggvCJg==}
+    engines: {node: '>=22.19.0'}
+
+  '@earendil-works/pi-coding-agent@0.99.1':
+    resolution: {integrity: sha512-cWUrTOqA5M73cOYMgsh9PlhDrsBhavd+n5kVY6F7BGbGl1RjqCteVCoeVMVqhngoGACVDyw1tbLjajL8l9jrHg==}
     engines: {node: '>=22.19.0'}
     hasBin: true
 
-  '@earendil-works/pi-telemetry@0.87.1':
-    resolution: {integrity: sha512-MC6TRQH5lgMXpcN+Vku2WMI2T8BsiUPzMQHGo81uqFZD3/9O79WWJAysEDGuzduP6R4tvtgwMLwmqIxynM10JQ==}
+  '@earendil-works/pi-mcp@0.99.1':
+    resolution: {integrity: sha512-YCFGPkmDzLwQuIzwfbP6Vuk/g/ukKpZhwTpbcfzomuI1Fkiu6hHRkOGwAqsO3G8cTkZWkM8vmOkFJjStQNC4qA==}
     engines: {node: '>=22.19.0'}
 
-  '@earendil-works/pi-tui@0.87.1':
-    resolution: {integrity: sha512-YEH2vRyOeiO7hhN6j6AE6YwKSq2Kz2f3XR8bj1TbR+aGE/JsnY1hLPMI2pvaZfRM1n9Y00tejxFQ4zbzvF7nkQ==}
+  '@earendil-works/pi-telemetry@0.99.1':
+    resolution: {integrity: sha512-9PBPjGk+TXRtuMianpqBbHBpYpyKusESF6rwdmgD0WTZSTUQXhcKEO0hAINRLuSwy4V7yPvXV+EVV0ONY7mbpQ==}
+    engines: {node: '>=22.19.0'}
+
+  '@earendil-works/pi-tui@0.99.1':
+    resolution: {integrity: sha512-gZp0Guat96Fr1AuC/xqVz5B2lulZakp/PxD1lXx3lSgBdjiqmwYhJbcQ0HRrGAfy0WtMGn9b05RJr5qJf7oIuw==}
     engines: {node: '>=22.19.0'}
 
   '@esbuild/aix-ppc64@0.28.2':
@@ -534,12 +542,25 @@ packages:
     resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
 
-  openai@6.40.0:
-    resolution: {integrity: sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==}
+  openai@7.19.0:
+    resolution: {integrity: sha512-MX2s3u2L5racTO0CC/SWpCOasJQBCJrqLKXK+l82cAhdeF8mPMBEe/gxMm0ZFa2xpKpOFLRjxv5afYEZbBXmbQ==}
+    engines: {node: '>=22.0.0'}
     peerDependencies:
-      ws: ^8.18.0
+      '@aws-sdk/credential-provider-node': '>=3.972.0 <4'
+      '@smithy/hash-node': '>=4.3.0 <5'
+      '@smithy/signature-v4': '>=5.4.0 <6'
+      undici: '>=5 <9'
+      ws: ^8.21.0
       zod: ^3.25 || ^4.0
     peerDependenciesMeta:
+      '@aws-sdk/credential-provider-node':
+        optional: true
+      '@smithy/hash-node':
+        optional: true
+      '@smithy/signature-v4':
+        optional: true
+      undici:
+        optional: true
       ws:
         optional: true
       zod:
@@ -572,6 +593,9 @@ packages:
       kerberos:
         optional: true
 
+  quickjs-wasi@3.6.2:
+    resolution: {integrity: sha512-FCqGtGOrMgzUiIrMNMA2YnsOxCNwo31dzqXvclXUC6xeT35NJLKXQJsvbeCTjvoFAwZgEAPg8U6+KAPDGXn8Mg==}
+
   retry@0.12.0:
     resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==}
     engines: {node: '>= 4'}
@@ -835,55 +859,69 @@ snapshots:
 
   '@babel/runtime@7.29.7': {}
 
-  '@earendil-works/chord@0.87.1':
+  '@earendil-works/chord@0.99.1':
     dependencies:
       esbuild: 0.28.2
 
-  '@earendil-works/pi-agent-core@0.87.1(ws@8.22.0)':
+  '@earendil-works/pi-agent-core@0.99.1(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(undici@8.10.2)(ws@8.22.0)':
     dependencies:
-      '@earendil-works/chord': 0.87.1
-      '@earendil-works/pi-ai': 0.87.1(ws@8.22.0)
-      '@earendil-works/pi-telemetry': 0.87.1
+      '@earendil-works/chord': 0.99.1
+      '@earendil-works/pi-ai': 0.99.1(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(undici@8.10.2)(ws@8.22.0)
+      '@earendil-works/pi-telemetry': 0.99.1
       diff: 8.0.4
       ignore: 7.0.8
       typebox: 1.3.27
       yaml: 2.9.0
     transitivePeerDependencies:
+      - '@aws-sdk/credential-provider-node'
       - '@modelcontextprotocol/sdk'
+      - '@smithy/hash-node'
+      - '@smithy/signature-v4'
       - bufferutil
       - kerberos
       - supports-color
+      - undici
       - utf-8-validate
       - ws
       - zod
 
-  '@earendil-works/pi-ai@0.87.1(ws@8.22.0)':
+  '@earendil-works/pi-ai@0.99.1(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(undici@8.10.2)(ws@8.22.0)':
     dependencies:
       '@anthropic-ai/sdk': 0.124.0
       '@aws-sdk/client-bedrock-runtime': 3.1127.0
-      '@earendil-works/pi-telemetry': 0.87.1
+      '@earendil-works/pi-telemetry': 0.99.1
       '@google/genai': 2.21.0
       '@smithy/node-http-handler': 4.12.1
       http-proxy-agent: 9.1.0
       https-proxy-agent: 9.1.0
-      openai: 6.40.0(ws@8.22.0)
+      openai: 7.19.0(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(undici@8.10.2)(ws@8.22.0)
       partial-json: 0.1.7
       typebox: 1.3.27
     transitivePeerDependencies:
+      - '@aws-sdk/credential-provider-node'
       - '@modelcontextprotocol/sdk'
+      - '@smithy/hash-node'
+      - '@smithy/signature-v4'
       - bufferutil
       - kerberos
       - supports-color
+      - undici
       - utf-8-validate
       - ws
       - zod
 
-  '@earendil-works/pi-coding-agent@0.87.1(ws@8.22.0)':
+  '@earendil-works/pi-codemode@0.99.1':
     dependencies:
-      '@earendil-works/chord': 0.87.1
-      '@earendil-works/pi-agent-core': 0.87.1(ws@8.22.0)
-      '@earendil-works/pi-ai': 0.87.1(ws@8.22.0)
-      '@earendil-works/pi-tui': 0.87.1
+      quickjs-wasi: 3.6.2
+
+  '@earendil-works/pi-coding-agent@0.99.1(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(ws@8.22.0)':
+    dependencies:
+      '@earendil-works/chord': 0.99.1
+      '@earendil-works/pi-agent-core': 0.99.1(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(undici@8.10.2)(ws@8.22.0)
+      '@earendil-works/pi-ai': 0.99.1(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(undici@8.10.2)(ws@8.22.0)
+      '@earendil-works/pi-codemode': 0.99.1
+      '@earendil-works/pi-mcp': 0.99.1
+      '@earendil-works/pi-tui': 0.99.1
       '@silvia-odwyer/photon-node': 0.3.4
       chalk: 6.0.0
       cross-spawn: 7.0.6
@@ -895,12 +933,16 @@ snapshots:
       jiti: 2.7.0
       minimatch: 10.2.6
       proper-lockfile: 4.1.2
+      quickjs-wasi: 3.6.2
       semver: 7.8.5
       typebox: 1.3.27
       undici: 8.10.2
       yaml: 2.9.0
     transitivePeerDependencies:
+      - '@aws-sdk/credential-provider-node'
       - '@modelcontextprotocol/sdk'
+      - '@smithy/hash-node'
+      - '@smithy/signature-v4'
       - bufferutil
       - kerberos
       - supports-color
@@ -908,9 +950,13 @@ snapshots:
       - ws
       - zod
 
-  '@earendil-works/pi-telemetry@0.87.1': {}
+  '@earendil-works/pi-mcp@0.99.1':
+    dependencies:
+      cross-spawn: 7.0.6
+
+  '@earendil-works/pi-telemetry@0.99.1': {}
 
-  '@earendil-works/pi-tui@0.87.1':
+  '@earendil-works/pi-tui@0.99.1':
     dependencies:
       get-east-asian-width: 1.6.0
       marked: 18.0.11
@@ -1259,8 +1305,11 @@ snapshots:
       fetch-blob: 3.2.0
       formdata-polyfill: 4.0.10
 
-  openai@6.40.0(ws@8.22.0):
+  openai@7.19.0(@aws-sdk/credential-provider-node@3.972.84)(@smithy/signature-v4@5.7.4)(undici@8.10.2)(ws@8.22.0):
     optionalDependencies:
+      '@aws-sdk/credential-provider-node': 3.972.84
+      '@smithy/signature-v4': 5.7.4
+      undici: 8.10.2
       ws: 8.22.0
 
   p-retry@4.6.2:
@@ -1294,6 +1343,8 @@ snapshots:
 
   proxy-agent-negotiate@1.1.0: {}
 
+  quickjs-wasi@3.6.2: {}
+
   retry@0.12.0: {}
 
   retry@0.13.1: {}

package/dist/agent-BYy0Tc6P.js

@@ -0,0 +1,334 @@
+import { a as fromCwd } from "./ids-Bp6ut7ZD.js";
+import { a as notesDir, u as writeNote } from "./files-CfsIRN7n.js";
+import { a as providerModel } from "./model-GFM8iOzM.js";
+import { a as stagedAttempts, i as listOutcomes, n as divergence, t as attemptSummary } from "./tools-B7SoW5kU.js";
+import { z } from "zod";
+import { generateText, stepCountIs, tool } from "ai";
+/** Pairs the fake compares, as a careful model would: a few, not all of them. */
+const FAKE_PAIRS = 3;
+const NOTE_PROMPT = `You write a short note on how one coding agent (the harness) did on one eval of a reeval run. You can't see transcripts; read the attempts with the tools.
+
+1. Call list_attempts first.
+2. If any attempt failed, call compare_attempts on its suggested pairs (at most three). Say where the failed attempt's steps diverged from the passing one's, and whether the miss looks like the text the agent was given (a skill, docs, rules), the eval itself (its prompt or checks), or the harness (errors, retries, tool failures). Open an attempt with open_attempt only when the comparison isn't enough.
+3. If nothing failed, say so in one line, and name anything unusual (step counts, files, verdicts close to the threshold).
+
+Plain text, at most 150 words, no headings. Name attempts by their id, e.g. cursor:composer-2.5/with-skill/docker#2. Say what the attempts show, not what you guess.`;
+/** The tools over `attempts`, with every result counted against `maxChars`. */
+function noteTools(attempts, maxChars) {
+	let left = maxChars;
+	const spend = (value) => {
+		const size = JSON.stringify(value).length;
+		if (size > left) return { error: "The input budget (analysis.maxInputTokens) is spent: write the note from what you have." };
+		left -= size;
+		return value;
+	};
+	const byId = (id) => attempts.find((a) => a.id === id);
+	const unknown = (id) => ({ error: `No attempt ${id}. The attempts are ${attempts.map((a) => a.id).join(", ")}.` });
+	return {
+		list_attempts: tool({
+			description: "Every attempt's outcome, error, failed checks and verdicts in a line, and failed-vs-passed pairs to compare.",
+			inputSchema: z.object({}),
+			execute: async () => spend(listOutcomes(attempts))
+		}),
+		open_attempt: tool({
+			description: "One attempt: each turn's prompt, steps (tool calls, edits, errors) and final message, its checks, verdicts and diff.",
+			inputSchema: z.object({ id: z.string().describe("An attempt id from list_attempts") }),
+			execute: async ({ id }) => {
+				const a = byId(id);
+				return a ? spend(attemptSummary(a)) : unknown(id);
+			}
+		}),
+		compare_attempts: tool({
+			description: "Where a failed attempt's steps first differ from a passing one's, the checks, verdicts and files that differ, and each diff's start.",
+			inputSchema: z.object({
+				failed: z.string().describe("The failed attempt id"),
+				passed: z.string().describe("The passing attempt id")
+			}),
+			execute: async ({ failed, passed }) => {
+				const f = byId(failed);
+				const p = byId(passed);
+				if (!f) return unknown(failed);
+				if (!p) return unknown(passed);
+				return spend(divergence(f, p));
+			}
+		})
+	};
+}
+/** Writes one note; a failed call is a note with `error`, never a throw. */
+async function writeAgentNote(input) {
+	const { model, runId, evalId, harness, attempts } = input;
+	const base = {
+		kind: "reeval-analysis-note",
+		schemaVersion: 1,
+		runId,
+		evalId,
+		harness,
+		model: model.name,
+		attempts: attempts.map((a) => a.id)
+	};
+	const count = (o) => attempts.filter((a) => a.outcome === o).length;
+	const prompt = [
+		`Eval: ${evalId}`,
+		`Harness: ${harness}`,
+		`Attempts: ${attempts.length} (${count("pass")} passed, ${count("fail")} failed, ${count("infra_error")} infra errors, ${count("skipped")} skipped)`,
+		...input.instructions ? ["", `What matters in this project: ${input.instructions}`] : []
+	].join("\n");
+	try {
+		const result = await generateText({
+			model: model.fake ? fakeNoteModel(model.fake, {
+				eval: evalId,
+				harness
+			}) : await providerModel(model),
+			system: NOTE_PROMPT,
+			prompt,
+			tools: noteTools(attempts, input.maxInputTokens * 4),
+			stopWhen: stepCountIs(8),
+			prepareStep: ({ stepNumber }) => stepNumber === 7 ? { toolChoice: "none" } : void 0,
+			temperature: .2,
+			maxOutputTokens: input.maxOutputTokens,
+			maxRetries: 3,
+			...input.signal ? { abortSignal: input.signal } : {}
+		});
+		const text = result.text.trim();
+		return {
+			...base,
+			resolvedModel: result.response?.modelId ?? null,
+			createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+			text,
+			...text ? {} : { error: `the model wrote no note in 8 steps` },
+			usage: {
+				inputTokens: result.totalUsage?.inputTokens ?? null,
+				outputTokens: result.totalUsage?.outputTokens ?? null
+			}
+		};
+	} catch (error) {
+		return {
+			...base,
+			resolvedModel: null,
+			createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+			text: "",
+			error: error instanceof Error ? error.message : String(error),
+			usage: {
+				inputTokens: null,
+				outputTokens: null
+			}
+		};
+	}
+}
+/** Each tool result so far in the conversation, by tool name. */
+function toolResults(prompt) {
+	const results = [];
+	for (const message of prompt) {
+		if (message.role !== "tool" || !Array.isArray(message.content)) continue;
+		for (const part of message.content) if (part.type === "tool-result" && part.toolName && part.output) results.push({
+			tool: part.toolName,
+			value: part.output.value
+		});
+	}
+	return results;
+}
+/**
+* A language model that does what the prompt asks of a real one: lists the attempts, compares up
+* to three failed-vs-passed pairs, then answers with the fake's note, through the AI SDK's own
+* tool loop, so the tools and the budget run as they would.
+*/
+function fakeNoteModel(fake, about) {
+	const usage = {
+		inputTokens: {
+			total: 0,
+			noCache: 0,
+			cacheRead: 0,
+			cacheWrite: 0
+		},
+		outputTokens: {
+			total: 0,
+			text: 0,
+			reasoning: 0
+		}
+	};
+	let calls = 0;
+	const toolCalls = (calls) => ({
+		content: calls.map((c, i) => ({
+			type: "tool-call",
+			toolCallId: `fake_${c.name}_${i}`,
+			toolName: c.name,
+			input: JSON.stringify(c.input)
+		})),
+		finishReason: {
+			unified: "tool-calls",
+			raw: "tool_calls"
+		},
+		usage,
+		warnings: [],
+		response: { modelId: "fake-analysis" }
+	});
+	return {
+		specificationVersion: "v4",
+		provider: "fake",
+		modelId: "fake-analysis",
+		supportedUrls: {},
+		doGenerate: async (options) => {
+			calls++;
+			const { note } = fake;
+			if (typeof note === "object" && note !== null) throw new Error(note.error);
+			const results = toolResults(options.prompt);
+			const listed = results.find((r) => r.tool === "list_attempts")?.value;
+			if (!listed) return toolCalls([{
+				name: "list_attempts",
+				input: {}
+			}]);
+			const compared = results.filter((r) => r.tool === "compare_attempts").map((r) => r.value);
+			const pairs = listed.pairs ?? [];
+			if (pairs.length && !compared.length && calls < 8) return toolCalls(pairs.slice(0, FAKE_PAIRS).map((p) => ({
+				name: "compare_attempts",
+				input: {
+					failed: p.failed,
+					passed: p.passed
+				}
+			})));
+			const input = {
+				...about,
+				attempts: (listed.attempts ?? []).map((a) => ({
+					id: a.id,
+					outcome: a.outcome
+				})),
+				compared: compared.filter((c) => c && "failed" in c).map((c) => ({
+					failed: c.failed,
+					passed: c.passed,
+					divergedAt: c.divergedAt
+				}))
+			};
+			return {
+				content: [{
+					type: "text",
+					text: typeof note === "function" ? note(input) : note ?? `${input.attempts.filter((a) => a.outcome === "pass").length} of ${input.attempts.length} attempts passed.`
+				}],
+				finishReason: {
+					unified: "stop",
+					raw: "stop"
+				},
+				usage,
+				warnings: [],
+				response: { modelId: "fake-analysis" }
+			};
+		},
+		doStream: async () => {
+			throw new Error("the fake analysis model does not stream");
+		}
+	};
+}
+//#endregion
+//#region src/analysis/agent/index.ts
+/** Notes in flight at once: a run with many evals and harnesses doesn't burst the provider. */
+const MAX_NOTES = 4;
+const groupKey = (evalId, harness) => `${evalId}\0${harness}`;
+function startNotes({ config, model, runId, stagingDir, planned, warn }) {
+	const expected = /* @__PURE__ */ new Map();
+	const order = new Map(planned.map((p, i) => [p.key, i]));
+	for (const p of planned) {
+		if (p.attempt.plan.skip) continue;
+		const key = groupKey(p.id, p.attempt.cell.harness);
+		expected.set(key, (expected.get(key) ?? 0) + 1);
+	}
+	const finished = /* @__PURE__ */ new Map();
+	const started = /* @__PURE__ */ new Set();
+	const pending = [];
+	const abort = new AbortController();
+	let running = 0;
+	const waiting = [];
+	const slot = async () => {
+		if (running < MAX_NOTES) running++;
+		else await new Promise((resolve) => waiting.push(resolve));
+	};
+	const release = () => {
+		const next = waiting.shift();
+		if (next) next();
+		else running--;
+	};
+	const start = (key) => {
+		const results = (finished.get(key) ?? []).sort((a, b) => (order.get(a.key) ?? 0) - (order.get(b.key) ?? 0));
+		started.add(key);
+		const first = results[0];
+		if (!first?.attempt || results.every((r) => r.state === "skipped")) return;
+		const harness = first.attempt.cell.harness;
+		const about = `analysis note on ${first.id} (${harness})`;
+		pending.push((async () => {
+			await slot();
+			try {
+				if (abort.signal.aborted) return null;
+				const note = await writeAgentNote({
+					model,
+					runId,
+					evalId: first.id,
+					harness,
+					attempts: await stagedAttempts(stagingDir, results),
+					maxInputTokens: config.analysis.maxInputTokens,
+					maxOutputTokens: config.analysis.maxOutputTokens,
+					...config.analysis.instructions ? { instructions: config.analysis.instructions } : {},
+					signal: abort.signal
+				});
+				if (abort.signal.aborted) return null;
+				if (note.error) warn(`${about} failed: ${note.error}`);
+				await writeNote(config.root, note);
+				return note;
+			} catch (error) {
+				if (!abort.signal.aborted) warn(`${about} wasn't written: ${error.message}`);
+				return null;
+			} finally {
+				release();
+			}
+		})());
+	};
+	return {
+		onResult(result) {
+			if (!result.attempt || abort.signal.aborted) return;
+			const key = groupKey(result.id, result.attempt.cell.harness);
+			if (!expected.has(key) || started.has(key)) return;
+			const results = [...finished.get(key) ?? [], result];
+			finished.set(key, results);
+			if (results.length >= (expected.get(key) ?? 0)) start(key);
+		},
+		async finish(cancelled) {
+			if (cancelled) abort.abort(/* @__PURE__ */ new Error("The run was cancelled"));
+			else for (const key of finished.keys()) if (!started.has(key)) start(key);
+			return (await Promise.all(pending)).filter((n) => n !== null);
+		}
+	};
+}
+/**
+* Writes the run's summary into the analysis cache, once the run is written; the line to print.
+* A failure is a warning: the run's outcomes and exit code stand.
+*/
+async function summarizeRun(config, runId, model) {
+	try {
+		const { writeRunSummary } = await import("./summary-DjbxgCDi.js");
+		const { doc, path } = await writeRunSummary({
+			config,
+			runId,
+			model
+		});
+		if (doc.error) return {
+			level: "warn",
+			text: `analysis summary failed: ${doc.error}`
+		};
+		return {
+			level: "info",
+			text: `analysis: summary in ${fromCwd(path)} (reeval analyze shows it)`
+		};
+	} catch (error) {
+		return {
+			level: "warn",
+			text: `analysis summary wasn't written: ${error.message}`
+		};
+	}
+}
+/** The line the run prints once its notes are written. */
+const notesLine = (root, runId, notes) => {
+	const failed = notes.filter((n) => n.error).length;
+	const n = notes.length;
+	return `analysis: ${n} note${n === 1 ? "" : "s"}${failed ? ` (${failed} failed)` : ""} in ${fromCwd(notesDir(root, runId))}`;
+};
+//#endregion
+export { notesLine, startNotes, summarizeRun };
+
+//# sourceMappingURL=agent-BYy0Tc6P.js.map

package/dist/agent-BYy0Tc6P.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-BYy0Tc6P.js","names":[],"sources":["../src/analysis/agent/note.ts","../src/analysis/agent/index.ts"],"mappings":";;;;;;;AAmBA,MAAM,aAAa;AAEnB,MAAa,cAAc;;;;;;;;AAqB3B,SAAgB,UAAU,UAAyB,UAAkB;CACnE,IAAI,OAAO;CACX,MAAM,SAAY,UAAoC;EACpD,MAAM,OAAO,KAAK,UAAU,KAAK,CAAC,CAAC;EACnC,IAAI,OAAO,MACT,OAAO,EACL,OACE,0FACJ;EAEF,QAAQ;EACR,OAAO;CACT;CACA,MAAM,QAAQ,OAAe,SAAS,MAAM,MAAM,EAAE,OAAO,EAAE;CAC7D,MAAM,WAAW,QAAgB,EAC/B,OAAO,cAAc,GAAG,qBAAqB,SAAS,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,GACpF;CACA,OAAO;EACL,eAAe,KAAK;GAClB,aACE;GACF,aAAa,EAAE,OAAO,CAAC,CAAC;GACxB,SAAS,YAAY,MAAM,aAAa,QAAQ,CAAC;EACnD,CAAC;EACD,cAAc,KAAK;GACjB,aACE;GACF,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,kCAAkC,EAAE,CAAC;GACrF,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,IAAI,KAAK,EAAE;IACjB,OAAO,IAAI,MAAM,eAAe,CAAC,CAAC,IAAI,QAAQ,EAAE;GAClD;EACF,CAAC;EACD,kBAAkB,KAAK;GACrB,aACE;GACF,aAAa,EAAE,OAAO;IACpB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,uBAAuB;IACnD,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,wBAAwB;GACtD,CAAC;GACD,SAAS,OAAO,EAAE,QAAQ,aAAa;IACrC,MAAM,IAAI,KAAK,MAAM;IACrB,MAAM,IAAI,KAAK,MAAM;IACrB,IAAI,CAAC,GAAG,OAAO,QAAQ,MAAM;IAC7B,IAAI,CAAC,GAAG,OAAO,QAAQ,MAAM;IAC7B,OAAO,MAAM,WAAW,GAAG,CAAC,CAAC;GAC/B;EACF,CAAC;CACH;AACF;;AAGA,eAAsB,eAAe,OAAyC;CAC5E,MAAM,EAAE,OAAO,OAAO,QAAQ,SAAS,aAAa;CACpD,MAAM,OAAO;EACX,MAAM;EACN,eAAe;EACf;EACA;EACA;EACA,OAAO,MAAM;EACb,UAAU,SAAS,KAAK,MAAM,EAAE,EAAE;CACpC;CACA,MAAM,SAAS,MAAc,SAAS,QAAQ,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;CACrE,MAAM,SAAS;EACb,SAAS;EACT,YAAY;EACZ,aAAa,SAAS,OAAO,IAAI,MAAM,MAAM,EAAE,WAAW,MAAM,MAAM,EAAE,WAAW,MAAM,aAAa,EAAE,iBAAiB,MAAM,SAAS,EAAE;EAC1I,GAAI,MAAM,eAAe,CAAC,IAAI,iCAAiC,MAAM,cAAc,IAAI,CAAC;CAC1F,CAAC,CAAC,KAAK,IAAI;CACX,IAAI;EACF,MAAM,SAAS,MAAM,aAAa;GAChC,OAAO,MAAM,OACT,cAAc,MAAM,MAAM;IAAE,MAAM;IAAQ;GAAQ,CAAC,IACnD,MAAM,cAAc,KAAK;GAC7B,QAAQ;GACR;GACA,OAAO,UAAU,UAAU,MAAM,iBAAiB,CAAC;GACnD,UAAU,YAAA,CAAsB;GAChC,cAAc,EAAE,iBACd,eAAe,IAAiB,EAAE,YAAY,OAAgB,IAAI,KAAA;GACpE,aAAa;GACb,iBAAiB,MAAM;GACvB,YAAY;GACZ,GAAI,MAAM,SAAS,EAAE,aAAa,MAAM,OAAO,IAAI,CAAC;EACtD,CAAC;EACD,MAAM,OAAO,OAAO,KAAK,KAAK;EAC9B,OAAO;GACL,GAAG;GACH,eAAe,OAAO,UAAU,WAAW;GAC3C,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;GAClC;GACA,GAAI,OAAO,CAAC,IAAI,EAAE,OAAO,qCAAiD;GAC1E,OAAO;IACL,aAAa,OAAO,YAAY,eAAe;IAC/C,cAAc,OAAO,YAAY,gBAAgB;GACnD;EACF;CACF,SAAS,OAAO;EACd,OAAO;GACL,GAAG;GACH,eAAe;GACf,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;GAClC,MAAM;GACN,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC5D,OAAO;IAAE,aAAa;IAAM,cAAc;GAAK;EACjD;CACF;AACF;;AASA,SAAS,YAAY,QAAmD;CACtE,MAAM,UAAmD,CAAC;CAC1D,KAAK,MAAM,WAAW,QAAQ;EAC5B,IAAI,QAAQ,SAAS,UAAU,CAAC,MAAM,QAAQ,QAAQ,OAAO,GAAG;EAChE,KAAK,MAAM,QAAQ,QAAQ,SACzB,IAAI,KAAK,SAAS,iBAAiB,KAAK,YAAY,KAAK,QACvD,QAAQ,KAAK;GAAE,MAAM,KAAK;GAAU,OAAO,KAAK,OAAO;EAAM,CAAC;CAGpE;CACA,OAAO;AACT;;;;;;AAOA,SAAS,cACP,MACA,OACe;CACf,MAAM,QAAQ;EACZ,aAAa;GAAE,OAAO;GAAG,SAAS;GAAG,WAAW;GAAG,YAAY;EAAE;EACjE,cAAc;GAAE,OAAO;GAAG,MAAM;GAAG,WAAW;EAAE;CAClD;CACA,IAAI,QAAQ;CACZ,MAAM,aAAa,WAAoD;EACrE,SAAS,MAAM,KAAK,GAAG,OAAO;GAC5B,MAAM;GACN,YAAY,QAAQ,EAAE,KAAK,GAAG;GAC9B,UAAU,EAAE;GACZ,OAAO,KAAK,UAAU,EAAE,KAAK;EAC/B,EAAE;EACF,cAAc;GAAE,SAAS;GAAc,KAAK;EAAa;EACzD;EACA,UAAU,CAAC;EACX,UAAU,EAAE,SAAS,gBAAgB;CACvC;CACA,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAmE;GACpF;GACA,MAAM,EAAE,SAAS;GACjB,IAAI,OAAO,SAAS,YAAY,SAAS,MAAM,MAAM,IAAI,MAAM,KAAK,KAAK;GACzE,MAAM,UAAU,YAAY,QAAQ,MAAM;GAC1C,MAAM,SAAS,QAAQ,MAAM,MAAM,EAAE,SAAS,eAAe,CAAC,EAAE;GAGhE,IAAI,CAAC,QAAQ,OAAO,UAAU,CAAC;IAAE,MAAM;IAAiB,OAAO,CAAC;GAAE,CAAC,CAAC;GACpE,MAAM,WAAW,QACd,QAAQ,MAAM,EAAE,SAAS,kBAAkB,CAAC,CAC5C,KAAK,MAAM,EAAE,KAAsC;GACtD,MAAM,QAAsB,OAAO,SAAS,CAAC;GAC7C,IAAI,MAAM,UAAU,CAAC,SAAS,UAAU,QAAA,GACtC,OAAO,UACL,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,KAAK,OAAO;IACrC,MAAM;IACN,OAAO;KAAE,QAAQ,EAAE;KAAQ,QAAQ,EAAE;IAAO;GAC9C,EAAE,CACJ;GAEF,MAAM,QAAuB;IAC3B,GAAG;IACH,WAAW,OAAO,YAAY,CAAC,EAAA,CAAG,KAAK,OAAO;KAAE,IAAI,EAAE;KAAI,SAAS,EAAE;IAAQ,EAAE;IAC/E,UAAU,SACP,QAAQ,MAAM,KAAK,YAAY,CAAC,CAAC,CACjC,KAAK,OAAO;KAAE,QAAQ,EAAE;KAAQ,QAAQ,EAAE;KAAQ,YAAY,EAAE;IAAW,EAAE;GAClF;GAMA,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ,MAL1B,OAAO,SAAS,aACZ,KAAK,KAAK,IACT,QACD,GAAG,MAAM,SAAS,QAAQ,MAAM,EAAE,YAAY,MAAM,CAAC,CAAC,OAAO,MAAM,MAAM,SAAS,OAAO;IAE9D,CAAC;IAChC,cAAc;KAAE,SAAS;KAAQ,KAAK;IAAO;IAC7C;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;;;ACrNA,MAAM,YAAY;AAElB,MAAM,YAAY,QAAgB,YAAoB,GAAG,OAAO,IAAI;AAEpE,SAAgB,WAAW,EAAE,QAAQ,OAAO,OAAO,YAAY,SAAS,QAA0B;CAChG,MAAM,2BAAW,IAAI,IAAoB;CACzC,MAAM,QAAQ,IAAI,IAAI,QAAQ,KAAK,GAAG,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;CACvD,KAAK,MAAM,KAAK,SAAS;EACvB,IAAI,EAAE,QAAQ,KAAK,MAAM;EACzB,MAAM,MAAM,SAAS,EAAE,IAAI,EAAE,QAAQ,KAAK,OAAO;EACjD,SAAS,IAAI,MAAM,SAAS,IAAI,GAAG,KAAK,KAAK,CAAC;CAChD;CACA,MAAM,2BAAW,IAAI,IAA0B;CAC/C,MAAM,0BAAU,IAAI,IAAY;CAChC,MAAM,UAA+C,CAAC;CACtD,MAAM,QAAQ,IAAI,gBAAgB;CAElC,IAAI,UAAU;CACd,MAAM,UAA6B,CAAC;CACpC,MAAM,OAAO,YAAY;EACvB,IAAI,UAAU,WAAW;OACpB,MAAM,IAAI,SAAe,YAAY,QAAQ,KAAK,OAAO,CAAC;CACjE;CACA,MAAM,gBAAgB;EACpB,MAAM,OAAO,QAAQ,MAAM;EAC3B,IAAI,MAAM,KAAK;OACV;CACP;CAEA,MAAM,SAAS,QAAgB;EAE7B,MAAM,WAAW,SAAS,IAAI,GAAG,KAAK,CAAC,EAAA,CAAG,MACvC,GAAG,OAAO,MAAM,IAAI,EAAE,GAAG,KAAK,MAAM,MAAM,IAAI,EAAE,GAAG,KAAK,EAC3D;EACA,QAAQ,IAAI,GAAG;EACf,MAAM,QAAQ,QAAQ;EAEtB,IAAI,CAAC,OAAO,WAAW,QAAQ,OAAO,MAAM,EAAE,UAAU,SAAS,GAAG;EACpE,MAAM,UAAU,MAAM,QAAQ,KAAK;EACnC,MAAM,QAAQ,oBAAoB,MAAM,GAAG,IAAI,QAAQ;EACvD,QAAQ,MACL,YAAY;GACX,MAAM,KAAK;GACX,IAAI;IACF,IAAI,MAAM,OAAO,SAAS,OAAO;IACjC,MAAM,OAAO,MAAM,eAAe;KAChC;KACA;KACA,QAAQ,MAAM;KACd;KACA,UAAU,MAAM,eAAe,YAAY,OAAO;KAClD,gBAAgB,OAAO,SAAS;KAChC,iBAAiB,OAAO,SAAS;KACjC,GAAI,OAAO,SAAS,eAAe,EAAE,cAAc,OAAO,SAAS,aAAa,IAAI,CAAC;KACrF,QAAQ,MAAM;IAChB,CAAC;IACD,IAAI,MAAM,OAAO,SAAS,OAAO;IACjC,IAAI,KAAK,OAAO,KAAK,GAAG,MAAM,WAAW,KAAK,OAAO;IACrD,MAAM,UAAU,OAAO,MAAM,IAAI;IACjC,OAAO;GACT,SAAS,OAAO;IACd,IAAI,CAAC,MAAM,OAAO,SAAS,KAAK,GAAG,MAAM,mBAAoB,MAAgB,SAAS;IACtF,OAAO;GACT,UAAU;IACR,QAAQ;GACV;EACF,EAAA,CAAG,CACL;CACF;CAEA,OAAO;EACL,SAAS,QAAQ;GACf,IAAI,CAAC,OAAO,WAAW,MAAM,OAAO,SAAS;GAC7C,MAAM,MAAM,SAAS,OAAO,IAAI,OAAO,QAAQ,KAAK,OAAO;GAC3D,IAAI,CAAC,SAAS,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,GAAG;GAC5C,MAAM,UAAU,CAAC,GAAI,SAAS,IAAI,GAAG,KAAK,CAAC,GAAI,MAAM;GACrD,SAAS,IAAI,KAAK,OAAO;GACzB,IAAI,QAAQ,WAAW,SAAS,IAAI,GAAG,KAAK,IAAI,MAAM,GAAG;EAC3D;EACA,MAAM,OAAO,WAAW;GACtB,IAAI,WAAW,MAAM,sBAAM,IAAI,MAAM,uBAAuB,CAAC;QACxD,KAAK,MAAM,OAAO,SAAS,KAAK,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG,MAAM,GAAG;GAExE,QADe,MAAM,QAAQ,IAAI,OAAO,EAAA,CAAG,QAAQ,MAAyB,MAAM,IACvE;EACb;CACF;AACF;;;;;AAMA,eAAsB,aACpB,QACA,OACA,OACmD;CACnD,IAAI;EACF,MAAM,EAAE,oBAAoB,MAAM,OAAO;EACzC,MAAM,EAAE,KAAK,SAAS,MAAM,gBAAgB;GAAE;GAAQ;GAAO;EAAM,CAAC;EACpE,IAAI,IAAI,OAAO,OAAO;GAAE,OAAO;GAAQ,MAAM,4BAA4B,IAAI;EAAQ;EACrF,OAAO;GACL,OAAO;GACP,MAAM,wBAAwB,QAAQ,IAAI,EAAE;EAC9C;CACF,SAAS,OAAO;EACd,OAAO;GAAE,OAAO;GAAQ,MAAM,oCAAqC,MAAgB;EAAU;CAC/F;AACF;;AAGA,MAAa,aAAa,MAAc,OAAe,UAA0B;CAC/E,MAAM,SAAS,MAAM,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC;CAC5C,MAAM,IAAI,MAAM;CAChB,OAAO,aAAa,EAAE,OAAO,MAAM,IAAI,KAAK,MAAM,SAAS,KAAK,OAAO,YAAY,GAAG,MAAM,QAAQ,SAAS,MAAM,KAAK,CAAC;AAC3H"}

package/dist/{agent-DVHkx8aT.js → agent-DcQAvomp.js}

@@ -1,9 +1,9 @@
 import { t as InfraError } from "./types-C5vI3g1C.js";
-import { d as agentFor, f as authHint, u as agentAuth } from "./egress-73SQfKdx.js";
-import { B as WORKSPACE_DIR, R as SANDBOX_ROOT } from "./src-DwwiVLqv.js";
+import { d as agentFor, f as authHint, u as agentAuth } from "./egress-D9vJCATi.js";
+import { B as WORKSPACE_DIR, R as SANDBOX_ROOT } from "./src-D3eW1e7Y.js";
 import { i as debugTrace, r as debugLog } from "./attempt-BdRQ9jKX.js";
 import { t as addUsage } from "./usage-DsYUqIUM.js";
-import { n as isDockerSandbox } from "./docker-oSbCExHL.js";
+import { n as isDockerSandbox } from "./docker-DwG9f76x.js";
 import { HarnessAgent } from "@ai-sdk/harness/agent";
 //#region src/harness/transcript.ts
 const str = (value) => typeof value === "string" ? value : "";
@@ -529,4 +529,4 @@ function createAgentHarness(descriptor) {
 //#endregion
 export { createAgentHarness };
 
-//# sourceMappingURL=agent-DVHkx8aT.js.map
+//# sourceMappingURL=agent-DcQAvomp.js.map