teapot-coding-agent 0.37.25 → 0.37.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent.js +142 -44
- package/dist/agent/goal-audit.js +86 -0
- package/dist/agent/tools.js +1 -1
- package/package.json +1 -1
- package/public/assets/{index-foPTo6nL.js → index-C3BmKnaM.js} +21 -21
- package/public/assets/index-C3BmKnaM.js.map +1 -0
- package/public/assets/{index-Br0Jspyp.css → index-DgRMTuvC.css} +1 -1
- package/public/assets/{shiki--qmx_kCE.js → shiki-C2QzQnp8.js} +2 -2
- package/public/assets/{shiki--qmx_kCE.js.map → shiki-C2QzQnp8.js.map} +1 -1
- package/public/index.html +2 -2
- package/public/assets/index-foPTo6nL.js.map +0 -1
package/dist/agent/agent.js
CHANGED
|
@@ -10,6 +10,7 @@ import { promises as fs, existsSync, readFileSync, readdirSync, statSync } from
|
|
|
10
10
|
import path from "node:path";
|
|
11
11
|
import { EventLog, readEvents } from "../log/events.js";
|
|
12
12
|
import { chat, chatStream } from "./llm.js";
|
|
13
|
+
import { auditPrompt, parseAuditVerdict } from "./goal-audit.js";
|
|
13
14
|
import { executeTool, toolSpecs, currentSkills, safeJoin, isContextOverflow, isRequestNodeLimit, hasRunningBgShells, hasRunningBgShellsFor, killBgShells } from "./tools.js";
|
|
14
15
|
import { effortChoosable } from "./llm.js";
|
|
15
16
|
import { sessionTitleFrom } from "../session-title.js";
|
|
@@ -229,6 +230,11 @@ const UPDATE_INSTRUCTION = [
|
|
|
229
230
|
/** framing around the prior notes when an update pass carries them forward */
|
|
230
231
|
const SUMMARY_OPEN = "<conversation_checkpoint>\n";
|
|
231
232
|
const SUMMARY_CLOSE = "\n</conversation_checkpoint>";
|
|
233
|
+
/**
|
|
234
|
+
* What the completion auditor may use. Read-only by construction — it inspects artifacts to check a
|
|
235
|
+
* claim, and must not be able to change what it is judging.
|
|
236
|
+
*/
|
|
237
|
+
const AUDIT_READ_ONLY_TOOLS = new Set(["read_file", "list_dir", "grep", "bash"]);
|
|
232
238
|
const SYSTEM_TEMPLATE = `You are a coding agent working autonomously inside a workspace.
|
|
233
239
|
|
|
234
240
|
Most session state is not injected into prompts — fetch it with tools instead:
|
|
@@ -3462,6 +3468,30 @@ export class Agent {
|
|
|
3462
3468
|
* it — the retry prompt should carry the same context an approved one
|
|
3463
3469
|
* would — while emitting no `final: true`.
|
|
3464
3470
|
*/
|
|
3471
|
+
/**
|
|
3472
|
+
* A COMPACT account of what the worker actually did, for the auditor.
|
|
3473
|
+
*
|
|
3474
|
+
* Not the conversation: see the note at the audit call. This is the last few assistant turns plus
|
|
3475
|
+
* the tools they ran, which is what a reviewer needs to see whether the claimed work happened —
|
|
3476
|
+
* and it stays small enough that the auditor can still answer the question it was asked.
|
|
3477
|
+
*/
|
|
3478
|
+
auditEvidence(summary) {
|
|
3479
|
+
const lines = [];
|
|
3480
|
+
for (let i = this.messages.length - 1; i >= 0 && lines.length < 24; i--) {
|
|
3481
|
+
const m = this.messages[i];
|
|
3482
|
+
const text = String(m.content ?? "").trim();
|
|
3483
|
+
if (!text)
|
|
3484
|
+
continue;
|
|
3485
|
+
if (m.role === "user")
|
|
3486
|
+
break; // stop at the request that started this round
|
|
3487
|
+
const who = m.role === "assistant" ? "worker" : m.role;
|
|
3488
|
+
lines.push(`${who}: ${text.replace(/\s+/g, " ").slice(0, 400)}`);
|
|
3489
|
+
}
|
|
3490
|
+
const tail = String(summary ?? "").trim();
|
|
3491
|
+
if (tail && !lines.some((l) => l.includes(tail.slice(0, 60))))
|
|
3492
|
+
lines.unshift(`finish summary: ${tail}`);
|
|
3493
|
+
return lines.reverse().join("\n").slice(0, 6000);
|
|
3494
|
+
}
|
|
3465
3495
|
async captureRecentTurns() {
|
|
3466
3496
|
const recent = [];
|
|
3467
3497
|
for (let i = this.messages.length - 1; i >= 0 && recent.length < 6; i--) {
|
|
@@ -3504,6 +3534,67 @@ export class Agent {
|
|
|
3504
3534
|
}
|
|
3505
3535
|
return recent;
|
|
3506
3536
|
}
|
|
3537
|
+
/**
|
|
3538
|
+
* Run the completion auditor, streaming its steps into the timeline.
|
|
3539
|
+
*
|
|
3540
|
+
* The auditor is a real review, not a single question: pi-goal-x gives it read-only tools so it
|
|
3541
|
+
* can inspect the artifacts instead of trusting the worker's summary, and it reports its progress
|
|
3542
|
+
* as it goes. Both matter here:
|
|
3543
|
+
*
|
|
3544
|
+
* - WITHOUT the tools the auditor could only re-read the claim it was asked to check, which is
|
|
3545
|
+
* how three audits in github-bug-research-1f48715f replied in prose and produced no verdict;
|
|
3546
|
+
* - WITHOUT the progress rows the timeline showed `audit-started` and then nothing until the
|
|
3547
|
+
* verdict, so an audit that produced no verdict was indistinguishable from one that never ran.
|
|
3548
|
+
*
|
|
3549
|
+
* Bounded by `AUDIT_MAX_STEPS`: an auditor that keeps reading is doing work the operator cannot
|
|
3550
|
+
* see the end of, and the worker is blocked while it runs.
|
|
3551
|
+
*/
|
|
3552
|
+
async runAuditor(first) {
|
|
3553
|
+
const AUDIT_MAX_STEPS = 12;
|
|
3554
|
+
const step = (data) => this.log.append("audit_step", this.currentSession, this.currentBranch, data).catch(() => { });
|
|
3555
|
+
const auditTools = toolSpecs().filter((t) => AUDIT_READ_ONLY_TOOLS.has(t.function.name));
|
|
3556
|
+
const messages = [...first];
|
|
3557
|
+
await step({ step: "started", tools: auditTools.map((t) => t.function.name) });
|
|
3558
|
+
for (let i = 0; i < AUDIT_MAX_STEPS; i++) {
|
|
3559
|
+
const res = await this.callLlm(messages, auditTools);
|
|
3560
|
+
const text = String(res.message.content ?? "");
|
|
3561
|
+
const calls = res.message.tool_calls ?? [];
|
|
3562
|
+
if (!calls.length) {
|
|
3563
|
+
if (text.trim())
|
|
3564
|
+
await step({ step: "reply", text: text.slice(0, 400) });
|
|
3565
|
+
return text;
|
|
3566
|
+
}
|
|
3567
|
+
messages.push({ role: "assistant", content: text, tool_calls: calls });
|
|
3568
|
+
for (const call of calls) {
|
|
3569
|
+
await step({ step: "tool", name: call.function?.name, args: String(call.function?.arguments ?? "").slice(0, 200) });
|
|
3570
|
+
let out;
|
|
3571
|
+
try {
|
|
3572
|
+
out = await executeTool(String(call.function?.name ?? ""), String(call.function?.arguments ?? "{}"), {
|
|
3573
|
+
...this.toolCtx,
|
|
3574
|
+
readOnly: true,
|
|
3575
|
+
allowMutatingForReview: ["bash"],
|
|
3576
|
+
});
|
|
3577
|
+
}
|
|
3578
|
+
catch (err) {
|
|
3579
|
+
out = { ok: false, result: `tool failed: ${String(err.message).slice(0, 200)}` };
|
|
3580
|
+
}
|
|
3581
|
+
await step({
|
|
3582
|
+
step: "tool_done",
|
|
3583
|
+
name: call.function?.name,
|
|
3584
|
+
ok: out.ok,
|
|
3585
|
+
chars: out.result.length,
|
|
3586
|
+
});
|
|
3587
|
+
messages.push({
|
|
3588
|
+
role: "tool",
|
|
3589
|
+
tool_call_id: call.id,
|
|
3590
|
+
content: out.result.slice(0, 20_000),
|
|
3591
|
+
});
|
|
3592
|
+
}
|
|
3593
|
+
}
|
|
3594
|
+
// out of steps: ask once for the verdict alone, with no tools to reach for
|
|
3595
|
+
const final = await this.callLlm([...messages, { role: "user", content: "Stop investigating. Reply with the verdict line only." }], []);
|
|
3596
|
+
return String(final.message.content ?? "");
|
|
3597
|
+
}
|
|
3507
3598
|
async handleFinish(argsJson) {
|
|
3508
3599
|
const args = safeParse(argsJson);
|
|
3509
3600
|
if (args.goalComplete === true && this.goal.status !== "done") {
|
|
@@ -3519,39 +3610,30 @@ export class Agent {
|
|
|
3519
3610
|
// number the attempts — the final carries the attempt that approved it
|
|
3520
3611
|
this.auditAttempts++;
|
|
3521
3612
|
try {
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
// loop". The same situation reached by another route gets the same
|
|
3547
|
-
// answer.
|
|
3548
|
-
const verdictLine = text.trim();
|
|
3549
|
-
// provider fallbacks for a message with no usable text; treating one
|
|
3550
|
-
// of these as a "reason" is worse than treating it as nothing
|
|
3551
|
-
const PLACEHOLDER = /^\((tool call|no content|no output)\)$/i;
|
|
3552
|
-
const usable = verdictLine.length > 0 && !PLACEHOLDER.test(verdictLine);
|
|
3553
|
-
const decided = usable && /^(APPROVED|CHANGES-REQUIRED)\b/i.test(verdictLine);
|
|
3554
|
-
if (!decided) {
|
|
3613
|
+
/**
|
|
3614
|
+
* The auditor runs in its OWN conversation.
|
|
3615
|
+
*
|
|
3616
|
+
* It used to be `buildMessages()` + one more user turn — the worker's entire conversation,
|
|
3617
|
+
* handed to a model that was then told "you are an auditor, not the worker". MEASURED in
|
|
3618
|
+
* github-bug-research-1f48715f: with ~220k tokens of work log in front of it, the auditor
|
|
3619
|
+
* replied by CONTINUING the work (prose, no verdict) on all three attempts, and the goal
|
|
3620
|
+
* was abandoned with the audit never actually running.
|
|
3621
|
+
*
|
|
3622
|
+
* A review is not a continuation. The auditor gets the objective, the contract and the
|
|
3623
|
+
* worker's claim — everything it needs to judge — and nothing to continue.
|
|
3624
|
+
*/
|
|
3625
|
+
const evidence = this.auditEvidence(args.summary);
|
|
3626
|
+
const auditMessages = [
|
|
3627
|
+
{ role: "user", content: auditPrompt({ goal: this.goal.text, verify: this.goal.verify, summary: String(args.summary ?? ""), evidence }) },
|
|
3628
|
+
];
|
|
3629
|
+
const text = await this.runAuditor(auditMessages);
|
|
3630
|
+
/**
|
|
3631
|
+
* The verdict is the reply's LAST line — see `parseAuditVerdict`. Anything else is NO
|
|
3632
|
+
* VERDICT, never a rejection: a reason-less "fix these gaps" carries no information and
|
|
3633
|
+
* only teaches everyone that the audit is noise.
|
|
3634
|
+
*/
|
|
3635
|
+
const verdict = parseAuditVerdict(text);
|
|
3636
|
+
if (verdict.kind === "none") {
|
|
3555
3637
|
// an unusable reply is NOT an approval. The old path failed
|
|
3556
3638
|
// OPEN — recorded the failure and marked the goal done anyway —
|
|
3557
3639
|
// so a verification contract was "satisfied" that nobody verified.
|
|
@@ -3560,11 +3642,22 @@ export class Agent {
|
|
|
3560
3642
|
this.auditUnavailableStreak++;
|
|
3561
3643
|
await this.log.append("system_note", this.currentSession, this.currentBranch, {
|
|
3562
3644
|
event: "audit-failed",
|
|
3563
|
-
detail:
|
|
3564
|
-
? `auditor reply had no verdict: ${verdictLine.slice(0, 200)}`
|
|
3565
|
-
: "auditor returned no verdict",
|
|
3645
|
+
detail: verdict.detail,
|
|
3566
3646
|
attempt: this.auditUnavailableStreak,
|
|
3567
3647
|
});
|
|
3648
|
+
/**
|
|
3649
|
+
* A no-verdict reply gets a bounded number of retries, then the operator decides.
|
|
3650
|
+
*
|
|
3651
|
+
* MEASURED in github-bug-research-1f48715f: three audits in six minutes all failed the
|
|
3652
|
+
* same way, and the retry prompt told the worker to "re-check the contract yourself, then
|
|
3653
|
+
* finish again" — which cannot help, because the worker's re-verification was never what
|
|
3654
|
+
* the auditor was missing. It re-ran the SAME call against the SAME model with the SAME
|
|
3655
|
+
* reply shape, so the loop cost three rounds and ended at the cap anyway.
|
|
3656
|
+
*
|
|
3657
|
+
* The retry is worth keeping for the transient case (a provider hiccup, a truncated
|
|
3658
|
+
* reply), and the prompt now asks for the thing that actually helps — a concise statement
|
|
3659
|
+
* of what was verified. Three attempts, then a loud note.
|
|
3660
|
+
*/
|
|
3568
3661
|
if (this.auditUnavailableStreak < 3) {
|
|
3569
3662
|
this.queueAuditUnavailableRetry();
|
|
3570
3663
|
await this.captureRecentTurns();
|
|
@@ -3581,8 +3674,9 @@ export class Agent {
|
|
|
3581
3674
|
await this.captureRecentTurns();
|
|
3582
3675
|
return;
|
|
3583
3676
|
}
|
|
3584
|
-
const approved =
|
|
3585
|
-
|
|
3677
|
+
const approved = verdict.kind === "approved";
|
|
3678
|
+
// the parser already separated marker from reason
|
|
3679
|
+
const feedback = verdict.reason;
|
|
3586
3680
|
// storing the literal "(no detail)" made the UI draw a card
|
|
3587
3681
|
// whose whole content was the absence of a reason, and that same
|
|
3588
3682
|
// string landed in goal.md for the right panel. An empty string is
|
|
@@ -3707,14 +3801,18 @@ export class Agent {
|
|
|
3707
3801
|
await this.appendFinalMessage(args);
|
|
3708
3802
|
}
|
|
3709
3803
|
/**
|
|
3710
|
-
*
|
|
3711
|
-
*
|
|
3712
|
-
*
|
|
3804
|
+
* The retry prompt for an audit that could not produce a verdict.
|
|
3805
|
+
*
|
|
3806
|
+
* It asks the worker to make the NEXT audit easier to answer, which is the only thing within its
|
|
3807
|
+
* power: a no-verdict reply means the auditor could not judge from what it was given, so a
|
|
3808
|
+
* concise statement of what was verified is the actual fix. It does NOT claim the work is
|
|
3809
|
+
* unverified-and-must-be-redone; the work may be fine and the auditor simply failed, and telling
|
|
3810
|
+
* the worker to redo it produced the repeat loop measured in github-bug-research-1f48715f.
|
|
3713
3811
|
*/
|
|
3714
3812
|
queueAuditUnavailableRetry() {
|
|
3715
|
-
const text = `[harness] The completion audit could not
|
|
3716
|
-
`
|
|
3717
|
-
`
|
|
3813
|
+
const text = `[harness] The completion audit could not produce a verdict — that is a problem with the ` +
|
|
3814
|
+
`AUDIT, not a finding about your work. Before finishing again, state plainly and briefly what ` +
|
|
3815
|
+
`you verified for each item of the contract, so the auditor has something to judge.\n\n` +
|
|
3718
3816
|
`${this.goal.verify?.slice(0, 2000) || this.goal.text.slice(0, 2000)}`;
|
|
3719
3817
|
this.pendingPrompts.push({ source: "harness", text });
|
|
3720
3818
|
// log the row: a push-only prompt is invisible on the timeline and lost to
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Completion-audit verdict parsing and the auditor prompt.
|
|
3
|
+
*
|
|
4
|
+
* The auditor is an INDEPENDENT reviewer: a separate LLM call that decides whether the worker's
|
|
5
|
+
* `finish(goalComplete=true)` is honest against the goal's verification contract. Its answer must
|
|
6
|
+
* be unambiguous, because the two readings are not symmetric — a missed approval costs one more
|
|
7
|
+
* round, while a misread rejection sends the worker to fix gaps that nobody named.
|
|
8
|
+
*/
|
|
9
|
+
/** provider fallbacks for a message with no usable text */
|
|
10
|
+
const PLACEHOLDER = /^\((tool call|no content|no output)\)$/i;
|
|
11
|
+
/**
|
|
12
|
+
* Read the auditor's verdict.
|
|
13
|
+
*
|
|
14
|
+
* The verdict must be the LAST non-empty line, and it must carry a reason. Upstream pi-goal-x made
|
|
15
|
+
* this exact correction: matching the marker ANYWHERE lets a prose mention of it be read as the
|
|
16
|
+
* verdict — "I would only approve if the tests passed" is a REJECTION that contains the word.
|
|
17
|
+
*
|
|
18
|
+
* MEASURED against the real failure: an auditor that answered `Independent verification against …`
|
|
19
|
+
* (prose, no marker) was scored as a verdict three times in a row, hit the unavailable cap, and the
|
|
20
|
+
* goal was abandoned with the audit never actually running. Requiring the marker on the last line
|
|
21
|
+
* turns that into an honest "no verdict", which the caller already handles by retrying or failing
|
|
22
|
+
* closed with a loud note.
|
|
23
|
+
*/
|
|
24
|
+
export function parseAuditVerdict(reply) {
|
|
25
|
+
const text = String(reply ?? "").trim();
|
|
26
|
+
if (!text || PLACEHOLDER.test(text))
|
|
27
|
+
return { kind: "none", detail: "auditor returned no text" };
|
|
28
|
+
const lines = text
|
|
29
|
+
.split("\n")
|
|
30
|
+
.map((l) => l.trim())
|
|
31
|
+
.filter(Boolean);
|
|
32
|
+
// the verdict line is the LAST one; a reason may wrap onto earlier lines
|
|
33
|
+
const last = lines[lines.length - 1] ?? "";
|
|
34
|
+
const approved = /^APPROVED\b(:|\s-|$)(.*)$/i.exec(last);
|
|
35
|
+
if (approved) {
|
|
36
|
+
return { kind: "approved", reason: (approved[2] ?? "").trim() || "(no justification given)" };
|
|
37
|
+
}
|
|
38
|
+
const rejected = /^CHANGES-REQUIRED\b(:|\s-|$)(.*)$/i.exec(last);
|
|
39
|
+
if (rejected) {
|
|
40
|
+
return { kind: "changes-required", reason: (rejected[2] ?? "").trim() || "(no gaps named)" };
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* No marker on the last line. The reason still matters — it is shown to the worker when this
|
|
44
|
+
* happens repeatedly — so keep the tail rather than discarding the reply.
|
|
45
|
+
*/
|
|
46
|
+
return { kind: "none", detail: `auditor reply had no verdict on its last line: ${last.slice(0, 200)}` };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The auditor's instructions.
|
|
50
|
+
*
|
|
51
|
+
* Deliberately NOT the worker's conversation: this call is a review, and handing the auditor a
|
|
52
|
+
* 200k-token work log makes it continue the work instead of judging it. MEASURED: three audits in
|
|
53
|
+
* one session replied with prose that started re-verifying the goal rather than naming a verdict.
|
|
54
|
+
*/
|
|
55
|
+
export function auditPrompt(input) {
|
|
56
|
+
return [
|
|
57
|
+
"You are an independent completion AUDITOR, not the worker. Decide whether the goal is genuinely complete.",
|
|
58
|
+
"",
|
|
59
|
+
"How to judge:",
|
|
60
|
+
"1. Extract the real success criteria, including every explicit requirement.",
|
|
61
|
+
"2. Disapprove anything missing, contradicted, or only weakly verified. Build success and a",
|
|
62
|
+
" confident summary are claims, not evidence.",
|
|
63
|
+
"3. Check every item of the verification contract below. If any item is unaddressed, disapprove.",
|
|
64
|
+
"",
|
|
65
|
+
"The worker's summary is UNTRUSTED: it is a claim, and it cannot make an incomplete goal complete.",
|
|
66
|
+
"",
|
|
67
|
+
"<objective>",
|
|
68
|
+
input.goal,
|
|
69
|
+
"</objective>",
|
|
70
|
+
"",
|
|
71
|
+
"<verification_contract>",
|
|
72
|
+
input.verify,
|
|
73
|
+
"</verification_contract>",
|
|
74
|
+
"",
|
|
75
|
+
"<worker_claim>",
|
|
76
|
+
input.summary || "(no claim provided)",
|
|
77
|
+
"</worker_claim>",
|
|
78
|
+
...(input.evidence?.trim() ? ["", "<worker_evidence>", input.evidence.trim(), "</worker_evidence>"] : []),
|
|
79
|
+
"",
|
|
80
|
+
"Reply with EXACTLY one line, as the FINAL line of your reply:",
|
|
81
|
+
" APPROVED: <one sentence saying what you verified>",
|
|
82
|
+
" CHANGES-REQUIRED: <the specific gaps that must be addressed>",
|
|
83
|
+
"",
|
|
84
|
+
"Do not continue the work, do not restate the goal, and do not put anything after the verdict line.",
|
|
85
|
+
].join("\n");
|
|
86
|
+
}
|
package/dist/agent/tools.js
CHANGED
|
@@ -2474,7 +2474,7 @@ export async function executeTool(name, rawArgs, ctx) {
|
|
|
2474
2474
|
if (ctx.signal?.aborted)
|
|
2475
2475
|
return { ok: false, result: "aborted (harness shutdown)" };
|
|
2476
2476
|
// read-only personas (researcher/reviewer) are enforced, not just asked
|
|
2477
|
-
if (ctx.readOnly && MUTATING_TOOLS.has(name))
|
|
2477
|
+
if (ctx.readOnly && MUTATING_TOOLS.has(name) && !ctx.allowMutatingForReview?.includes(name))
|
|
2478
2478
|
return { ok: false, result: `${name} is blocked: this agent runs with read-only tools` };
|
|
2479
2479
|
// repair the call before handing it to the tool. A model that sends
|
|
2480
2480
|
// {"cmd":"ls"} or {"limit":"100"} should get the work done, not a generic parse
|