teapot-coding-agent 0.37.24 → 0.37.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,7 @@ import { promises as fs, existsSync, readFileSync, readdirSync, statSync } from
10
10
  import path from "node:path";
11
11
  import { EventLog, readEvents } from "../log/events.js";
12
12
  import { chat, chatStream } from "./llm.js";
13
+ import { auditPrompt, parseAuditVerdict } from "./goal-audit.js";
13
14
  import { executeTool, toolSpecs, currentSkills, safeJoin, isContextOverflow, isRequestNodeLimit, hasRunningBgShells, hasRunningBgShellsFor, killBgShells } from "./tools.js";
14
15
  import { effortChoosable } from "./llm.js";
15
16
  import { sessionTitleFrom } from "../session-title.js";
@@ -229,6 +230,11 @@ const UPDATE_INSTRUCTION = [
229
230
  /** framing around the prior notes when an update pass carries them forward */
230
231
  const SUMMARY_OPEN = "<conversation_checkpoint>\n";
231
232
  const SUMMARY_CLOSE = "\n</conversation_checkpoint>";
233
+ /**
234
+ * What the completion auditor may use. Read-only by construction — it inspects artifacts to check a
235
+ * claim, and must not be able to change what it is judging.
236
+ */
237
+ const AUDIT_READ_ONLY_TOOLS = new Set(["read_file", "list_dir", "grep", "bash"]);
232
238
  const SYSTEM_TEMPLATE = `You are a coding agent working autonomously inside a workspace.
233
239
 
234
240
  Most session state is not injected into prompts — fetch it with tools instead:
@@ -270,6 +276,17 @@ Most session state is not injected into prompts — fetch it with tools instead:
270
276
  valid string), and it took longer to repair than doing the edits one by one.
271
277
  A genuine bulk transform across many files (a rename, an import change) IS a
272
278
  job for a script — then keep the pattern specific and read the diff.
279
+ - **NEVER \`sleep\` to wait for anything.** Not alone, not as "a quick poll".
280
+ A bash call with a sleep inside is one opaque command: the operator cannot
281
+ interrupt it, cannot see whether anything is happening, and the whole turn is
282
+ spent doing nothing. Two interruptible shapes instead:
283
+ · long single-purpose work (CI, a release, a tarball appearing, a dev
284
+ server, a watcher) → bash(background=true). It returns a job id at once
285
+ and you are NOTIFIED when it exits, so keep working and let it report.
286
+ · waiting on a condition you are actively watching → poll with a bounded
287
+ deadline in ONE call, then stop.
288
+ A fixed wait also cannot tell "finished 5 minutes ago" from "still running",
289
+ which is how a turn gets wasted twice.
273
290
  - When a loaded skill matches your task, follow its playbook.
274
291
  - Turn proven procedures into skills: once something non-trivial worked well,
275
292
  save_skill(name, description, content, files=[{name, content}]) so future
@@ -3451,6 +3468,30 @@ export class Agent {
3451
3468
  * it — the retry prompt should carry the same context an approved one
3452
3469
  * would — while emitting no `final: true`.
3453
3470
  */
3471
+ /**
3472
+ * A COMPACT account of what the worker actually did, for the auditor.
3473
+ *
3474
+ * Not the conversation: see the note at the audit call. This is the last few assistant turns plus
3475
+ * the tools they ran, which is what a reviewer needs to see whether the claimed work happened —
3476
+ * and it stays small enough that the auditor can still answer the question it was asked.
3477
+ */
3478
+ auditEvidence(summary) {
3479
+ const lines = [];
3480
+ for (let i = this.messages.length - 1; i >= 0 && lines.length < 24; i--) {
3481
+ const m = this.messages[i];
3482
+ const text = String(m.content ?? "").trim();
3483
+ if (!text)
3484
+ continue;
3485
+ if (m.role === "user")
3486
+ break; // stop at the request that started this round
3487
+ const who = m.role === "assistant" ? "worker" : m.role;
3488
+ lines.push(`${who}: ${text.replace(/\s+/g, " ").slice(0, 400)}`);
3489
+ }
3490
+ const tail = String(summary ?? "").trim();
3491
+ if (tail && !lines.some((l) => l.includes(tail.slice(0, 60))))
3492
+ lines.unshift(`finish summary: ${tail}`);
3493
+ return lines.reverse().join("\n").slice(0, 6000);
3494
+ }
3454
3495
  async captureRecentTurns() {
3455
3496
  const recent = [];
3456
3497
  for (let i = this.messages.length - 1; i >= 0 && recent.length < 6; i--) {
@@ -3493,6 +3534,67 @@ export class Agent {
3493
3534
  }
3494
3535
  return recent;
3495
3536
  }
3537
+ /**
3538
+ * Run the completion auditor, streaming its steps into the timeline.
3539
+ *
3540
+ * The auditor is a real review, not a single question: pi-goal-x gives it read-only tools so it
3541
+ * can inspect the artifacts instead of trusting the worker's summary, and it reports its progress
3542
+ * as it goes. Both matter here:
3543
+ *
3544
+ * - WITHOUT the tools the auditor could only re-read the claim it was asked to check, which is
3545
+ * how three audits in github-bug-research-1f48715f replied in prose and produced no verdict;
3546
+ * - WITHOUT the progress rows the timeline showed `audit-started` and then nothing until the
3547
+ * verdict, so an audit that produced no verdict was indistinguishable from one that never ran.
3548
+ *
3549
+ * Bounded by `AUDIT_MAX_STEPS`: an auditor that keeps reading is doing work the operator cannot
3550
+ * see the end of, and the worker is blocked while it runs.
3551
+ */
3552
+ async runAuditor(first) {
3553
+ const AUDIT_MAX_STEPS = 12;
3554
+ const step = (data) => this.log.append("audit_step", this.currentSession, this.currentBranch, data).catch(() => { });
3555
+ const auditTools = toolSpecs().filter((t) => AUDIT_READ_ONLY_TOOLS.has(t.function.name));
3556
+ const messages = [...first];
3557
+ await step({ step: "started", tools: auditTools.map((t) => t.function.name) });
3558
+ for (let i = 0; i < AUDIT_MAX_STEPS; i++) {
3559
+ const res = await this.callLlm(messages, auditTools);
3560
+ const text = String(res.message.content ?? "");
3561
+ const calls = res.message.tool_calls ?? [];
3562
+ if (!calls.length) {
3563
+ if (text.trim())
3564
+ await step({ step: "reply", text: text.slice(0, 400) });
3565
+ return text;
3566
+ }
3567
+ messages.push({ role: "assistant", content: text, tool_calls: calls });
3568
+ for (const call of calls) {
3569
+ await step({ step: "tool", name: call.function?.name, args: String(call.function?.arguments ?? "").slice(0, 200) });
3570
+ let out;
3571
+ try {
3572
+ out = await executeTool(String(call.function?.name ?? ""), String(call.function?.arguments ?? "{}"), {
3573
+ ...this.toolCtx,
3574
+ readOnly: true,
3575
+ allowMutatingForReview: ["bash"],
3576
+ });
3577
+ }
3578
+ catch (err) {
3579
+ out = { ok: false, result: `tool failed: ${String(err.message).slice(0, 200)}` };
3580
+ }
3581
+ await step({
3582
+ step: "tool_done",
3583
+ name: call.function?.name,
3584
+ ok: out.ok,
3585
+ chars: out.result.length,
3586
+ });
3587
+ messages.push({
3588
+ role: "tool",
3589
+ tool_call_id: call.id,
3590
+ content: out.result.slice(0, 20_000),
3591
+ });
3592
+ }
3593
+ }
3594
+ // out of steps: ask once for the verdict alone, with no tools to reach for
3595
+ const final = await this.callLlm([...messages, { role: "user", content: "Stop investigating. Reply with the verdict line only." }], []);
3596
+ return String(final.message.content ?? "");
3597
+ }
3496
3598
  async handleFinish(argsJson) {
3497
3599
  const args = safeParse(argsJson);
3498
3600
  if (args.goalComplete === true && this.goal.status !== "done") {
@@ -3508,39 +3610,30 @@ export class Agent {
3508
3610
  // number the attempts — the final carries the attempt that approved it
3509
3611
  this.auditAttempts++;
3510
3612
  try {
3511
- const auditPrompt = `You are an independent completion AUDITOR (not the worker). Decide whether the goal is genuinely complete.\n\n` +
3512
- `GOAL:\n${this.goal.text.slice(0, 4000)}\n\nVERIFICATION CONTRACT (must ALL hold):\n${this.goal.verify.slice(0, 2000)}\n\n` +
3513
- `WORKER'S FINAL SUMMARY:\n${String(args.summary ?? "").slice(0, 3000)}\n\n` +
3514
- `Conversation evidence follows. Reply with EXACTLY one line starting either\n` +
3515
- `"APPROVED: <one-sentence justification>" or\n` +
3516
- `"CHANGES-REQUIRED: <the specific gaps that must be addressed>".`;
3517
- const res = await this.callLlm([
3518
- ...this.buildMessages(),
3519
- { role: "user", content: auditPrompt },
3520
- ], []);
3521
- const text = String(res.message.content ?? "");
3522
- // a reply that is NOT a verdict must not be read as a rejection.
3523
- //
3524
- // `/^APPROVED\b/i.test("")` is false, so an auditor that returned
3525
- // nothing — truncated, refused, or leaked provider tool-call markup —
3526
- // was scored `changes-required` with no reason. Seven times in one
3527
- // session: the worker was told to "address these gaps" while being
3528
- // given none, and the fallback text was the contract it had already
3529
- // read. A reason-less rejection carries no information and only
3530
- // teaches everyone that the audit is noise.
3531
- //
3532
- // So an unusable reply is treated as NO VERDICT, exactly like an
3533
- // auditor that threw — the `catch` below already fails open with the
3534
- // reasoning "a flaky provider can't trap work in an unauditable
3535
- // loop". The same situation reached by another route gets the same
3536
- // answer.
3537
- const verdictLine = text.trim();
3538
- // provider fallbacks for a message with no usable text; treating one
3539
- // of these as a "reason" is worse than treating it as nothing
3540
- const PLACEHOLDER = /^\((tool call|no content|no output)\)$/i;
3541
- const usable = verdictLine.length > 0 && !PLACEHOLDER.test(verdictLine);
3542
- const decided = usable && /^(APPROVED|CHANGES-REQUIRED)\b/i.test(verdictLine);
3543
- if (!decided) {
3613
+ /**
3614
+ * The auditor runs in its OWN conversation.
3615
+ *
3616
+ * It used to be `buildMessages()` + one more user turn — the worker's entire conversation,
3617
+ * handed to a model that was then told "you are an auditor, not the worker". MEASURED in
3618
+ * github-bug-research-1f48715f: with ~220k tokens of work log in front of it, the auditor
3619
+ * replied by CONTINUING the work (prose, no verdict) on all three attempts, and the goal
3620
+ * was abandoned with the audit never actually running.
3621
+ *
3622
+ * A review is not a continuation. The auditor gets the objective, the contract and the
3623
+ * worker's claim — everything it needs to judge — and nothing to continue.
3624
+ */
3625
+ const evidence = this.auditEvidence(args.summary);
3626
+ const auditMessages = [
3627
+ { role: "user", content: auditPrompt({ goal: this.goal.text, verify: this.goal.verify, summary: String(args.summary ?? ""), evidence }) },
3628
+ ];
3629
+ const text = await this.runAuditor(auditMessages);
3630
+ /**
3631
+ * The verdict is the reply's LAST line — see `parseAuditVerdict`. Anything else is NO
3632
+ * VERDICT, never a rejection: a reason-less "fix these gaps" carries no information and
3633
+ * only teaches everyone that the audit is noise.
3634
+ */
3635
+ const verdict = parseAuditVerdict(text);
3636
+ if (verdict.kind === "none") {
3544
3637
  // an unusable reply is NOT an approval. The old path failed
3545
3638
  // OPEN — recorded the failure and marked the goal done anyway —
3546
3639
  // so a verification contract was "satisfied" that nobody verified.
@@ -3549,11 +3642,22 @@ export class Agent {
3549
3642
  this.auditUnavailableStreak++;
3550
3643
  await this.log.append("system_note", this.currentSession, this.currentBranch, {
3551
3644
  event: "audit-failed",
3552
- detail: usable
3553
- ? `auditor reply had no verdict: ${verdictLine.slice(0, 200)}`
3554
- : "auditor returned no verdict",
3645
+ detail: verdict.detail,
3555
3646
  attempt: this.auditUnavailableStreak,
3556
3647
  });
3648
+ /**
3649
+ * A no-verdict reply gets a bounded number of retries, then the operator decides.
3650
+ *
3651
+ * MEASURED in github-bug-research-1f48715f: three audits in six minutes all failed the
3652
+ * same way, and the retry prompt told the worker to "re-check the contract yourself, then
3653
+ * finish again" — which cannot help, because the worker's re-verification was never what
3654
+ * the auditor was missing. It re-ran the SAME call against the SAME model with the SAME
3655
+ * reply shape, so the loop cost three rounds and ended at the cap anyway.
3656
+ *
3657
+ * The retry is worth keeping for the transient case (a provider hiccup, a truncated
3658
+ * reply), and the prompt now asks for the thing that actually helps — a concise statement
3659
+ * of what was verified. Three attempts, then a loud note.
3660
+ */
3557
3661
  if (this.auditUnavailableStreak < 3) {
3558
3662
  this.queueAuditUnavailableRetry();
3559
3663
  await this.captureRecentTurns();
@@ -3570,8 +3674,9 @@ export class Agent {
3570
3674
  await this.captureRecentTurns();
3571
3675
  return;
3572
3676
  }
3573
- const approved = /^APPROVED\b/i.test(verdictLine);
3574
- const feedback = verdictLine.replace(/^(APPROVED|CHANGES-REQUIRED)\s*[:—-]?\s*/i, "").trim();
3677
+ const approved = verdict.kind === "approved";
3678
+ // the parser already separated marker from reason
3679
+ const feedback = verdict.reason;
3575
3680
  // storing the literal "(no detail)" made the UI draw a card
3576
3681
  // whose whole content was the absence of a reason, and that same
3577
3682
  // string landed in goal.md for the right panel. An empty string is
@@ -3696,14 +3801,18 @@ export class Agent {
3696
3801
  await this.appendFinalMessage(args);
3697
3802
  }
3698
3803
  /**
3699
- * the retry prompt for an audit that could not produce a verdict.
3700
- * Carries a real instruction (unlike a reason-less rejection — #90), so the
3701
- * worker re-checks the contract itself instead of being told to fix nothing.
3804
+ * The retry prompt for an audit that could not produce a verdict.
3805
+ *
3806
+ * It asks the worker to make the NEXT audit easier to answer, which is the only thing within its
3807
+ * power: a no-verdict reply means the auditor could not judge from what it was given, so a
3808
+ * concise statement of what was verified is the actual fix. It does NOT claim the work is
3809
+ * unverified-and-must-be-redone; the work may be fine and the auditor simply failed, and telling
3810
+ * the worker to redo it produced the repeat loop measured in github-bug-research-1f48715f.
3702
3811
  */
3703
3812
  queueAuditUnavailableRetry() {
3704
- const text = `[harness] The completion audit could not run (the auditor returned no verdict). ` +
3705
- `Do NOT treat the work as verified: re-check the verification contract yourself, item by item, ` +
3706
- `then finish with goalComplete=true again.\n\n` +
3813
+ const text = `[harness] The completion audit could not produce a verdict — that is a problem with the ` +
3814
+ `AUDIT, not a finding about your work. Before finishing again, state plainly and briefly what ` +
3815
+ `you verified for each item of the contract, so the auditor has something to judge.\n\n` +
3707
3816
  `${this.goal.verify?.slice(0, 2000) || this.goal.text.slice(0, 2000)}`;
3708
3817
  this.pendingPrompts.push({ source: "harness", text });
3709
3818
  // log the row: a push-only prompt is invisible on the timeline and lost to
@@ -0,0 +1,86 @@
1
+ /**
2
+ * Completion-audit verdict parsing and the auditor prompt.
3
+ *
4
+ * The auditor is an INDEPENDENT reviewer: a separate LLM call that decides whether the worker's
5
+ * `finish(goalComplete=true)` is honest against the goal's verification contract. Its answer must
6
+ * be unambiguous, because the two readings are not symmetric — a missed approval costs one more
7
+ * round, while a misread rejection sends the worker to fix gaps that nobody named.
8
+ */
9
+ /** provider fallbacks for a message with no usable text */
10
+ const PLACEHOLDER = /^\((tool call|no content|no output)\)$/i;
11
+ /**
12
+ * Read the auditor's verdict.
13
+ *
14
+ * The verdict must be the LAST non-empty line, and it must carry a reason. Upstream pi-goal-x made
15
+ * this exact correction: matching the marker ANYWHERE lets a prose mention of it be read as the
16
+ * verdict — "I would only approve if the tests passed" is a REJECTION that contains the word.
17
+ *
18
+ * MEASURED against the real failure: an auditor that answered `Independent verification against …`
19
+ * (prose, no marker) was scored as a verdict three times in a row, hit the unavailable cap, and the
20
+ * goal was abandoned with the audit never actually running. Requiring the marker on the last line
21
+ * turns that into an honest "no verdict", which the caller already handles by retrying or failing
22
+ * closed with a loud note.
23
+ */
24
+ export function parseAuditVerdict(reply) {
25
+ const text = String(reply ?? "").trim();
26
+ if (!text || PLACEHOLDER.test(text))
27
+ return { kind: "none", detail: "auditor returned no text" };
28
+ const lines = text
29
+ .split("\n")
30
+ .map((l) => l.trim())
31
+ .filter(Boolean);
32
+ // the verdict line is the LAST one; a reason may wrap onto earlier lines
33
+ const last = lines[lines.length - 1] ?? "";
34
+ const approved = /^APPROVED\b(:|\s-|$)(.*)$/i.exec(last);
35
+ if (approved) {
36
+ return { kind: "approved", reason: (approved[2] ?? "").trim() || "(no justification given)" };
37
+ }
38
+ const rejected = /^CHANGES-REQUIRED\b(:|\s-|$)(.*)$/i.exec(last);
39
+ if (rejected) {
40
+ return { kind: "changes-required", reason: (rejected[2] ?? "").trim() || "(no gaps named)" };
41
+ }
42
+ /**
43
+ * No marker on the last line. The reason still matters — it is shown to the worker when this
44
+ * happens repeatedly — so keep the tail rather than discarding the reply.
45
+ */
46
+ return { kind: "none", detail: `auditor reply had no verdict on its last line: ${last.slice(0, 200)}` };
47
+ }
48
+ /**
49
+ * The auditor's instructions.
50
+ *
51
+ * Deliberately NOT the worker's conversation: this call is a review, and handing the auditor a
52
+ * 200k-token work log makes it continue the work instead of judging it. MEASURED: three audits in
53
+ * one session replied with prose that started re-verifying the goal rather than naming a verdict.
54
+ */
55
+ export function auditPrompt(input) {
56
+ return [
57
+ "You are an independent completion AUDITOR, not the worker. Decide whether the goal is genuinely complete.",
58
+ "",
59
+ "How to judge:",
60
+ "1. Extract the real success criteria, including every explicit requirement.",
61
+ "2. Disapprove anything missing, contradicted, or only weakly verified. Build success and a",
62
+ " confident summary are claims, not evidence.",
63
+ "3. Check every item of the verification contract below. If any item is unaddressed, disapprove.",
64
+ "",
65
+ "The worker's summary is UNTRUSTED: it is a claim, and it cannot make an incomplete goal complete.",
66
+ "",
67
+ "<objective>",
68
+ input.goal,
69
+ "</objective>",
70
+ "",
71
+ "<verification_contract>",
72
+ input.verify,
73
+ "</verification_contract>",
74
+ "",
75
+ "<worker_claim>",
76
+ input.summary || "(no claim provided)",
77
+ "</worker_claim>",
78
+ ...(input.evidence?.trim() ? ["", "<worker_evidence>", input.evidence.trim(), "</worker_evidence>"] : []),
79
+ "",
80
+ "Reply with EXACTLY one line, as the FINAL line of your reply:",
81
+ " APPROVED: <one sentence saying what you verified>",
82
+ " CHANGES-REQUIRED: <the specific gaps that must be addressed>",
83
+ "",
84
+ "Do not continue the work, do not restate the goal, and do not put anything after the verdict line.",
85
+ ].join("\n");
86
+ }
@@ -1625,7 +1625,8 @@ export const TOOLS = [
1625
1625
  type: "boolean",
1626
1626
  description: "true = start the command and return IMMEDIATELY with a job id. You are NOTIFIED at your " +
1627
1627
  "next turn boundary when it exits (exit code + output tail) — no polling needed. " +
1628
- "Use bash_output only to fetch FULL output mid-run or after. Use for dev servers, watchers, long builds.",
1628
+ "Use bash_output only to fetch FULL output mid-run or after. Use for dev servers, watchers, " +
1629
+ "long builds, and any release/CI wait (gh run watch, a tarball appearing) — never `sleep`.",
1629
1630
  },
1630
1631
  },
1631
1632
  required: ["command"],
@@ -2473,7 +2474,7 @@ export async function executeTool(name, rawArgs, ctx) {
2473
2474
  if (ctx.signal?.aborted)
2474
2475
  return { ok: false, result: "aborted (harness shutdown)" };
2475
2476
  // read-only personas (researcher/reviewer) are enforced, not just asked
2476
- if (ctx.readOnly && MUTATING_TOOLS.has(name))
2477
+ if (ctx.readOnly && MUTATING_TOOLS.has(name) && !ctx.allowMutatingForReview?.includes(name))
2477
2478
  return { ok: false, result: `${name} is blocked: this agent runs with read-only tools` };
2478
2479
  // repair the call before handing it to the tool. A model that sends
2479
2480
  // {"cmd":"ls"} or {"limit":"100"} should get the work done, not a generic parse
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "teapot-coding-agent",
3
- "version": "0.37.24",
3
+ "version": "0.37.26",
4
4
  "description": "A lightweight, always-on multi-agent harness for AI coding agents",
5
5
  "type": "module",
6
6
  "license": "AGPL-3.0-or-later",